Storing a pointer to a template type of the same template class

Let's say we have the following:

template<typename A, typename B>
class Foo {
private:
    A m_a;
    B m_b;
    Foo<A,B>* m_pFoo;

public:
    Foo( A a, B b, Foo<A,B>* pFoo = nullptr );
};

With this class I can save any pFoo into m_pFoo as long as the instantiation of types match as such:

 int main() {
     Foo<int, int> foo1( 3, 5 );
     Foo<int, int> foo2( 2, 4, &foo1 ); // This Works
     Foo<int, int> foo3( 5, 7, &foo2 ); // This Still Works 

     Foo<double, int> foo4( 2.4, 5 );
     Foo<double, int> foo5( 7.5, 2, &foo4 ); // This Works
     Foo<double, int> foo6( 9.2, 6, &foo5 ); // This Still Works

     // Compiler Error - Can not deduce template arguments
     Foo<double, int> foo7( 3.7, 2, &foo1 ); // Doesn't Work

     return 0;
 }

In a previous question I demonstrated a similar problem as this and my initial question was how to pass a pointer to a class template type into the same class template constructor, however one of the responses I received regarding this was that passing in the pointer is not the problem, but the storage is. So with this post my new question becomes this:

How would I be able to have the same class template with the same or similar constructor as above where each of these types:

Foo<short, short>           ssFoo;
Foo<short, int>             siFoo;
Foo<short, int64_t>         si64Foo;
Foo<short, unsigned>        suFoo;
Foo<short, float>           sfFoo;
Foo<short, double>          sdFoo;
Foo<short, long>            slFoo;
Foo<short, long long>       sllFoo;

Foo<int, short>             isFoo;
Foo<int, int>               iiFoo;
Foo<int, int64_t>           ii64Foo;
Foo<int, unsigned>          iuFoo;
Foo<int, float>             ifFoo;
Foo<int, double>            idFoo;
Foo<int, long>              ilFoo;
Foo<int, long long>         illFoo;

Foo<int64_t, short>        i64sFoo;
Foo<int64_t, int>          i64iFoo;
Foo<int64_t, int64_t>      i64i64Foo;
Foo<int64_t, unsigned>     i64uFoo;
Foo<int64_t, float>        i64fFoo;
Foo<int64_t, double>       i64dFoo;
Foo<int64_t, long>         i64lFoo;
Foo<int64_t, long long>    i64llFoo;

Foo<unsigned, short>        usFoo;
Foo<unsigned, int>          uiFoo;
Foo<unsigned, int64_t>      ui64Foo;
Foo<unsigned, unsigned>     uuFoo;
Foo<unsigned, float>        ufFoo;
Foo<unsigned, double>       udFoo;
Foo<unsigned, long>         ulFoo;
Foo<unsigned, long long>    ullFoo;

Foo<float, short>           fsFoo;
Foo<float, int>             fiFoo;
Foo<float, int64_t>         fi64Foo;
Foo<float, unsigned>        fuFoo;
Foo<float, float>           ffFoo;
Foo<float, double>          fdFoo;
Foo<float, long>            flFoo;
Foo<float, long long>       fllFoo;

Foo<double, short>          dsFoo;
Foo<double, int>            diFoo;
Foo<double, int64_t>        di64Foo;
Foo<double, unsigned>       duFoo;
Foo<double, float>          dfFoo;
Foo<double, double>         ddFoo;
Foo<double, long>           dlFoo;
Foo<double, long long>      dllFoo;

Foo<long, short>            lsFoo;
Foo<long, int>              liFoo;
Foo<long, int64_t>          li64Foo;
Foo<long, unsigned>         luFoo;
Foo<long, float>            lfFoo;
Foo<long, double>           ldFoo;
Foo<long, long>             llFoo;
Foo<long, long long>        l_llFoo;

Foo<long long, short>       llsFoo;
Foo<long long, int>         lliFoo;
Foo<long long, int64_t>     lli64Foo;
Foo<long long, unsigned>    lluFoo;
Foo<long long, float>       llfFoo;
Foo<long long, double>      lldFoo;
Foo<long long, long>        ll_lFoo;
Foo<long long, long long>   ll_llFoo;

Are all valid types to store within the class template upon construction where the address of the previous instance is being passed into the new instance's constructor? Also; how would I be able to prevent this class from accepting anything that is a custom or user defined object or character, string types, enumerations and Boolean types? I would like the typenames being passed into the class templates argument list as being numerical types only.