reference data member to instantiations of template

I have a base class which is a template, and two derived classes like below,

template <class parm>
class Base{
};

class Derived1: public Base<Derived1>
{};

class Derived2: public Base<Derived2>
{};

Now I have another class holding a reference to Derived1 object or Derived2 object depending on which is passed into the constructor, but I don't know the syntax, is it achievable in C++?

struct Observer{

  // I want to hold a reference to Derived1 or Derived2 based on 
  // which is passed into the constructor
  template <class parm>
  Base<parm> & derived_reference;

  Observer(Derived1 & d1):derived_reference(d1) // what's the right syntax here?
 {}

  Observer(Derived2 & d2):derived_reference(d2)
  {}
};