Observer Pattern implementation without reciprocal references and smart pointers

I'm trying to implement the Observer pattern, but i don't want observers being responsible for my program safety by maintaining the reference list in ObservableSubject.

I have come up with an idea to use use pointer references in ObservableSubject.

My question are: Is implementation described above and attempted below possible? And what is happening in my program, how do i prevent dereferencing trash?

My solution attempt:

// Example program
#include <iostream>
#include <string>
#include <vector>

class ObserverInterface {
public:
    virtual ~ObserverInterface() {};
    virtual void handleMessage() = 0;
};

class ObservableSubject
{
    std::vector<std::reference_wrapper<ObserverInterface*>> listeners;

public:
    void addListener(ObserverInterface* obs)
    {
        if (&obs)
        {
            // is this a reference to the copied ptr?
            // still, why doesnt my guard in notify protect me
            this->listeners.push_back(obs);
        }
    }

    void removeListener(ObserverInterface* obs)
    {
        // todo
    }

    void notify()
    {
        for (ObserverInterface* listener : this->listeners)
        {
            if (listener)
            {
                listener->handleMessage();
            }
        }
    }
};

class ConcreteObserver : public ObserverInterface {
    void handleMessage()
    {
        std::cout << "ConcreteObserver: I'm doing work..." << std::endl;
    }
};

int main()
{
    ObservableSubject o;

    {
        ConcreteObserver c;
        o.addListener(&c);
    }

    o.notify();

    std::cin.get();
}