template substitution failure with sfinae

The following code does not compiler with GCC 5.2 or later, have not tried older compiler. Amy idea on what I am doing wrong here.

#include <iostream>
#include <string>
#include <type_traits>

template <typename T>
typename std::enable_if<std::is_enum<T>::value, T>::type
operator^=(T &lhs, const T &rhs)
{
    lhs = static_cast<T>(
                 static_cast<typename std::underlying_type<T>::type>(lhs) ^
                 static_cast<typename std::underlying_type<T>::type>(rhs));
    return lhs;
}

template <typename T, typename S>
typename std::enable_if<std::is_enum<T>::value && std::is_integral<S>::value, T>::type
operator^=(T &lhs, const S &rhs)
{
    lhs = static_cast<T>(
                 static_cast<typename std::underlying_type<T>::type>(lhs) ^
                 static_cast<typename std::underlying_type<T>::type>(rhs));
    return lhs;
}

enum Enum_t{
    Red,
    Blue,
    Green,
    Black
} ;
#define somedef 2
int main(){
    Enum_t b = Red;
    b ^= somedef;

    std::cout << std::to_string(b) << std::endl;
}

I get the following error:

g++ -std=gnu++11 main.cpp

main.cpp: In function ‘int main()’:
main.cpp:34:4: error: invalid conversion from ‘int’ to ‘Enum_t’ [-fpermissive]
b ^= somedef;