Calling EVP_OpenInit fires terminated by signal 11 error

I have a couple of functions that encrypt and decrypt a file using the public key from a certificate.

Unfortunately when the code is ran it terminates with a signal 11 error when I run the EVP_OpenInit function. Investigation has led me understand that this is due to a memory access error. So I checked all the pointers and they are pointing to same memory locations which they pointed to to for the Seal routines.

The entry point to the code is via Google Test:

TEST_F(EncryptTest, TestEncryptConfigFileWithRSAPublicKey)
{
    // extract the key from certificate
    const std::string cert_file = "config/certificate.pem.crt";
    EVP_PKEY* pkey = ekko::ExtractRSAPublicKey(cert_file);
    ASSERT_TRUE(pkey);

    unsigned char* encryptedMessage = NULL;
    unsigned char* decryptedMessage = NULL;
    unsigned char* encryptedKey;
    unsigned char* iv;
    size_t encryptedMessageLength = 0;
    size_t decryptedMessageLength = 0;
    size_t encryptedKeyLength = 0;

    // load in the example data
    std::ifstream config_file("config/config.cfg.example");
    ekko::secure_string config_data((std::istreambuf_iterator<char>(config_file)), std::istreambuf_iterator<char>());

    std::cout << "Encrypting " << config_data.length() << " bytes..." << std::endl;

    encryptedMessageLength = ekko::Encrypt((const unsigned char*)config_data.c_str(), config_data.size()+1,
        &encryptedMessage, &encryptedKey, &encryptedKeyLength, &iv, pkey);

    // write the encrypted data to file
    std::ofstream out("config/config.cfg.example.enc");
    out.write((char*)encryptedMessage, encryptedMessageLength);


std::cout << "EncryptTest" << std::endl;
std::cout << "EKL " << encryptedKeyLength << std::endl;
std::cout << "EK " << &encryptedKey << std::endl;
std::cout << "IV " << &iv << std::endl;
std::cout << "PK " << pkey << std::endl << std::endl;


    // load in the encrypted data
    std::ifstream encrypted_file("config/config.cfg.example.enc");
    ekko::secure_string encrypted_data((std::istreambuf_iterator<char>(encrypted_file)), std::istreambuf_iterator<char>());

    std::cout << "Decrypting " << encrypted_data.length() << " bytes..." << std::endl;

    // decrypt it
    decryptedMessageLength = ekko::Decrypt((unsigned char*)encryptedMessage, encryptedMessageLength,
        &decryptedMessage, &encryptedKey, &encryptedKeyLength, &iv, pkey);

    free(iv);
    free(encryptedKey);
    free(decryptedMessage);
    free(encryptedMessage);
    EVP_PKEY_free(pkey);

    // compare
    EXPECT_STREQ(config_data.c_str(), (char*)decryptedMessage);
}

All pretty standard stuff: Get the public key, load in a file, encrypt it, store the file, load the encrypted file, decrypt it and compare with original.

There are no surprises in the ExtractRSAPublicKey function:

EVP_PKEY* ExtractRSAPublicKey(const std::string& cert_filename) {
    EVP_PKEY*   pkey = NULL;

    BIO_ptr     certbio(BIO_new(BIO_s_file()), BIO_free);
    X509*       cert    = NULL;

    // read in the certificate
    BIO_read_filename(certbio.get(), cert_filename.c_str());
    if (!(cert = PEM_read_bio_X509(certbio.get(), NULL, 0, NULL))) {
        throw std::runtime_error("Error loading cert into memory");
    }

    // extract certificate public key data
    pkey = EVP_PKEY_new();
    if ((pkey = X509_get_pubkey(cert)) == NULL) {
        throw std::runtime_error("Error getting public key from certificate");
    }

    X509_free(cert);

    return pkey;
}

Create a PKey and return it. It gets freed at the end of the unit test.

Likewise nothing special about the encryption routine.

int Encrypt(const unsigned char *message, size_t messageLength,
    unsigned char **encryptedMessage, unsigned char **encryptedKey,
    size_t *encryptedKeyLength, unsigned char **iv,
    EVP_PKEY* publicKey) {

    EVP_CIPHER_CTX_ptr ctx(EVP_CIPHER_CTX_new(), ::EVP_CIPHER_CTX_free);

    // Allocate memory for everything
    size_t encryptedMessageLength = 0;
    size_t blockLength = 0;

    *encryptedKey = (unsigned char*)malloc(EVP_PKEY_size(publicKey));
    *iv = (unsigned char*)malloc(EVP_MAX_IV_LENGTH);

    if (*encryptedKey == NULL || *iv == NULL) {
        std::runtime_error("encrypted key or iv is NULL");
    }

    *encryptedMessage = (unsigned char*)malloc(messageLength + EVP_MAX_IV_LENGTH);
    if (encryptedMessage == NULL) {
        std::runtime_error("unable to allocate encryptedMessage");
    }

    // Encrypt it!
    if (!EVP_SealInit(ctx.get(), EVP_aes_256_cbc(), encryptedKey, (int*)encryptedKeyLength, *iv, &publicKey, 1)) {
        std::runtime_error("EVP_SealInit failed");
    }

    if (!EVP_SealUpdate(ctx.get(), *encryptedMessage + encryptedMessageLength, (int*)&blockLength,
        (const unsigned char*)message, (int)messageLength)) {
        std::runtime_error("EVP_SealUpdate failed");
    }
    encryptedMessageLength += blockLength;

    if (!EVP_SealFinal(ctx.get(), *encryptedMessage + encryptedMessageLength, (int*)&blockLength)) {
        std::runtime_error("EVP_SealFinal failed");
    }
    encryptedMessageLength += blockLength;

std::cout << "Encrypt" << std::endl;
std::cout << "EKL " << *encryptedKeyLength << std::endl;
std::cout << "EK " << encryptedKey << std::endl;
std::cout << "IV " << iv << std::endl;
std::cout << "PK " << publicKey << std::endl << std::endl;

    return (int)encryptedMessageLength;
}

Not too happy about the malloc and stuff, but once I have it working I can look to improve it using containers.

Finally the decryption function. The line that fails is the EVP_OpenInit one. The std::cout in the previous code blocks all give the same values for locations and size (where appropriate).

int Decrypt(unsigned char *encryptedMessage, size_t encryptedMessageLength,
    unsigned char **decryptedMessage, unsigned char **encryptedKey,
    size_t *encryptedKeyLength, unsigned char **iv,
    EVP_PKEY* publicKey) {

    EVP_CIPHER_CTX_ptr ctx(EVP_CIPHER_CTX_new(), ::EVP_CIPHER_CTX_free);

    // Allocate memory for everything
    size_t decryptedMessageLength = 0;
    size_t blockLength = 0;

    *decryptedMessage = (unsigned char*)malloc(encryptedMessageLength);
    if (*decryptedMessage == NULL) {
        std::runtime_error("decryptedMessage is NULL");
    }

std::cout << "Decrypt" << std::endl;
std::cout << "CTX " << ctx.get() << std::endl;
std::cout << "EKL " << *encryptedKeyLength << std::endl;
std::cout << "EK " << encryptedKey << std::endl;
std::cout << "IV " << iv << std::endl;
std::cout << "PK " << publicKey << std::endl << std::endl;

    // Decrypt it!
    if (!EVP_OpenInit(ctx.get(), EVP_aes_256_cbc(), *encryptedKey, (int)*encryptedKeyLength, *iv, publicKey)) {
        std::runtime_error("EVP_OpenInit failed");
    }

    if (!EVP_OpenUpdate(ctx.get(), *decryptedMessage + decryptedMessageLength, (int*)&blockLength, 
        encryptedMessage, (int)encryptedMessageLength)) {
        std::runtime_error("EVP_OpenUpdate failed");
    }
    decryptedMessageLength += blockLength;

    if (!EVP_OpenFinal(ctx.get(), *decryptedMessage + decryptedMessageLength, (int*)&blockLength)) {
        std::runtime_error("EVP_OpenFinal failed");
    }
    decryptedMessageLength += blockLength;

    return (int)decryptedMessageLength;
}

So what is wrong? Sample output:

Encrypting 49481 bytes...
Encrypt
EKL 256
EK 0x7fff70537d00
IV 0x7fff70537cf8
PK 0x55c036652c70

EncryptTest
EKL 256
EK 0x7fff70537d00
IV 0x7fff70537cf8
PK 0x55c036652c70

Decrypting 49488 bytes...
Decrypt
CTX 0x55c0366510c0
EKL 256
EK 0x7fff70537d00
IV 0x7fff70537cf8
PK 0x55c036652c70

find: ‘build/tests/encrypt-test’ terminated by signal 11