CryptDecrypt function fails with NTE_BAD_DATA in C

Question

Im trying to decrypt a .png file but for some reson CryptDecrypt returns Bad Data error. I'm new to C and Cryptography so it would be awesome if someone could tell me why is it happaning and how to fix it. Also I used CryptEncrypt function to encrypt the image and it worked well.

Here is a descryption from microsoft documentation of NTE_BAD_DATA error:

The data to be decrypted is not valid. For example, when a block cipher is used and the Final flag is FALSE, the value specified by pdwDataLen must be a multiple of the block size. This error can also be returned when the padding is found to be not valid.

Link to the CryptDecrypt function docs:

https://learn.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-cryptdecrypt

Here is full code:

#include <windows.h>
#include <wincrypt.h>
#include <stdio.h>

#define BUFFER_SIZE 1024

void displayErrorMessage(DWORD errorCode) {
    LPSTR messageBuffer = NULL;
    FormatMessageA(
        FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
        NULL,
        errorCode,
        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
        (LPSTR)&messageBuffer,
        0,
        NULL);

    if (messageBuffer != NULL) {
        printf("Error: %s\n", messageBuffer);
        LocalFree(messageBuffer);
    }
    else {
        printf("Error: Unable to get error message for code %d\n", errorCode);
    }
}


void decryptFile(const char* filePath, const char* password) {
    HANDLE hFile = CreateFileA(filePath, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
    if (hFile == INVALID_HANDLE_VALUE) {
        printf("Error opening file \n");
        return;
    }
    printf("[+] Succesfully opened the file\n");



    HCRYPTPROV hCryptProv;
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        printf("Error acquiring crypto context\n");
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully acquired crypto context\n");


    HCRYPTHASH hHash;
    if (!CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash)) {
        printf("Error creating hash\n");
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully created hash: %d\n", hHash);


    if (!CryptHashData(hHash, (const BYTE*)password, strlen(password), 0)) {
        printf("Error hashing data\n");
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully hashed the key: %s\n", password);


    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hCryptProv, CALG_AES_256, hHash, 0, &hKey)) {
        printf("Error deriving key\n");
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully derived AES key: %s\n", hKey);


    DWORD dwFileSize = GetFileSize(hFile, NULL);
    BYTE* pBuffer = (BYTE*)malloc(dwFileSize);
    DWORD dwBytesRead = 0;
    if (!ReadFile(hFile, pBuffer, dwFileSize, &dwBytesRead, NULL)) {
        printf("Error reading file\n");
        free(pBuffer);
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully saved encrypted file to a buffer: %s\n", pBuffer);

    if (!CryptDecrypt(hKey, 0, TRUE, 0, pBuffer, &dwBytesRead)) {
        printf("Error Decrypting data \n"); 
        displayErrorMessage(GetLastError());
        free(pBuffer);
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }

    SetFilePointer(hFile, 0, NULL, FILE_BEGIN);
    DWORD dwBytesWritten = 0;
    if (!WriteFile(hFile, pBuffer, dwBytesRead, &dwBytesWritten, NULL)) {
        printf("Error writing decrypted data to file\n");
    }
    else {
        printf("File decrypted successfully\n");
    }

    free(pBuffer);
    CryptDestroyKey(hKey);
    CryptDestroyHash(hHash);
    CryptReleaseContext(hCryptProv, 0);
    CloseHandle(hFile);
}

int main() {
    const char* filePath = "C:/Users/mjank/Desktop/a.png";
    const char* password = "mysecretkey";
    decryptFile(filePath, password);
    return 0;
}

As people below suggested, here is the encryption code:

#include <windows.h>
#include <wincrypt.h>
#include <stdio.h>

#define BUFFER_SIZE 1024

void encryptFile(const char* filePath, const char* password) {
    HANDLE hFile = CreateFileA(filePath, GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
    if (hFile == INVALID_HANDLE_VALUE) {
        printf("Error opening file %d\n", GetLastError());
        return;
    }

    HCRYPTPROV hCryptProv;
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        printf("Error acquiring crypto context\n");
        CloseHandle(hFile);
        return;
    }

    HCRYPTHASH hHash;
    if (!CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash)) {
        printf("Error creating hash\n");
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }

    if (!CryptHashData(hHash, (const BYTE*)password, strlen(password), 0)) {
        printf("Error hashing data\n");
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }

    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hCryptProv, CALG_AES_256, hHash, 0, &hKey)) {
        printf("Error deriving key\n");
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }

    DWORD dwFileSize = GetFileSize(hFile, NULL);
    BYTE* pBuffer = (BYTE*)malloc(dwFileSize);
    DWORD dwBytesRead = 0;
    if (!ReadFile(hFile, pBuffer, dwFileSize, &dwBytesRead, NULL)) {
        printf("Error reading file\n");
        free(pBuffer);
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully saved plaintext file to a buffer: %s\n", pBuffer);



    DWORD dwBlockLen = 0;
    if (!CryptEncrypt(hKey, NULL, TRUE, 0, pBuffer, &dwBlockLen, dwFileSize)) {
        printf("Error encrypting data %d\n", GetLastError());
        free(pBuffer);
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        CloseHandle(hFile);
        return;
    }
    printf("[+] Succesfully encrypted the buffer: %s\n", pBuffer);


    SetFilePointer(hFile, 0, NULL, FILE_BEGIN);
    DWORD dwBytesWritten = 0;
    if (!WriteFile(hFile, pBuffer, dwBytesRead, &dwBytesWritten, NULL)) {
        printf("Error writing encrypted data to file\n");
    }
    else {
        printf("File encrypted successfully\n");
    }

    free(pBuffer);
    CryptDestroyKey(hKey);
    CryptDestroyHash(hHash);
    CryptReleaseContext(hCryptProv, 0);
    CloseHandle(hFile);
}



int main() {

    const char* filePath = "C:/Users/mjank/Desktop/a.png";
    const char* key = "mysecretkey";
    encryptFile(filePath, key);
    return 0;
}

Answer 1

Your encryption code is buggy:

• First, the plaintext size is specified incorrectly in CryptEncrypt() (see parameter pdwDataLen).
• Second, the buffer used is too small (see parameter dwBufLen). The current code uses the size of the plaintext as buffer size and therefore implicitly assumes that the ciphertext has the same size as the plaintext. However, this is incorrect, as the ciphertext is larger than the plaintext due to the padding. Therefore, the size of the ciphertext must first be determined before allocating the buffer (see parameter pbData).

The bugs cause the alleged ciphertext being identical to the plaintext apart from the first block, i.e. no ciphertext is actually generated at all. This is the reason for the failure of the decryption. It is also problematic that the bugs in conjunction with the design of using the same file for plaintext and ciphertext reveal the plaintext (apart from the first block).

A possible fix (for simplicity without exception handling and memory release):

...
DWORD dwFileSize = GetFileSize(hFile, NULL);
DWORD dwBytesCt = dwFileSize;
CryptEncrypt(hKey, NULL, TRUE, 0, NULL, &dwBytesCt, dwFileSize); // Fix 1: determine size of ciphertext (dwBytesCt)

BYTE* pBuffer = (BYTE*)malloc(dwBytesCt); // Fix 2: allocate buffer of sufficient size for plaintext AND ciphertext
DWORD dwBytesRead = 0;
ReadFile(hFile, pBuffer, dwFileSize, &dwBytesRead, NULL);

CryptEncrypt(hKey, NULL, TRUE, 0, pBuffer, &dwBytesRead, dwBytesCt); // FIX 3: pass size of plaintext (dwBytesRead) and size of buffer (dwBytesCt)
...

When decrypting, as already mentioned in the comment, the padding must be removed, e.g. by calling SetEndOfFile() (after WriteFile()).

With these changes, encryption and decryption work for me.