Fix decryption of RC4-40 files.

This commit is contained in:
Michael R Sweet
2025-04-13 08:37:24 -04:00
parent 4219b8fd77
commit 20dd2a6d28
3 changed files with 28 additions and 11 deletions

View File

@@ -490,20 +490,20 @@ _pdfioCryptoMakeReader(
case PDFIO_ENCRYPTION_RC4_40 :
// Copy the key data for the MD5 hash.
memcpy(data, file_key, 16);
data[16] = (uint8_t)obj->number;
data[17] = (uint8_t)(obj->number >> 8);
data[18] = (uint8_t)(obj->number >> 16);
data[19] = (uint8_t)obj->generation;
data[20] = (uint8_t)(obj->generation >> 8);
memcpy(data, file_key, 5);
data[5] = (uint8_t)obj->number;
data[6] = (uint8_t)(obj->number >> 8);
data[7] = (uint8_t)(obj->number >> 16);
data[8] = (uint8_t)obj->generation;
data[9] = (uint8_t)(obj->generation >> 8);
// Hash it...
_pdfioCryptoMD5Init(&md5);
_pdfioCryptoMD5Append(&md5, data, sizeof(data));
_pdfioCryptoMD5Append(&md5, data, 10);
_pdfioCryptoMD5Finish(&md5, digest);
// Initialize the RC4 context using 40 bits of the digest...
_pdfioCryptoRC4Init(&ctx->rc4, digest, 5);
// Initialize the RC4 context using 80 bits of the digest...
_pdfioCryptoRC4Init(&ctx->rc4, digest, 10);
*ivlen = 0;
return ((_pdfio_crypto_cb_t)_pdfioCryptoRC4Crypt);