Fix PDF decryption of ancient 40-bit RC4 with R=2

CHANGELOG.md

@@ -99,4 +99,4 @@ This is v10.0.9, a release candidate for v10.1.0. I don't expect there to be maj
 
 ## Fixes on master (not yet released):
 
-- (none)
+- Fix a bug where decrypting a 40-bit RC4 pdf with R=2 didn't work.

DeDRM_plugin/ineptpdf.py

@@ -1369,11 +1369,12 @@ class PDFDocument(object):
             keylen = len(key)
             iv = bytes([0x00]*keylen)
 
+        aes = AES.new(key, AES.MODE_CBC, iv)
+
         if not encrypt:
-            plaintext = AES.new(key,AES.MODE_CBC,iv, True).decrypt(data)
+            plaintext = aes.decrypt(data)
             return plaintext
         else:
-            aes = AES.new(key, AES.MODE_CBC, iv, False)
             new_data = bytes(data * repetitions)
             crypt = aes.encrypt(new_data)
             return crypt
@@ -1478,14 +1479,23 @@ class PDFDocument(object):
             EncMetadata = b'True'
         if (EncMetadata == ('False' or 'false') or V < 4) and R >= 4:
             hash.update(codecs.decode(b'ffffffff','hex'))
+
+        # Finish hash:
+        hash = hash.digest()
+        
         if R >= 3:
             # 8
             for _ in range(50):
-                hash = hashlib.md5(hash.digest()[:length//8])
+                hash = hashlib.md5(hash[:length//8]).digest()
-        key = hash.digest()[:length//8]
+        if R == 2:
+            # R=2 only uses first five bytes.
+            key = hash[:5]
+        else:
+            key = hash[:length//8]
+
         if R == 2:
             # Algorithm 3.4
-            u1 = ARC4.new(key).decrypt(password)
+            u1 = ARC4.new(key).decrypt(self.PASSWORD_PADDING)
         elif R >= 3:
             # Algorithm 3.5
             hash = hashlib.md5(self.PASSWORD_PADDING) # 2
@@ -1498,6 +1508,7 @@ class PDFDocument(object):
                     k = b''.join(bytes([c ^ i]) for c in key )
                 x = ARC4.new(k).decrypt(x)
             u1 = x+x # 32bytes total
+        
         if R == 2:
             is_authenticated = (u1 == U)
         else: