Merge pull request #344 from abrignoni/feat/adcrypt

Read FTK Imager AD-encrypted images with their password

Author: abrignoni

admin/docs/raw_image_input.md

@@ -40,7 +40,7 @@ or a 4Kn drive, has its partitions counted in 4096-byte sectors.
   one, a damaged encrypted header and a `.dmgpart` opened on its own that must be
   refused.
 
-## Encrypted Apple disk images
+## Encrypted images
 
 An Apple disk image encrypted with a password (`hdiutil -encryption`: a `.dmg`, a
 split one, a `.sparseimage` or a sparse bundle) opens with that password, which the
@@ -57,6 +57,13 @@ taken as an argument's value, which would put it in the process list and the she
 history. An image that will not open for another reason (a damaged header, no cipher
 package) is reported by name rather than asked about again.
 
+An E01, SMART or raw (dd) set FTK Imager encrypted with AD encryption opens the same
+way, with the same prompts and options. Every file of such a set is encrypted and only
+the first carries the header, so an E01 or SMART set is opened from its first file and
+a raw set from any of its numbered files (`.001`, `.002`, ...); the reader decrypts and
+joins them itself, and the run log names the set as the reader reports it rather than
+as a split image. `needs_password(path)` answers for both kinds.
+
 ## How it reads
 
 `FileSeekerRaw` opens the image through the reader's `open_image()` (which joins

admin/test/scripts/test_raw_image_seeker.py

@@ -10,7 +10,8 @@
 The E01, AFF, AFD, Apple disk image and split-segment paths are exercised by
 wrapping the same NTFS fixture at test time: small EWF, AFF, UDIF and sparse image
 writers live in this file (copied from ewfprobe's own test suite, MIT), and a split
-set is the raw image cut into numbered pieces.
+set is the raw image cut into numbered pieces. FTK Imager's AD encryption is written
+here too, around a raw set and an E01 set, with the cipher library's CTR mode.
 The expected values are the fixture's, written out, never read back from the
 seeker.
 """
@@ -339,6 +340,38 @@ def write_encrypted_sparsebundle(folder, data, band=1 << 20):
     return folder
 
 
+AD_PASSWORD = 'raw-image-ad-password'
+
+
+def write_adcrypt(plain_files, out_files, password=AD_PASSWORD):
+    """Each of ``plain_files`` encrypted into ``out_files`` as FTK Imager's AD
+    encryption writes a set: one AES-256 key, the 512-byte header in the first file
+    only, file i in CTR mode from counter i << 64, the counter little endian, and the
+    key encrypted under PBKDF2-HMAC-SHA1 of the password's SHA-512."""
+    import hmac  # pylint: disable=import-outside-toplevel
+    from Crypto.Cipher import AES  # pylint: disable=import-outside-toplevel
+    from Crypto.Util import Counter  # pylint: disable=import-outside-toplevel
+
+    def ctr(key, data, first):
+        counter = Counter.new(128, initial_value=first, little_endian=True)
+        return AES.new(key, AES.MODE_CTR, counter=counter).encrypt(data)
+
+    rng = __import__('random').Random(9)
+    file_key, salt = rng.randbytes(32), rng.randbytes(16)
+    made = hashlib.pbkdf2_hmac('sha1', hashlib.sha512(password.encode()).digest(), salt,
+                               1000, 32)
+    wrapped = ctr(made, file_key, 0)
+    header = (struct.pack('<8sIIhhh2sIIIIII', b'ADCRYPT\x00', 1, 512, -1, -1, -1,
+                          b'\x00\x00', 3, 2, 1000, 16, 32, 64)
+              + salt + wrapped + hmac.new(made, wrapped, 'sha512').digest()).ljust(512, b'\0')
+    for index, (src, dst) in enumerate(zip(plain_files, out_files)):
+        with open(src, 'rb') as handle:
+            body = ctr(file_key, handle.read(), index << 64)
+        with open(dst, 'wb') as handle:
+            handle.write((header if index == 0 else b'') + body)
+    return out_files
+
+
 def with_mbr(volume_bytes, start_lba=2048):
     """The volume behind an MBR whose one entry covers it in full, as a disk carries it."""
     entry = bytearray(16)
@@ -948,6 +981,68 @@ def test_a_damaged_encrypted_dmg_is_reported_not_asked_about_again(self):
             with mock.patch.dict(os.environ, {'RAW_IMAGE_TEST_PW': ENC_PASSWORD}):
                 raw_image.cli_image_password(path, password_env='RAW_IMAGE_TEST_PW')
 
+    def _ad_raw_set(self):
+        """The NTFS fixture as a raw set of two files FTK Imager encrypted."""
+        with open(self.ntfs, 'rb') as handle:
+            data = handle.read()
+        folder = tempfile.mkdtemp(prefix='raw_image_ad_', dir=self.work)
+        half = len(data) // 2 // 512 * 512
+        plain = []
+        for index, piece in enumerate((data[:half], data[half:]), start=1):
+            plain.append(os.path.join(folder, f'plain.{index:03d}'))
+            with open(plain[-1], 'wb') as handle:
+                handle.write(piece)
+        out = [os.path.join(folder, f'evidence.{index:03d}') for index in (1, 2)]
+        return write_adcrypt(plain, out)
+
+    def test_an_ad_encrypted_raw_set_reads_with_its_password_from_either_file(self):
+        first, second = self._ad_raw_set()
+        self.assertTrue(raw_image.needs_password(first))
+        self.assertTrue(raw_image.needs_password(second))
+        seeker = FileSeekerRaw(second, self.data, password=AD_PASSWORD)
+        self.addCleanup(seeker.cleanup)
+        text = self.log.text()
+        self.assertIn('a raw (dd) image of 2 segments, joined by the reader: evidence.001 '
+                      '.. evidence.002, encrypted (AES-256-CTR) and opened with its password',
+                      text)
+        self.assertNotIn('one segment of a split image', text)
+        self.assertNotIn('segments joined in order', text)
+        self.assertEqual(seeker.name_list, self._seeker(self.ntfs).name_list)
+        for rel in ('many/file_0007.txt', 'many/file_0400.txt'):
+            found = seeker.search(f'*/{rel}')
+            self.assertEqual(_sha256(found[0]), self.ntfs_hashes[rel])
+        self.assertNotIn(AD_PASSWORD, repr(vars(seeker)))
+
+    def test_an_ad_encrypted_e01_set_reads_with_its_password(self):
+        with open(self.ntfs, 'rb') as handle:
+            data = handle.read()
+        folder = tempfile.mkdtemp(prefix='raw_image_ad_e01_', dir=self.work)
+        plain = write_ewf(folder, 'plain', data, chunks_per_segment=200)
+        self.assertGreater(len(plain), 1)
+        out = write_adcrypt(plain, [p.replace('plain.', 'evidence.') for p in plain])
+        self.assertTrue(raw_image.needs_password(out[0]))
+        seeker = FileSeekerRaw(out[0], self.data, password=AD_PASSWORD)
+        self.addCleanup(seeker.cleanup)
+        self.assertIn(f'an EWF acquisition of {len(out)} segments', self.log.text())
+        found = seeker.search('*/many/file_0007.txt')
+        self.assertEqual(_sha256(found[0]), self.ntfs_hashes['many/file_0007.txt'])
+
+    def test_an_ad_encrypted_set_without_its_password_is_refused_as_such(self):
+        first, _second = self._ad_raw_set()
+        with self.assertRaises(qnxprobe.ImagePasswordError) as caught:
+            FileSeekerRaw(first, self.data)
+        self.assertFalse(caught.exception.wrong)
+        with self.assertRaises(qnxprobe.ImagePasswordError) as caught:
+            FileSeekerRaw(first, self.data, password='not it')
+        self.assertTrue(caught.exception.wrong)
+        with mock.patch.object(raw_image.sys, 'stdin', None):
+            with self.assertRaisesRegex(ValueError, 'evidence.001 is an acquisition FTK Imager '
+                                        'encrypted with AD encryption and opens only'):
+                raw_image.cli_image_password(first)
+        with mock.patch.dict(os.environ, {'RAW_IMAGE_TEST_PW': AD_PASSWORD}):
+            self.assertEqual(raw_image.cli_image_password(first, password_env='RAW_IMAGE_TEST_PW'),
+                             AD_PASSWORD)
+
     def test_an_l01_is_refused_as_logical_evidence(self):
         folder = tempfile.mkdtemp(prefix='raw_image_l01_', dir=self.work)
         path = os.path.join(folder, 'evidence.L01')

dleapp.py

@@ -220,12 +220,13 @@ def main():
                         help='Path to base output folder (this must exist)')
     parser.add_argument('-i', '--input_path', required=False, action="store", help='Path to input file/folder')
     parser.add_argument('--image_password_file', required=False, action="store",
-                        help='For an encrypted Apple disk image (-t raw): read its password '
-                             'from the first line of this file')
+                        help='For an encrypted image (-t raw; an Apple disk image or an FTK Imager '
+                             'AD-encrypted set): read its password from the first line '
+                             'of this file')
     parser.add_argument('--image_password_env', required=False, action="store",
-                        help='For an encrypted Apple disk image (-t raw): take its password '
-                             'from this environment variable. Without either, it is asked '
-                             'for at a terminal')
+                        help='For an encrypted image (-t raw): take its password from this '
+                             'environment variable. Without either, it is asked for at a '
+                             'terminal')
     parser.add_argument('-w', '--wrap_text', required=False, action="store_false", default=True,
                         help='Do not wrap text for output of data files')
     parser.add_argument('-m', '--load_profile', required=False, action="store", help="Path to DLEAPP Profile file (.rlprofile).")

scripts/raw_image.py

@@ -8,9 +8,10 @@
 ``scripts/vendor/`` (qnxprobe, with ewfprobe beside it for the acquisitions).
 EnCase logical evidence (``.L01``) holds copies of files rather than a disk, and the
 reader refuses it with a message saying so. An Apple disk image encrypted with a
-password opens with that password, which the GUI asks for and the command line
-takes from ``--image_password_file`` or ``--image_password_env``, or asks for at a
-terminal; it is checked against the image before the run and never stored. The reader
+password, and an E01, SMART or raw set FTK Imager encrypted with AD encryption, open
+with that password, which the GUI asks for and the command line takes from
+``--image_password_file`` or ``--image_password_env``, or asks for at a terminal; it
+is checked against the image before the run and never stored. The reader
 finds the partitions, identifies each volume by its own on-disk structure and
 walks its directory tree; this module turns that into the same contract the zip
 seeker offers: a list of member names to match artifact patterns against, and a
@@ -104,9 +105,17 @@ def names_an_image_folder(path):
 
 
 def needs_password(path):
-    """True when ``path`` is an Apple disk image encrypted with a password, which the
-    reader opens only with that password."""
-    return qnxprobe.acquisition_format(path) == 'DMG_ENCRYPTED'
+    """True when ``path`` is an image the reader opens only with its password: an
+    encrypted Apple disk image, or an E01, SMART or raw set FTK Imager encrypted with
+    AD encryption (a raw set from any of its numbered files)."""
+    return qnxprobe.needs_password(path)
+
+
+def _encrypted_kind(path):
+    """What an encrypted image is, as a sentence names it."""
+    if qnxprobe.acquisition_format(path) == 'AD_ENCRYPTED':
+        return 'an acquisition FTK Imager encrypted with AD encryption'
+    return 'an encrypted Apple disk image'
 
 
 def _open_errors():
@@ -127,7 +136,7 @@ def password_opens(path, password):
 
 
 def cli_image_password(path, password_file=None, password_env=None):
-    """The password for an encrypted Apple disk image, for the command line: the
+    """The password for an encrypted image, for the command line: the
     first line of ``password_file``, else the environment variable ``password_env``,
     else, at a terminal, asked for (three tries). None for an image that needs none.
 
@@ -155,7 +164,7 @@ def cli_image_password(path, password_file=None, password_env=None):
                 print('That password does not open the image.', file=sys.stderr)
             raise ValueError(f'the password does not open {name}')
         else:
-            raise ValueError(f'{name} is an encrypted Apple disk image and opens only '
+            raise ValueError(f'{name} is {_encrypted_kind(path)} and opens only '
                              f'with its password; give it with --image_password_file '
                              f'or --image_password_env, or run at a terminal to be '
                              f'asked for it')
@@ -167,12 +176,12 @@ def cli_image_password(path, password_file=None, password_env=None):
 
 
 def ask_image_password(parent, path):
-    """The password for an encrypted Apple disk image, asked for in a dialog over
+    """The password for an encrypted image, asked for in a dialog over
     ``parent`` until it opens the image. None when the examiner cancels, or when the
     image will not open for another reason, which is shown."""
     from tkinter import messagebox, simpledialog  # pylint: disable=import-outside-toplevel
     name = os.path.basename(os.path.normpath(path))
-    prompt = f'{name} is an encrypted Apple disk image. Its password:'
+    prompt = f'{name} is {_encrypted_kind(path)}. Its password:'
     while True:
         password = simpledialog.askstring('Encrypted disk image', prompt, show='*',
                                           parent=parent)
@@ -278,7 +287,7 @@ def __init__(self, image_path, data_folder, password=None):
         self._entries = {}
         self._image = None
         try:
-            # the password opens an encrypted Apple disk image and is not kept
+            # the password opens an encrypted image and is not kept
             self._open(password)
         except BaseException:
             # A failed or interrupted (Ctrl-C on a slow walk) build never binds an
@@ -294,15 +303,18 @@ def _open(self, password=None):
         logfunc(f'Reading {name} in place with the vendored qnxprobe '
                 f'{qnxprobe.QNXPROBE_VERSION}; only the files an artifact asks for '
                 f'leave the image.')
-        sibling = split_image_sibling(self.image_path)
+        # an acquisition's reader joins its own files; a raw set FTK Imager encrypted
+        # is numbered like a split image and its files are ciphertext until then
+        acquisition = qnxprobe.acquisition_format(self.image_path)
+        sibling = None if acquisition else split_image_sibling(self.image_path)
         if sibling:
             logfunc(f'{name} is one segment of a split image: '
                     f'{os.path.basename(sibling)} sits beside it. The reader joins '
                     f'every segment of the set in order.')
         # A set with a hole in its numbering raises SplitImageError here, by
         # name, and the run stops with that message rather than reading half a
         # disk as though it were whole.
-        segments = qnxprobe.split_segments(self.image_path)
+        segments = [] if acquisition else qnxprobe.split_segments(self.image_path)
         self._image = qnxprobe.open_image(self.image_path, segments, password=password)
         size = qnxprobe.image_size(self._image)
         if segments: