derivepassphrase.git

1) # SPDX-FileCopyrightText: 2024 Marco Ricci <software@the13thletter.info>

2) #
3) # SPDX-License-Identifier: MIT
4) 

5) """Exporter for the vault "storeroom" configuration format.
6) 
7) The "storeroom" format is the experimental format used in alpha and beta
8) versions of vault beyond v0.3.0.  The configuration is stored as
9) a separate directory, which acts like a hash table (i.e. has named
10) slots) and provides an impure quasi-filesystem interface.  Each hash
11) table entry is separately encrypted and authenticated.  James Coglan
12) designed this format to avoid concurrent write issues when updating or
13) synchronizing the vault configuration with e.g. a cloud service.
14) 

15) The public interface is the [`export_storeroom_data`][] function.
16) Multiple *non-public* functions are additionally documented here for
17) didactical and educational reasons, but they are not part of the module
18) API, are subject to change without notice (including removal), and
19) should *not* be used or relied on.

20) 
21) """

22) 

23) from __future__ import annotations
24) 

25) import base64

26) import fnmatch

27) import json
28) import logging
29) import os
30) import os.path
31) import struct

32) from typing import TYPE_CHECKING, Any, TypedDict

33) 

34) from derivepassphrase import exporter
35) 

36) if TYPE_CHECKING:
37)     from collections.abc import Iterator
38) 

39)     from cryptography.hazmat.primitives import ciphers, hashes, hmac, padding
40)     from cryptography.hazmat.primitives.ciphers import algorithms, modes
41)     from cryptography.hazmat.primitives.kdf import pbkdf2
42) else:
43)     try:
44)         from cryptography.hazmat.primitives import (
45)             ciphers,
46)             hashes,
47)             hmac,
48)             padding,
49)         )
50)         from cryptography.hazmat.primitives.ciphers import algorithms, modes
51)         from cryptography.hazmat.primitives.kdf import pbkdf2
52)     except ModuleNotFoundError as exc:
53) 

54)         class _DummyModule:  # pragma: no cover

55)             def __init__(self, exc: type[Exception]) -> None:
56)                 self.exc = exc
57) 

58)             def __getattr__(self, name: str) -> Any:  # noqa: ANN401
59)                 def func(*args: Any, **kwargs: Any) -> Any:  # noqa: ANN401,ARG001

60)                     raise self.exc
61) 
62)                 return func
63) 

64)         ciphers = hashes = hmac = padding = _DummyModule(exc)
65)         algorithms = modes = pbkdf2 = _DummyModule(exc)

66)         STUBBED = True
67)     else:
68)         STUBBED = False
69) 

70) STOREROOM_MASTER_KEYS_UUID = b'35b7c7ed-f71e-4adf-9051-02fb0f1e0e17'
71) VAULT_CIPHER_UUID = b'73e69e8a-cb05-4b50-9f42-59d76a511299'
72) IV_SIZE = 16
73) KEY_SIZE = MAC_SIZE = 32
74) ENCRYPTED_KEYPAIR_SIZE = 128
75) VERSION_SIZE = 1
76) 

77) __all__ = ('export_storeroom_data',)
78) 

79) logger = logging.getLogger(__name__)

80) 
81) 
82) class KeyPair(TypedDict):

83)     """A pair of AES256 keys, one for encryption and one for signing.
84) 
85)     Attributes:
86)         encryption_key:
87)             AES256 key, used for encryption with AES256-CBC (with PKCS#7
88)             padding).
89)         signing_key:
90)             AES256 key, used for signing with HMAC-SHA256.
91) 
92)     """
93) 

94)     encryption_key: bytes

95)     """"""

96)     signing_key: bytes

97)     """"""

98) 
99) 
100) class MasterKeys(TypedDict):

101)     """A triple of AES256 keys, for encryption, signing and hashing.
102) 
103)     Attributes:
104)         hashing_key:
105)             AES256 key, used for hashing with HMAC-SHA256 to derive
106)             a hash table slot for an item.
107)         encryption_key:
108)             AES256 key, used for encryption with AES256-CBC (with PKCS#7
109)             padding).
110)         signing_key:
111)             AES256 key, used for signing with HMAC-SHA256.
112) 
113)     """
114) 

115)     hashing_key: bytes

116)     """"""

117)     encryption_key: bytes

118)     """"""

119)     signing_key: bytes

120)     """"""

121) 
122) 
123) def derive_master_keys_keys(password: str | bytes, iterations: int) -> KeyPair:

124)     """Derive encryption and signing keys for the master keys data.
125) 
126)     The master password is run through a key derivation function to
127)     obtain a 64-byte string, which is then split to yield two 32-byte
128)     keys.  The key derivation function is PBKDF2, using HMAC-SHA1 and
129)     salted with the storeroom master keys UUID.
130) 
131)     Args:
132)         password:
133)             A master password for the storeroom instance.  Usually read
134)             from the `VAULT_KEY` environment variable, otherwise
135)             defaults to the username.
136)         iterations:
137)             A count of rounds for the underlying key derivation
138)             function.  Usually stored as a setting next to the encrypted
139)             master keys data.
140) 
141)     Returns:
142)         A 2-tuple of keys, the encryption key and the signing key, to
143)         decrypt and verify the master keys data with.
144) 

145)     Warning:
146)         Non-public function, provided for didactical and educational
147)         purposes only.  Subject to change without notice, including
148)         removal.
149) 

150)     """

151)     if isinstance(password, str):
152)         password = password.encode('ASCII')
153)     master_keys_keys_blob = pbkdf2.PBKDF2HMAC(

154)         algorithm=hashes.SHA1(),

155)         length=2 * KEY_SIZE,

156)         salt=STOREROOM_MASTER_KEYS_UUID,
157)         iterations=iterations,
158)     ).derive(password)
159)     encryption_key, signing_key = struct.unpack(
160)         f'{KEY_SIZE}s {KEY_SIZE}s', master_keys_keys_blob
161)     )
162)     logger.debug(
163)         (
164)             'derived master_keys_keys bytes.fromhex(%s) (encryption) '
165)             'and bytes.fromhex(%s) (signing) '
166)             'from password bytes.fromhex(%s), '
167)             'using call '
168)             'pbkdf2(algorithm=%s, length=%d, salt=%s, iterations=%d)'
169)         ),
170)         repr(encryption_key.hex(' ')),
171)         repr(signing_key.hex(' ')),
172)         repr(password.hex(' ')),
173)         repr('SHA256'),
174)         64,
175)         repr(STOREROOM_MASTER_KEYS_UUID),
176)         iterations,
177)     )
178)     return {
179)         'encryption_key': encryption_key,
180)         'signing_key': signing_key,
181)     }
182) 
183) 
184) def decrypt_master_keys_data(data: bytes, keys: KeyPair) -> MasterKeys:

185)     r"""Decrypt the master keys data.

186) 
187)     The master keys data contains:
188) 
189)     - a 16-byte IV,

190)     - a 96-byte AES256-CBC-encrypted payload, plus 16 further bytes of
191)       PKCS7 padding, and
192)     - a 32-byte MAC of the preceding 128 bytes.

193) 
194)     The decrypted payload itself consists of three 32-byte keys: the
195)     hashing, encryption and signing keys, in that order.
196) 
197)     The encrypted payload is encrypted with the encryption key, and the
198)     MAC is created based on the signing key.  As per standard
199)     cryptographic procedure, the MAC can be verified before attempting
200)     to decrypt the payload.
201) 

202)     Because the payload size is both fixed and a multiple of the cipher
203)     blocksize, in this case, the PKCS7 padding always is `b'\x10' * 16`.

204) 
205)     Args:
206)         data:
207)             The encrypted master keys data.
208)         keys:
209)             The encryption and signing keys for the master keys data.
210)             These should have previously been derived via the

211)             [`derive_master_keys_keys`][] function.

212) 
213)     Returns:
214)         The master encryption, signing and hashing keys.
215) 

216)     Raises:
217)         cryptography.exceptions.InvalidSignature:
218)             The data does not contain a valid signature under the given
219)             key.
220)         ValueError:
221)             The format is invalid, in a non-cryptographic way.  (For
222)             example, it contains an unsupported version marker, or
223)             unexpected extra contents, or invalid padding.)
224) 

225)     Warning:
226)         Non-public function, provided for didactical and educational
227)         purposes only.  Subject to change without notice, including
228)         removal.
229) 

230)     """

231)     ciphertext, claimed_mac = struct.unpack(
232)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
233)     )
234)     actual_mac = hmac.HMAC(keys['signing_key'], hashes.SHA256())
235)     actual_mac.update(ciphertext)
236)     logger.debug(
237)         (
238)             'master_keys_data mac_key = bytes.fromhex(%s), '
239)             'hashed_content = bytes.fromhex(%s), '
240)             'claimed_mac = bytes.fromhex(%s), '
241)             'actual_mac = bytes.fromhex(%s)'
242)         ),
243)         repr(keys['signing_key'].hex(' ')),
244)         repr(ciphertext.hex(' ')),
245)         repr(claimed_mac.hex(' ')),
246)         repr(actual_mac.copy().finalize().hex(' ')),
247)     )
248)     actual_mac.verify(claimed_mac)
249) 

250)     try:
251)         iv, payload = struct.unpack(
252)             f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext

253)         )

254)         decryptor = ciphers.Cipher(
255)             algorithms.AES256(keys['encryption_key']), modes.CBC(iv)
256)         ).decryptor()
257)         padded_plaintext = bytearray()
258)         padded_plaintext.extend(decryptor.update(payload))
259)         padded_plaintext.extend(decryptor.finalize())
260)         unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
261)         plaintext = bytearray()
262)         plaintext.extend(unpadder.update(padded_plaintext))
263)         plaintext.extend(unpadder.finalize())
264)         hashing_key, encryption_key, signing_key = struct.unpack(
265)             f'{KEY_SIZE}s {KEY_SIZE}s {KEY_SIZE}s', plaintext
266)         )
267)     except (ValueError, struct.error) as exc:
268)         msg = 'Invalid encrypted master keys payload'
269)         raise ValueError(msg) from exc

270)     return {
271)         'hashing_key': hashing_key,
272)         'encryption_key': encryption_key,
273)         'signing_key': signing_key,
274)     }
275) 
276) 

277) def decrypt_session_keys(data: bytes, master_keys: MasterKeys) -> KeyPair:

278)     r"""Decrypt the bucket item's session keys.

279) 
280)     The bucket item's session keys are single-use keys for encrypting
281)     and signing a single item in the storage bucket.  The encrypted
282)     session key data consists of:
283) 
284)     - a 16-byte IV,

285)     - a 64-byte AES256-CBC-encrypted payload, plus 16 further bytes of
286)       PKCS7 padding, and
287)     - a 32-byte MAC of the preceding 96 bytes.

288) 
289)     The encrypted payload is encrypted with the master encryption key,
290)     and the MAC is created with the master signing key.  As per standard
291)     cryptographic procedure, the MAC can be verified before attempting
292)     to decrypt the payload.
293) 

294)     Because the payload size is both fixed and a multiple of the cipher
295)     blocksize, in this case, the PKCS7 padding always is `b'\x10' * 16`.

296) 
297)     Args:
298)         data:
299)             The encrypted bucket item session key data.
300)         master_keys:
301)             The master keys.  Presumably these have previously been

302)             obtained via the [`decrypt_master_keys_data`][] function.

303) 
304)     Returns:
305)         The bucket item's encryption and signing keys.
306) 

307)     Raises:
308)         cryptography.exceptions.InvalidSignature:
309)             The data does not contain a valid signature under the given
310)             key.
311)         ValueError:
312)             The format is invalid, in a non-cryptographic way.  (For
313)             example, it contains an unsupported version marker, or
314)             unexpected extra contents, or invalid padding.)

315) 

316)     Warning:
317)         Non-public function, provided for didactical and educational
318)         purposes only.  Subject to change without notice, including
319)         removal.
320) 

321)     """

322)     ciphertext, claimed_mac = struct.unpack(
323)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
324)     )

325)     actual_mac = hmac.HMAC(master_keys['signing_key'], hashes.SHA256())

326)     actual_mac.update(ciphertext)
327)     logger.debug(
328)         (

329)             'decrypt_bucket_item (session_keys): '

330)             'mac_key = bytes.fromhex(%s) (master), '
331)             'hashed_content = bytes.fromhex(%s), '
332)             'claimed_mac = bytes.fromhex(%s), '
333)             'actual_mac = bytes.fromhex(%s)'
334)         ),

335)         repr(master_keys['signing_key'].hex(' ')),

336)         repr(ciphertext.hex(' ')),
337)         repr(claimed_mac.hex(' ')),
338)         repr(actual_mac.copy().finalize().hex(' ')),
339)     )
340)     actual_mac.verify(claimed_mac)
341) 

342)     try:
343)         iv, payload = struct.unpack(
344)             f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext
345)         )
346)         decryptor = ciphers.Cipher(
347)             algorithms.AES256(master_keys['encryption_key']), modes.CBC(iv)
348)         ).decryptor()
349)         padded_plaintext = bytearray()
350)         padded_plaintext.extend(decryptor.update(payload))
351)         padded_plaintext.extend(decryptor.finalize())
352)         unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
353)         plaintext = bytearray()
354)         plaintext.extend(unpadder.update(padded_plaintext))
355)         plaintext.extend(unpadder.finalize())
356)         session_encryption_key, session_signing_key = struct.unpack(
357)             f'{KEY_SIZE}s {KEY_SIZE}s', plaintext
358)         )
359)     except (ValueError, struct.error) as exc:
360)         msg = 'Invalid encrypted session keys payload'
361)         raise ValueError(msg) from exc

362) 
363)     session_keys: KeyPair = {
364)         'encryption_key': session_encryption_key,
365)         'signing_key': session_signing_key,
366)     }
367) 
368)     logger.debug(
369)         (

370)             'decrypt_bucket_item (session_keys): '

371)             'decrypt_aes256_cbc_and_unpad(key=bytes.fromhex(%s), '
372)             'iv=bytes.fromhex(%s))(bytes.fromhex(%s)) '
373)             '= bytes.fromhex(%s) '
374)             '= {"encryption_key": bytes.fromhex(%s), '
375)             '"signing_key": bytes.fromhex(%s)}'
376)         ),

377)         repr(master_keys['encryption_key'].hex(' ')),

378)         repr(iv.hex(' ')),
379)         repr(payload.hex(' ')),
380)         repr(plaintext.hex(' ')),
381)         repr(session_keys['encryption_key'].hex(' ')),
382)         repr(session_keys['signing_key'].hex(' ')),
383)     )
384) 
385)     return session_keys
386) 
387) 

388) def decrypt_contents(data: bytes, session_keys: KeyPair) -> bytes:
389)     """Decrypt the bucket item's contents.
390) 
391)     The data consists of:
392) 
393)     - a 16-byte IV,
394)     - a variable-sized AES256-CBC-encrypted payload (using PKCS7 padding
395)       on the inside), and

396)     - a 32-byte MAC of the preceding bytes.

397) 
398)     The encrypted payload is encrypted with the bucket item's session
399)     encryption key, and the MAC is created with the bucket item's
400)     session signing key.  As per standard cryptographic procedure, the
401)     MAC can be verified before attempting to decrypt the payload.
402) 
403)     Args:
404)         data:
405)             The encrypted bucket item payload data.
406)         session_keys:
407)             The bucket item's session keys.  Presumably these have

408)             previously been obtained via the [`decrypt_session_keys`][]

409)             function.
410) 
411)     Returns:
412)         The bucket item's payload.
413) 

414)     Raises:
415)         cryptography.exceptions.InvalidSignature:
416)             The data does not contain a valid signature under the given
417)             key.
418)         ValueError:
419)             The format is invalid, in a non-cryptographic way.  (For
420)             example, it contains an unsupported version marker, or
421)             unexpected extra contents, or invalid padding.)

422) 

423)     Warning:
424)         Non-public function, provided for didactical and educational
425)         purposes only.  Subject to change without notice, including
426)         removal.
427) 

428)     """

429)     ciphertext, claimed_mac = struct.unpack(
430)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
431)     )

432)     actual_mac = hmac.HMAC(session_keys['signing_key'], hashes.SHA256())

433)     actual_mac.update(ciphertext)
434)     logger.debug(
435)         (

436)             'decrypt_bucket_item (contents): '

437)             'mac_key = bytes.fromhex(%s), '
438)             'hashed_content = bytes.fromhex(%s), '
439)             'claimed_mac = bytes.fromhex(%s), '
440)             'actual_mac = bytes.fromhex(%s)'
441)         ),

442)         repr(session_keys['signing_key'].hex(' ')),

443)         repr(ciphertext.hex(' ')),
444)         repr(claimed_mac.hex(' ')),
445)         repr(actual_mac.copy().finalize().hex(' ')),
446)     )
447)     actual_mac.verify(claimed_mac)
448) 
449)     iv, payload = struct.unpack(
450)         f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext
451)     )
452)     decryptor = ciphers.Cipher(

453)         algorithms.AES256(session_keys['encryption_key']), modes.CBC(iv)

454)     ).decryptor()
455)     padded_plaintext = bytearray()
456)     padded_plaintext.extend(decryptor.update(payload))
457)     padded_plaintext.extend(decryptor.finalize())
458)     unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
459)     plaintext = bytearray()
460)     plaintext.extend(unpadder.update(padded_plaintext))
461)     plaintext.extend(unpadder.finalize())
462) 
463)     logger.debug(
464)         (

465)             'decrypt_bucket_item (contents): '

466)             'decrypt_aes256_cbc_and_unpad(key=bytes.fromhex(%s), '
467)             'iv=bytes.fromhex(%s))(bytes.fromhex(%s)) '
468)             '= bytes.fromhex(%s)'
469)         ),

470)         repr(session_keys['encryption_key'].hex(' ')),

471)         repr(iv.hex(' ')),
472)         repr(payload.hex(' ')),
473)         repr(plaintext.hex(' ')),
474)     )
475) 
476)     return plaintext
477) 
478) 

479) def decrypt_bucket_item(bucket_item: bytes, master_keys: MasterKeys) -> bytes:

480)     """Decrypt a bucket item.
481) 
482)     Args:
483)         bucket_item:
484)             The encrypted bucket item.
485)         master_keys:
486)             The master keys.  Presumably these have previously been

487)             obtained via the [`decrypt_master_keys_data`][] function.

488) 
489)     Returns:
490)         The decrypted bucket item.
491) 
492)     Raises:
493)         cryptography.exceptions.InvalidSignature:
494)             The data does not contain a valid signature under the given
495)             key.
496)         ValueError:
497)             The format is invalid, in a non-cryptographic way.  (For
498)             example, it contains an unsupported version marker, or
499)             unexpected extra contents, or invalid padding.)
500) 

501)     Warning:
502)         Non-public function, provided for didactical and educational
503)         purposes only.  Subject to change without notice, including
504)         removal.
505) 

506)     """

507)     logger.debug(
508)         (

509)             'decrypt_bucket_item: data = bytes.fromhex(%s), '

510)             'encryption_key = bytes.fromhex(%s), '
511)             'signing_key = bytes.fromhex(%s)'
512)         ),

513)         repr(bucket_item.hex(' ')),

514)         repr(master_keys['encryption_key'].hex(' ')),
515)         repr(master_keys['signing_key'].hex(' ')),
516)     )
517)     data_version, encrypted_session_keys, data_contents = struct.unpack(
518)         (
519)             f'B {ENCRYPTED_KEYPAIR_SIZE}s '

520)             f'{len(bucket_item) - 1 - ENCRYPTED_KEYPAIR_SIZE}s'

521)         ),

522)         bucket_item,

523)     )
524)     if data_version != 1:
525)         msg = f'Cannot handle version {data_version} encrypted data'

526)         raise ValueError(msg)

527)     session_keys = decrypt_session_keys(encrypted_session_keys, master_keys)
528)     return decrypt_contents(data_contents, session_keys)
529) 
530) 

531) def decrypt_bucket_file(

532)     filename: str,
533)     master_keys: MasterKeys,
534)     *,
535)     root_dir: str | bytes | os.PathLike = '.',

536) ) -> Iterator[bytes]:

537)     """Decrypt a complete bucket.

538) 
539)     Args:
540)         filename:
541)             The bucket file's filename.
542)         master_keys:
543)             The master keys.  Presumably these have previously been

544)             obtained via the [`decrypt_master_keys_data`][] function.

545)         root_dir:
546)             The root directory of the data store.  The filename is
547)             interpreted relatively to this directory.
548) 
549)     Yields:

550)         A decrypted bucket item.

551) 
552)     Raises:
553)         cryptography.exceptions.InvalidSignature:
554)             The data does not contain a valid signature under the given
555)             key.
556)         ValueError:
557)             The format is invalid, in a non-cryptographic way.  (For
558)             example, it contains an unsupported version marker, or
559)             unexpected extra contents, or invalid padding.)
560) 

561)     Warning:
562)         Non-public function, provided for didactical and educational
563)         purposes only.  Subject to change without notice, including
564)         removal.
565) 

566)     """

567)     with open(
568)         os.path.join(os.fsdecode(root_dir), filename), 'rb'
569)     ) as bucket_file:

570)         header_line = bucket_file.readline()
571)         try:
572)             header = json.loads(header_line)
573)         except ValueError as exc:
574)             msg = f'Invalid bucket file: {filename}'

575)             raise ValueError(msg) from exc

576)         if header != {'version': 1}:
577)             msg = f'Invalid bucket file: {filename}'

578)             raise ValueError(msg) from None

579)         for line in bucket_file:

580)             yield decrypt_bucket_item(
581)                 base64.standard_b64decode(line), master_keys

582)             )
583) 
584) 

585) def _store(config: dict[str, Any], path: str, json_contents: bytes) -> None:

586)     """Store the JSON contents at path in the config structure.
587) 
588)     Traverse the config structure according to path, and set the value
589)     of the leaf to the decoded JSON contents.
590) 
591)     A path `/foo/bar/xyz` translates to the JSON structure
592)     `{"foo": {"bar": {"xyz": ...}}}`.
593) 
594)     Args:
595)         config:
596)             The (top-level) configuration structure to update.
597)         path:
598)             The path within the configuration structure to traverse.
599)         json_contents:
600)             The contents to set the item to, after JSON-decoding.
601) 
602)     Raises:
603)         json.JSONDecodeError:
604)             There was an error parsing the JSON contents.
605) 
606)     """
607)     contents = json.loads(json_contents)
608)     path_parts = [part for part in path.split('/') if part]
609)     for part in path_parts[:-1]:
610)         config = config.setdefault(part, {})
611)     if path_parts:
612)         config[path_parts[-1]] = contents
613) 
614) 

615) def export_storeroom_data(  # noqa: C901,PLR0912,PLR0914,PLR0915

616)     storeroom_path: str | bytes | os.PathLike | None = None,
617)     master_keys_key: str | bytes | None = None,

618) ) -> dict[str, Any]:
619)     """Export the full configuration stored in the storeroom.
620) 
621)     Args:
622)         storeroom_path:

623)             Path to the storeroom; usually `~/.vault`.  If not given,

624)             then query [`exporter.get_vault_path`][] for the value.

625)         master_keys_key:

626)             Encryption key/password for the master keys, usually the
627)             username, or passed via the `VAULT_KEY` environment
628)             variable.  If not given, then query

629)             [`exporter.get_vault_key`][] for the value.

630) 
631)     Returns:
632)         The full configuration, as stored in the storeroom.
633) 
634)         This may or may not be a valid configuration according to vault
635)         or derivepassphrase.
636) 
637)     Raises:
638)         RuntimeError:
639)             Something went wrong during data collection, e.g. we
640)             encountered unsupported or corrupted data in the storeroom.
641)         json.JSONDecodeError:
642)             An internal JSON data structure failed to parse from disk.
643)             The storeroom is probably corrupted.
644) 
645)     """

646)     if storeroom_path is None:
647)         storeroom_path = exporter.get_vault_path()

648)     if master_keys_key is None:

649)         master_keys_key = exporter.get_vault_key()

650)     with open(
651)         os.path.join(os.fsdecode(storeroom_path), '.keys'), encoding='utf-8'
652)     ) as master_keys_file:

653)         header = json.loads(master_keys_file.readline())
654)         if header != {'version': 1}:
655)             msg = 'bad or unsupported keys version header'
656)             raise RuntimeError(msg)
657)         raw_keys_data = base64.standard_b64decode(master_keys_file.readline())
658)         encrypted_keys_params, encrypted_keys = struct.unpack(
659)             f'B {len(raw_keys_data) - 1}s', raw_keys_data
660)         )
661)         if master_keys_file.read():
662)             msg = 'trailing data; cannot make sense of .keys file'
663)             raise RuntimeError(msg)
664)     encrypted_keys_version = encrypted_keys_params >> 4
665)     if encrypted_keys_version != 1:
666)         msg = f'cannot handle version {encrypted_keys_version} encrypted keys'
667)         raise RuntimeError(msg)
668)     encrypted_keys_iterations = 2 ** (10 + (encrypted_keys_params & 0x0F))
669)     master_keys_keys = derive_master_keys_keys(

670)         master_keys_key, encrypted_keys_iterations

671)     )
672)     master_keys = decrypt_master_keys_data(encrypted_keys, master_keys_keys)
673) 

674)     config_structure: dict[str, Any] = {}
675)     json_contents: dict[str, bytes] = {}

676)     # Use glob.glob(..., root_dir=...) here once Python 3.9 becomes
677)     # unsupported.
678)     storeroom_path_str = os.fsdecode(storeroom_path)
679)     valid_hashdirs = [
680)         hashdir_name
681)         for hashdir_name in os.listdir(storeroom_path_str)
682)         if fnmatch.fnmatch(hashdir_name, '[01][0-9a-f]')
683)     ]
684)     for file in valid_hashdirs:

685)         bucket_contents = list(
686)             decrypt_bucket_file(file, master_keys, root_dir=storeroom_path)
687)         )

688)         bucket_index = json.loads(bucket_contents.pop(0))
689)         for pos, item in enumerate(bucket_index):
690)             json_contents[item] = bucket_contents[pos]
691)             logger.debug(
692)                 'Found bucket item: %s -> %s', item, bucket_contents[pos]
693)             )
694)     dirs_to_check: dict[str, list[str]] = {}
695)     json_payload: Any
696)     for path, json_content in sorted(json_contents.items()):
697)         if path.endswith('/'):
698)             logger.debug(
699)                 'Postponing dir check: %s -> %s',
700)                 path,
701)                 json_content.decode('utf-8'),
702)             )
703)             json_payload = json.loads(json_content)
704)             if not isinstance(json_payload, list) or any(
705)                 not isinstance(x, str) for x in json_payload
706)             ):
707)                 msg = (
708)                     f'Directory index is not actually an index: '
709)                     f'{json_content!r}'
710)                 )
711)                 raise RuntimeError(msg)
712)             dirs_to_check[path] = json_payload
713)             logger.debug(
714)                 'Setting contents (empty directory): %s -> %s', path, '{}'
715)             )

716)             _store(config_structure, path, b'{}')

717)         else:
718)             logger.debug(
719)                 'Setting contents: %s -> %s',
720)                 path,
721)                 json_content.decode('utf-8'),
722)             )

723)             _store(config_structure, path, json_content)

724)     # Sorted order is important; see `mabye_obj` below.
725)     for _dir, namelist in sorted(dirs_to_check.items()):

726)         namelist = [x.rstrip('/') for x in namelist]  # noqa: PLW2901

727)         obj: dict[Any, Any] = config_structure
728)         for part in _dir.split('/'):
729)             if part:
730)                 # Because we iterate paths in sorted order, parent
731)                 # directories are encountered before child directories.
732)                 # So parent directories always exist (lest we would have
733)                 # aborted earlier).
734)                 #
735)                 # Of course, the type checker doesn't necessarily know
736)                 # this, so we need to use assertions anyway.
737)                 maybe_obj = obj.get(part)
738)                 assert isinstance(
739)                     maybe_obj, dict
740)                 ), f'Cannot traverse storage path {_dir!r}'
741)                 obj = maybe_obj

742)         if set(obj.keys()) != set(namelist):
743)             msg = f'Object key mismatch for path {_dir!r}'
744)             raise RuntimeError(msg)
745)     return config_structure

746) 
747) 
748) if __name__ == '__main__':