3fe3634dc06a5339cfd939ba01062d4778f4f064
Marco Ricci Change the author e-mail ad...

Marco Ricci authored 3 months ago

1) # SPDX-FileCopyrightText: 2024 Marco Ricci <software@the13thletter.info>
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

2) #
3) # SPDX-License-Identifier: MIT
4) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

5) """Exporter for the vault "storeroom" configuration format.
6) 
7) The "storeroom" format is the experimental format used in alpha and beta
8) versions of vault beyond v0.3.0.  The configuration is stored as
9) a separate directory, which acts like a hash table (i.e. has named
10) slots) and provides an impure quasi-filesystem interface.  Each hash
11) table entry is separately encrypted and authenticated.  James Coglan
12) designed this format to avoid concurrent write issues when updating or
13) synchronizing the vault configuration with e.g. a cloud service.
14) 
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

15) The public interface is the [`export_storeroom_data`][] function.
16) Multiple *non-public* functions are additionally documented here for
17) didactical and educational reasons, but they are not part of the module
18) API, are subject to change without notice (including removal), and
19) should *not* be used or relied on.
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

20) 
21) """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

22) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

23) from __future__ import annotations
24) 
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

25) import base64
26) import glob
27) import json
28) import logging
29) import os
30) import os.path
31) import struct
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

32) from typing import TYPE_CHECKING, Any, TypedDict
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

33) 
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

34) from derivepassphrase import exporter
35) 
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

36) if TYPE_CHECKING:
37)     from collections.abc import Iterator
38) 
Marco Ricci Add preliminary tests for t...

Marco Ricci authored 3 months ago

39)     from cryptography.hazmat.primitives import ciphers, hashes, hmac, padding
40)     from cryptography.hazmat.primitives.ciphers import algorithms, modes
41)     from cryptography.hazmat.primitives.kdf import pbkdf2
42) else:
43)     try:
44)         from cryptography.hazmat.primitives import (
45)             ciphers,
46)             hashes,
47)             hmac,
48)             padding,
49)         )
50)         from cryptography.hazmat.primitives.ciphers import algorithms, modes
51)         from cryptography.hazmat.primitives.kdf import pbkdf2
52)     except ModuleNotFoundError as exc:
53) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

54)         class _DummyModule:  # pragma: no cover
Marco Ricci Add preliminary tests for t...

Marco Ricci authored 3 months ago

55)             def __init__(self, exc: type[Exception]) -> None:
56)                 self.exc = exc
57) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

58)             def __getattr__(self, name: str) -> Any:  # noqa: ANN401
59)                 def func(*args: Any, **kwargs: Any) -> Any:  # noqa: ANN401,ARG001
Marco Ricci Add preliminary tests for t...

Marco Ricci authored 3 months ago

60)                     raise self.exc
61) 
62)                 return func
63) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

64)         ciphers = hashes = hmac = padding = _DummyModule(exc)
65)         algorithms = modes = pbkdf2 = _DummyModule(exc)
Marco Ricci Add preliminary tests for t...

Marco Ricci authored 3 months ago

66)         STUBBED = True
67)     else:
68)         STUBBED = False
69) 
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

70) STOREROOM_MASTER_KEYS_UUID = b'35b7c7ed-f71e-4adf-9051-02fb0f1e0e17'
71) VAULT_CIPHER_UUID = b'73e69e8a-cb05-4b50-9f42-59d76a511299'
72) IV_SIZE = 16
73) KEY_SIZE = MAC_SIZE = 32
74) ENCRYPTED_KEYPAIR_SIZE = 128
75) VERSION_SIZE = 1
76) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

77) __all__ = ('export_storeroom_data',)
78) 
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

79) logger = logging.getLogger(__name__)
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

80) 
81) 
82) class KeyPair(TypedDict):
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

83)     """A pair of AES256 keys, one for encryption and one for signing.
84) 
85)     Attributes:
86)         encryption_key:
87)             AES256 key, used for encryption with AES256-CBC (with PKCS#7
88)             padding).
89)         signing_key:
90)             AES256 key, used for signing with HMAC-SHA256.
91) 
92)     """
93) 
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

94)     encryption_key: bytes
Marco Ricci Enable cross-references on...

Marco Ricci authored 2 months ago

95)     """"""
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

96)     signing_key: bytes
Marco Ricci Enable cross-references on...

Marco Ricci authored 2 months ago

97)     """"""
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

98) 
99) 
100) class MasterKeys(TypedDict):
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

101)     """A triple of AES256 keys, for encryption, signing and hashing.
102) 
103)     Attributes:
104)         hashing_key:
105)             AES256 key, used for hashing with HMAC-SHA256 to derive
106)             a hash table slot for an item.
107)         encryption_key:
108)             AES256 key, used for encryption with AES256-CBC (with PKCS#7
109)             padding).
110)         signing_key:
111)             AES256 key, used for signing with HMAC-SHA256.
112) 
113)     """
114) 
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

115)     hashing_key: bytes
Marco Ricci Enable cross-references on...

Marco Ricci authored 2 months ago

116)     """"""
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

117)     encryption_key: bytes
Marco Ricci Enable cross-references on...

Marco Ricci authored 2 months ago

118)     """"""
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

119)     signing_key: bytes
Marco Ricci Enable cross-references on...

Marco Ricci authored 2 months ago

120)     """"""
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

121) 
122) 
123) def derive_master_keys_keys(password: str | bytes, iterations: int) -> KeyPair:
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

124)     """Derive encryption and signing keys for the master keys data.
125) 
126)     The master password is run through a key derivation function to
127)     obtain a 64-byte string, which is then split to yield two 32-byte
128)     keys.  The key derivation function is PBKDF2, using HMAC-SHA1 and
129)     salted with the storeroom master keys UUID.
130) 
131)     Args:
132)         password:
133)             A master password for the storeroom instance.  Usually read
134)             from the `VAULT_KEY` environment variable, otherwise
135)             defaults to the username.
136)         iterations:
137)             A count of rounds for the underlying key derivation
138)             function.  Usually stored as a setting next to the encrypted
139)             master keys data.
140) 
141)     Returns:
142)         A 2-tuple of keys, the encryption key and the signing key, to
143)         decrypt and verify the master keys data with.
144) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

145)     Warning:
146)         Non-public function, provided for didactical and educational
147)         purposes only.  Subject to change without notice, including
148)         removal.
149) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

150)     """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

151)     if isinstance(password, str):
152)         password = password.encode('ASCII')
153)     master_keys_keys_blob = pbkdf2.PBKDF2HMAC(
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

154)         algorithm=hashes.SHA1(),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

155)         length=2 * KEY_SIZE,
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

156)         salt=STOREROOM_MASTER_KEYS_UUID,
157)         iterations=iterations,
158)     ).derive(password)
159)     encryption_key, signing_key = struct.unpack(
160)         f'{KEY_SIZE}s {KEY_SIZE}s', master_keys_keys_blob
161)     )
162)     logger.debug(
163)         (
164)             'derived master_keys_keys bytes.fromhex(%s) (encryption) '
165)             'and bytes.fromhex(%s) (signing) '
166)             'from password bytes.fromhex(%s), '
167)             'using call '
168)             'pbkdf2(algorithm=%s, length=%d, salt=%s, iterations=%d)'
169)         ),
170)         repr(encryption_key.hex(' ')),
171)         repr(signing_key.hex(' ')),
172)         repr(password.hex(' ')),
173)         repr('SHA256'),
174)         64,
175)         repr(STOREROOM_MASTER_KEYS_UUID),
176)         iterations,
177)     )
178)     return {
179)         'encryption_key': encryption_key,
180)         'signing_key': signing_key,
181)     }
182) 
183) 
184) def decrypt_master_keys_data(data: bytes, keys: KeyPair) -> MasterKeys:
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

185)     """Decrypt the master keys data.
186) 
187)     The master keys data contains:
188) 
189)     - a 16-byte IV,
190)     - a 96-byte AES256-CBC-encrypted payload (using PKCS7 padding on the
191)       inside), and
192)     - a 32-byte MAC of the preceding 112 bytes.
193) 
194)     The decrypted payload itself consists of three 32-byte keys: the
195)     hashing, encryption and signing keys, in that order.
196) 
197)     The encrypted payload is encrypted with the encryption key, and the
198)     MAC is created based on the signing key.  As per standard
199)     cryptographic procedure, the MAC can be verified before attempting
200)     to decrypt the payload.
201) 
202)     Because the payload size is both fixed and a multiple of the
203)     cipher blocksize, in this case, the PKCS7 padding is a no-op.
204) 
205)     Args:
206)         data:
207)             The encrypted master keys data.
208)         keys:
209)             The encryption and signing keys for the master keys data.
210)             These should have previously been derived via the
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

211)             [`derive_master_keys_keys`][] function.
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

212) 
213)     Returns:
214)         The master encryption, signing and hashing keys.
215) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

216)     Raises:
217)         cryptography.exceptions.InvalidSignature:
218)             The data does not contain a valid signature under the given
219)             key.
220)         ValueError:
221)             The format is invalid, in a non-cryptographic way.  (For
222)             example, it contains an unsupported version marker, or
223)             unexpected extra contents, or invalid padding.)
224) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

225)     Warning:
226)         Non-public function, provided for didactical and educational
227)         purposes only.  Subject to change without notice, including
228)         removal.
229) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

230)     """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

231)     ciphertext, claimed_mac = struct.unpack(
232)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
233)     )
234)     actual_mac = hmac.HMAC(keys['signing_key'], hashes.SHA256())
235)     actual_mac.update(ciphertext)
236)     logger.debug(
237)         (
238)             'master_keys_data mac_key = bytes.fromhex(%s), '
239)             'hashed_content = bytes.fromhex(%s), '
240)             'claimed_mac = bytes.fromhex(%s), '
241)             'actual_mac = bytes.fromhex(%s)'
242)         ),
243)         repr(keys['signing_key'].hex(' ')),
244)         repr(ciphertext.hex(' ')),
245)         repr(claimed_mac.hex(' ')),
246)         repr(actual_mac.copy().finalize().hex(' ')),
247)     )
248)     actual_mac.verify(claimed_mac)
249) 
250)     iv, payload = struct.unpack(
251)         f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext
252)     )
253)     decryptor = ciphers.Cipher(
254)         algorithms.AES256(keys['encryption_key']), modes.CBC(iv)
255)     ).decryptor()
256)     padded_plaintext = bytearray()
257)     padded_plaintext.extend(decryptor.update(payload))
258)     padded_plaintext.extend(decryptor.finalize())
259)     unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
260)     plaintext = bytearray()
261)     plaintext.extend(unpadder.update(padded_plaintext))
262)     plaintext.extend(unpadder.finalize())
263)     if len(plaintext) != 3 * KEY_SIZE:
264)         msg = (
265)             f'Expecting 3 encrypted keys at {3 * KEY_SIZE} bytes total, '
266)             f'but found {len(plaintext)} instead'
267)         )
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

268)         raise ValueError(msg)
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

269)     hashing_key, encryption_key, signing_key = struct.unpack(
270)         f'{KEY_SIZE}s {KEY_SIZE}s {KEY_SIZE}s', plaintext
271)     )
272)     return {
273)         'hashing_key': hashing_key,
274)         'encryption_key': encryption_key,
275)         'signing_key': signing_key,
276)     }
277) 
278) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

279) def decrypt_session_keys(data: bytes, master_keys: MasterKeys) -> KeyPair:
280)     """Decrypt the bucket item's session keys.
281) 
282)     The bucket item's session keys are single-use keys for encrypting
283)     and signing a single item in the storage bucket.  The encrypted
284)     session key data consists of:
285) 
286)     - a 16-byte IV,
287)     - a 64-byte AES256-CBC-encrypted payload (using PKCS7 padding on the
288)       inside), and
289)     - a 32-byte MAC of the preceding 80 bytes.
290) 
291)     The encrypted payload is encrypted with the master encryption key,
292)     and the MAC is created with the master signing key.  As per standard
293)     cryptographic procedure, the MAC can be verified before attempting
294)     to decrypt the payload.
295) 
296)     Because the payload size is both fixed and a multiple of the
297)     cipher blocksize, in this case, the PKCS7 padding is a no-op.
298) 
299)     Args:
300)         data:
301)             The encrypted bucket item session key data.
302)         master_keys:
303)             The master keys.  Presumably these have previously been
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

304)             obtained via the [`decrypt_master_keys_data`][] function.
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

305) 
306)     Returns:
307)         The bucket item's encryption and signing keys.
308) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

309)     Raises:
310)         cryptography.exceptions.InvalidSignature:
311)             The data does not contain a valid signature under the given
312)             key.
313)         ValueError:
314)             The format is invalid, in a non-cryptographic way.  (For
315)             example, it contains an unsupported version marker, or
316)             unexpected extra contents, or invalid padding.)
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

317) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

318)     Warning:
319)         Non-public function, provided for didactical and educational
320)         purposes only.  Subject to change without notice, including
321)         removal.
322) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

323)     """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

324)     ciphertext, claimed_mac = struct.unpack(
325)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
326)     )
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

327)     actual_mac = hmac.HMAC(master_keys['signing_key'], hashes.SHA256())
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

328)     actual_mac.update(ciphertext)
329)     logger.debug(
330)         (
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

331)             'decrypt_bucket_item (session_keys): '
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

332)             'mac_key = bytes.fromhex(%s) (master), '
333)             'hashed_content = bytes.fromhex(%s), '
334)             'claimed_mac = bytes.fromhex(%s), '
335)             'actual_mac = bytes.fromhex(%s)'
336)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

337)         repr(master_keys['signing_key'].hex(' ')),
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

338)         repr(ciphertext.hex(' ')),
339)         repr(claimed_mac.hex(' ')),
340)         repr(actual_mac.copy().finalize().hex(' ')),
341)     )
342)     actual_mac.verify(claimed_mac)
343) 
344)     iv, payload = struct.unpack(
345)         f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext
346)     )
347)     decryptor = ciphers.Cipher(
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

348)         algorithms.AES256(master_keys['encryption_key']), modes.CBC(iv)
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

349)     ).decryptor()
350)     padded_plaintext = bytearray()
351)     padded_plaintext.extend(decryptor.update(payload))
352)     padded_plaintext.extend(decryptor.finalize())
353)     unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
354)     plaintext = bytearray()
355)     plaintext.extend(unpadder.update(padded_plaintext))
356)     plaintext.extend(unpadder.finalize())
357) 
358)     session_encryption_key, session_signing_key, inner_payload = struct.unpack(
359)         f'{KEY_SIZE}s {KEY_SIZE}s {len(plaintext) - 2 * KEY_SIZE}s',
360)         plaintext,
361)     )
362)     session_keys: KeyPair = {
363)         'encryption_key': session_encryption_key,
364)         'signing_key': session_signing_key,
365)     }
366) 
367)     logger.debug(
368)         (
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

369)             'decrypt_bucket_item (session_keys): '
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

370)             'decrypt_aes256_cbc_and_unpad(key=bytes.fromhex(%s), '
371)             'iv=bytes.fromhex(%s))(bytes.fromhex(%s)) '
372)             '= bytes.fromhex(%s) '
373)             '= {"encryption_key": bytes.fromhex(%s), '
374)             '"signing_key": bytes.fromhex(%s)}'
375)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

376)         repr(master_keys['encryption_key'].hex(' ')),
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

377)         repr(iv.hex(' ')),
378)         repr(payload.hex(' ')),
379)         repr(plaintext.hex(' ')),
380)         repr(session_keys['encryption_key'].hex(' ')),
381)         repr(session_keys['signing_key'].hex(' ')),
382)     )
383) 
384)     if inner_payload:
385)         logger.debug(
386)             'ignoring misplaced inner payload bytes.fromhex(%s)',
387)             repr(inner_payload.hex(' ')),
388)         )
389) 
390)     return session_keys
391) 
392) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

393) def decrypt_contents(data: bytes, session_keys: KeyPair) -> bytes:
394)     """Decrypt the bucket item's contents.
395) 
396)     The data consists of:
397) 
398)     - a 16-byte IV,
399)     - a variable-sized AES256-CBC-encrypted payload (using PKCS7 padding
400)       on the inside), and
401)     - a 32-byte MAC of the preceding 80 bytes.
402) 
403)     The encrypted payload is encrypted with the bucket item's session
404)     encryption key, and the MAC is created with the bucket item's
405)     session signing key.  As per standard cryptographic procedure, the
406)     MAC can be verified before attempting to decrypt the payload.
407) 
408)     Args:
409)         data:
410)             The encrypted bucket item payload data.
411)         session_keys:
412)             The bucket item's session keys.  Presumably these have
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

413)             previously been obtained via the [`decrypt_session_keys`][]
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

414)             function.
415) 
416)     Returns:
417)         The bucket item's payload.
418) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

419)     Raises:
420)         cryptography.exceptions.InvalidSignature:
421)             The data does not contain a valid signature under the given
422)             key.
423)         ValueError:
424)             The format is invalid, in a non-cryptographic way.  (For
425)             example, it contains an unsupported version marker, or
426)             unexpected extra contents, or invalid padding.)
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

427) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

428)     Warning:
429)         Non-public function, provided for didactical and educational
430)         purposes only.  Subject to change without notice, including
431)         removal.
432) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

433)     """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

434)     ciphertext, claimed_mac = struct.unpack(
435)         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data
436)     )
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

437)     actual_mac = hmac.HMAC(session_keys['signing_key'], hashes.SHA256())
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

438)     actual_mac.update(ciphertext)
439)     logger.debug(
440)         (
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

441)             'decrypt_bucket_item (contents): '
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

442)             'mac_key = bytes.fromhex(%s), '
443)             'hashed_content = bytes.fromhex(%s), '
444)             'claimed_mac = bytes.fromhex(%s), '
445)             'actual_mac = bytes.fromhex(%s)'
446)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

447)         repr(session_keys['signing_key'].hex(' ')),
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

448)         repr(ciphertext.hex(' ')),
449)         repr(claimed_mac.hex(' ')),
450)         repr(actual_mac.copy().finalize().hex(' ')),
451)     )
452)     actual_mac.verify(claimed_mac)
453) 
454)     iv, payload = struct.unpack(
455)         f'{IV_SIZE}s {len(ciphertext) - IV_SIZE}s', ciphertext
456)     )
457)     decryptor = ciphers.Cipher(
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

458)         algorithms.AES256(session_keys['encryption_key']), modes.CBC(iv)
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

459)     ).decryptor()
460)     padded_plaintext = bytearray()
461)     padded_plaintext.extend(decryptor.update(payload))
462)     padded_plaintext.extend(decryptor.finalize())
463)     unpadder = padding.PKCS7(IV_SIZE * 8).unpadder()
464)     plaintext = bytearray()
465)     plaintext.extend(unpadder.update(padded_plaintext))
466)     plaintext.extend(unpadder.finalize())
467) 
468)     logger.debug(
469)         (
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

470)             'decrypt_bucket_item (contents): '
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

471)             'decrypt_aes256_cbc_and_unpad(key=bytes.fromhex(%s), '
472)             'iv=bytes.fromhex(%s))(bytes.fromhex(%s)) '
473)             '= bytes.fromhex(%s)'
474)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

475)         repr(session_keys['encryption_key'].hex(' ')),
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

476)         repr(iv.hex(' ')),
477)         repr(payload.hex(' ')),
478)         repr(plaintext.hex(' ')),
479)     )
480) 
481)     return plaintext
482) 
483) 
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

484) def decrypt_bucket_item(bucket_item: bytes, master_keys: MasterKeys) -> bytes:
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

485)     """Decrypt a bucket item.
486) 
487)     Args:
488)         bucket_item:
489)             The encrypted bucket item.
490)         master_keys:
491)             The master keys.  Presumably these have previously been
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

492)             obtained via the [`decrypt_master_keys_data`][] function.
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

493) 
494)     Returns:
495)         The decrypted bucket item.
496) 
497)     Raises:
498)         cryptography.exceptions.InvalidSignature:
499)             The data does not contain a valid signature under the given
500)             key.
501)         ValueError:
502)             The format is invalid, in a non-cryptographic way.  (For
503)             example, it contains an unsupported version marker, or
504)             unexpected extra contents, or invalid padding.)
505) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

506)     Warning:
507)         Non-public function, provided for didactical and educational
508)         purposes only.  Subject to change without notice, including
509)         removal.
510) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

511)     """
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

512)     logger.debug(
513)         (
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

514)             'decrypt_bucket_item: data = bytes.fromhex(%s), '
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

515)             'encryption_key = bytes.fromhex(%s), '
516)             'signing_key = bytes.fromhex(%s)'
517)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

518)         repr(bucket_item.hex(' ')),
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

519)         repr(master_keys['encryption_key'].hex(' ')),
520)         repr(master_keys['signing_key'].hex(' ')),
521)     )
522)     data_version, encrypted_session_keys, data_contents = struct.unpack(
523)         (
524)             f'B {ENCRYPTED_KEYPAIR_SIZE}s '
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

525)             f'{len(bucket_item) - 1 - ENCRYPTED_KEYPAIR_SIZE}s'
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

526)         ),
Marco Ricci Add docstrings and better v...

Marco Ricci authored 4 months ago

527)         bucket_item,
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

528)     )
529)     if data_version != 1:
530)         msg = f'Cannot handle version {data_version} encrypted data'
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

531)         raise ValueError(msg)
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

532)     session_keys = decrypt_session_keys(encrypted_session_keys, master_keys)
533)     return decrypt_contents(data_contents, session_keys)
534) 
535) 
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

536) def decrypt_bucket_file(
Marco Ricci Support exports from outsid...

Marco Ricci authored 4 months ago

537)     filename: str,
538)     master_keys: MasterKeys,
539)     *,
540)     root_dir: str | bytes | os.PathLike = '.',
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

541) ) -> Iterator[bytes]:
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

542)     """Decrypt a complete bucket.
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

543) 
544)     Args:
545)         filename:
546)             The bucket file's filename.
547)         master_keys:
548)             The master keys.  Presumably these have previously been
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

549)             obtained via the [`decrypt_master_keys_data`][] function.
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

550)         root_dir:
551)             The root directory of the data store.  The filename is
552)             interpreted relatively to this directory.
553) 
554)     Yields:
Marco Ricci Convert old syntax for Yiel...

Marco Ricci authored 2 months ago

555)         A decrypted bucket item.
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

556) 
557)     Raises:
558)         cryptography.exceptions.InvalidSignature:
559)             The data does not contain a valid signature under the given
560)             key.
561)         ValueError:
562)             The format is invalid, in a non-cryptographic way.  (For
563)             example, it contains an unsupported version marker, or
564)             unexpected extra contents, or invalid padding.)
565) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

566)     Warning:
567)         Non-public function, provided for didactical and educational
568)         purposes only.  Subject to change without notice, including
569)         removal.
570) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

571)     """
Marco Ricci Support exports from outsid...

Marco Ricci authored 4 months ago

572)     with open(
573)         os.path.join(os.fsdecode(root_dir), filename), 'rb'
574)     ) as bucket_file:
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

575)         header_line = bucket_file.readline()
576)         try:
577)             header = json.loads(header_line)
578)         except ValueError as exc:
579)             msg = f'Invalid bucket file: {filename}'
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

580)             raise ValueError(msg) from exc
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

581)         if header != {'version': 1}:
582)             msg = f'Invalid bucket file: {filename}'
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

583)             raise ValueError(msg) from None
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

584)         for line in bucket_file:
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

585)             yield decrypt_bucket_item(
586)                 base64.standard_b64decode(line), master_keys
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

587)             )
588) 
589) 
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

590) def _store(config: dict[str, Any], path: str, json_contents: bytes) -> None:
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

591)     """Store the JSON contents at path in the config structure.
592) 
593)     Traverse the config structure according to path, and set the value
594)     of the leaf to the decoded JSON contents.
595) 
596)     A path `/foo/bar/xyz` translates to the JSON structure
597)     `{"foo": {"bar": {"xyz": ...}}}`.
598) 
599)     Args:
600)         config:
601)             The (top-level) configuration structure to update.
602)         path:
603)             The path within the configuration structure to traverse.
604)         json_contents:
605)             The contents to set the item to, after JSON-decoding.
606) 
607)     Raises:
608)         json.JSONDecodeError:
609)             There was an error parsing the JSON contents.
610) 
611)     """
612)     contents = json.loads(json_contents)
613)     path_parts = [part for part in path.split('/') if part]
614)     for part in path_parts[:-1]:
615)         config = config.setdefault(part, {})
616)     if path_parts:
617)         config[path_parts[-1]] = contents
618) 
619) 
Marco Ricci Apply new ruff ruleset to c...

Marco Ricci authored 3 months ago

620) def export_storeroom_data(  # noqa: C901,PLR0912,PLR0914,PLR0915
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

621)     storeroom_path: str | bytes | os.PathLike | None = None,
622)     master_keys_key: str | bytes | None = None,
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

623) ) -> dict[str, Any]:
624)     """Export the full configuration stored in the storeroom.
625) 
626)     Args:
627)         storeroom_path:
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

628)             Path to the storeroom; usually `~/.vault`.  If not given,
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

629)             then query [`exporter.get_vault_path`][] for the value.
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

630)         master_keys_key:
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

631)             Encryption key/password for the master keys, usually the
632)             username, or passed via the `VAULT_KEY` environment
633)             variable.  If not given, then query
Marco Ricci Generate nicer documentatio...

Marco Ricci authored 2 months ago

634)             [`exporter.get_vault_key`][] for the value.
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

635) 
636)     Returns:
637)         The full configuration, as stored in the storeroom.
638) 
639)         This may or may not be a valid configuration according to vault
640)         or derivepassphrase.
641) 
642)     Raises:
643)         RuntimeError:
644)             Something went wrong during data collection, e.g. we
645)             encountered unsupported or corrupted data in the storeroom.
646)         json.JSONDecodeError:
647)             An internal JSON data structure failed to parse from disk.
648)             The storeroom is probably corrupted.
649) 
650)     """
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

651)     if storeroom_path is None:
652)         storeroom_path = exporter.get_vault_path()
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

653)     if master_keys_key is None:
Marco Ricci Move vault key and path det...

Marco Ricci authored 4 months ago

654)         master_keys_key = exporter.get_vault_key()
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

655)     with open(
656)         os.path.join(os.fsdecode(storeroom_path), '.keys'), encoding='utf-8'
657)     ) as master_keys_file:
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

658)         header = json.loads(master_keys_file.readline())
659)         if header != {'version': 1}:
660)             msg = 'bad or unsupported keys version header'
661)             raise RuntimeError(msg)
662)         raw_keys_data = base64.standard_b64decode(master_keys_file.readline())
663)         encrypted_keys_params, encrypted_keys = struct.unpack(
664)             f'B {len(raw_keys_data) - 1}s', raw_keys_data
665)         )
666)         if master_keys_file.read():
667)             msg = 'trailing data; cannot make sense of .keys file'
668)             raise RuntimeError(msg)
669)     encrypted_keys_version = encrypted_keys_params >> 4
670)     if encrypted_keys_version != 1:
671)         msg = f'cannot handle version {encrypted_keys_version} encrypted keys'
672)         raise RuntimeError(msg)
673)     encrypted_keys_iterations = 2 ** (10 + (encrypted_keys_params & 0x0F))
674)     master_keys_keys = derive_master_keys_keys(
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

675)         master_keys_key, encrypted_keys_iterations
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

676)     )
677)     master_keys = decrypt_master_keys_data(encrypted_keys, master_keys_keys)
678) 
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

679)     config_structure: dict[str, Any] = {}
680)     json_contents: dict[str, bytes] = {}
Marco Ricci Support exports from outsid...

Marco Ricci authored 4 months ago

681)     for file in glob.glob(
682)         '[01][0-9a-f]', root_dir=os.fsdecode(storeroom_path)
683)     ):
684)         bucket_contents = list(
685)             decrypt_bucket_file(file, master_keys, root_dir=storeroom_path)
686)         )
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

687)         bucket_index = json.loads(bucket_contents.pop(0))
688)         for pos, item in enumerate(bucket_index):
689)             json_contents[item] = bucket_contents[pos]
690)             logger.debug(
691)                 'Found bucket item: %s -> %s', item, bucket_contents[pos]
692)             )
693)     dirs_to_check: dict[str, list[str]] = {}
694)     json_payload: Any
695)     for path, json_content in sorted(json_contents.items()):
696)         if path.endswith('/'):
697)             logger.debug(
698)                 'Postponing dir check: %s -> %s',
699)                 path,
700)                 json_content.decode('utf-8'),
701)             )
702)             json_payload = json.loads(json_content)
703)             if not isinstance(json_payload, list) or any(
704)                 not isinstance(x, str) for x in json_payload
705)             ):
706)                 msg = (
707)                     f'Directory index is not actually an index: '
708)                     f'{json_content!r}'
709)                 )
710)                 raise RuntimeError(msg)
711)             dirs_to_check[path] = json_payload
712)             logger.debug(
713)                 'Setting contents (empty directory): %s -> %s', path, '{}'
714)             )
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

715)             _store(config_structure, path, b'{}')
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

716)         else:
717)             logger.debug(
718)                 'Setting contents: %s -> %s',
719)                 path,
720)                 json_content.decode('utf-8'),
721)             )
Marco Ricci Add vault_native exporter f...

Marco Ricci authored 3 months ago

722)             _store(config_structure, path, json_content)
Marco Ricci Add an actual storeroom exp...

Marco Ricci authored 4 months ago

723)     for _dir, namelist in dirs_to_check.items():
724)         namelist = [x.rstrip('/') for x in namelist]  # noqa: PLW2901
725)         try:
726)             obj = config_structure
727)             for part in _dir.split('/'):
728)                 if part:
729)                     obj = obj[part]
730)         except KeyError as exc:
731)             msg = f'Cannot traverse storage path: {_dir!r}'
732)             raise RuntimeError(msg) from exc
733)         if set(obj.keys()) != set(namelist):
734)             msg = f'Object key mismatch for path {_dir!r}'
735)             raise RuntimeError(msg)
736)     return config_structure
Marco Ricci Add prototype for "storeroo...

Marco Ricci authored 4 months ago

737) 
738) 
739) if __name__ == '__main__':