derivepassphrase.git

@@ -47,7 +47,9 @@ def _load_data(

                 raise ModuleNotFoundError

             with open(path, 'rb') as infile:

                 contents = base64.standard_b64decode(infile.read())

-            return module.VaultNativeV02ConfigParser(contents, key)()

+            return module.export_vault_native_data(

+                contents, key, try_formats=['v0.2']

+            )

         case 'v0.3':

             module = importlib.import_module(

                 'derivepassphrase.exporter.vault_native'

@@ -56,7 +58,9 @@ def _load_data(

                 raise ModuleNotFoundError

             with open(path, 'rb') as infile:

                 contents = base64.standard_b64decode(infile.read())

-            return module.VaultNativeV03ConfigParser(contents, key)()

+            return module.export_vault_native_data(

+                contents, key, try_formats=['v0.3']

+            )

         case 'storeroom':

             module = importlib.import_module(

                 'derivepassphrase.exporter.storeroom'

@@ -2,7 +2,24 @@

 #

 # SPDX-License-Identifier: MIT

-"""Exporter for the vault "storeroom" configuration format."""

+"""Exporter for the vault "storeroom" configuration format.

+

+The "storeroom" format is the experimental format used in alpha and beta

+versions of vault beyond v0.3.0.  The configuration is stored as

+a separate directory, which acts like a hash table (i.e. has named

+slots) and provides an impure quasi-filesystem interface.  Each hash

+table entry is separately encrypted and authenticated.  James Coglan

+designed this format to avoid concurrent write issues when updating or

+synchronizing the vault configuration with e.g. a cloud service.

+

+The public interface is the

+[`derivepassphrase.exporter.storeroom.export_storeroom_data`][]

+function.  Multiple *non-public* functions are additionally documented

+here for didactical and educational reasons, but they are not part of

+the module API, are subject to change without notice (including

+removal), and should *not* be used or relied on.

+

+"""

 from __future__ import annotations

@@ -58,6 +75,8 @@ KEY_SIZE = MAC_SIZE = 32

 ENCRYPTED_KEYPAIR_SIZE = 128

 VERSION_SIZE = 1

+__all__ = ('export_storeroom_data',)

+

 logger = logging.getLogger(__name__)

@@ -119,6 +138,11 @@ def derive_master_keys_keys(password: str | bytes, iterations: int) -> KeyPair:

         A 2-tuple of keys, the encryption key and the signing key, to

         decrypt and verify the master keys data with.

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     if isinstance(password, str):

         password = password.encode('ASCII')

@@ -195,6 +219,11 @@ def decrypt_master_keys_data(data: bytes, keys: KeyPair) -> MasterKeys:

             example, it contains an unsupported version marker, or

             unexpected extra contents, or invalid padding.)

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     ciphertext, claimed_mac = struct.unpack(

         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data

@@ -285,6 +314,11 @@ def decrypt_session_keys(data: bytes, master_keys: MasterKeys) -> KeyPair:

             example, it contains an unsupported version marker, or

             unexpected extra contents, or invalid padding.)

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     ciphertext, claimed_mac = struct.unpack(

         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data

@@ -391,6 +425,11 @@ def decrypt_contents(data: bytes, session_keys: KeyPair) -> bytes:

             example, it contains an unsupported version marker, or

             unexpected extra contents, or invalid padding.)

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     ciphertext, claimed_mac = struct.unpack(

         f'{len(data) - MAC_SIZE}s {MAC_SIZE}s', data

@@ -466,6 +505,11 @@ def decrypt_bucket_item(bucket_item: bytes, master_keys: MasterKeys) -> bytes:

             example, it contains an unsupported version marker, or

             unexpected extra contents, or invalid padding.)

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     logger.debug(

         (

@@ -497,7 +541,7 @@ def decrypt_bucket_file(

     *,

     root_dir: str | bytes | os.PathLike = '.',

 ) -> Iterator[bytes]:

-    """Decrypt a bucket item.

+    """Decrypt a complete bucket.

     Args:

         filename:

@@ -524,6 +568,11 @@ def decrypt_bucket_file(

             example, it contains an unsupported version marker, or

             unexpected extra contents, or invalid padding.)

+    Warning:

+        Non-public function, provided for didactical and educational

+        purposes only.  Subject to change without notice, including

+        removal.

+

     """

     with open(

         os.path.join(os.fsdecode(root_dir), filename), 'rb'

@@ -543,7 +592,7 @@ def decrypt_bucket_file(

             )

-def store(config: dict[str, Any], path: str, json_contents: bytes) -> None:

+def _store(config: dict[str, Any], path: str, json_contents: bytes) -> None:

     """Store the JSON contents at path in the config structure.

     Traverse the config structure according to path, and set the value

@@ -669,14 +718,14 @@ def export_storeroom_data(  # noqa: C901,PLR0912,PLR0914,PLR0915

             logger.debug(

                 'Setting contents (empty directory): %s -> %s', path, '{}'

             )

-            store(config_structure, path, b'{}')

+            _store(config_structure, path, b'{}')

         else:

             logger.debug(

                 'Setting contents: %s -> %s',

                 path,

                 json_content.decode('utf-8'),

             )

-            store(config_structure, path, json_content)

+            _store(config_structure, path, json_content)

     for _dir, namelist in dirs_to_check.items():

         namelist = [x.rstrip('/') for x in namelist]  # noqa: PLW2901

         try:

@@ -2,7 +2,25 @@

 #

 # SPDX-License-Identifier: MIT

-"""Exporter for the vault native configuration format (v0.2 or v0.3)."""

+"""Exporter for the vault native configuration format (v0.2 or v0.3).

+

+The vault native formats are the configuration formats used by vault

+v0.2 and v0.3.  The configuration is stored as a single encrypted file,

+which is encrypted and authenticated.  v0.2 and v0.3 differ in some

+details concerning key derivation and expected format of internal

+structures, so they are *not* compatible.  v0.2 additionally contains

+cryptographic weaknesses (API misuse of a key derivation function, and

+a low-entropy method of generating initialization vectors for CBC block

+encryption mode) and should thus be avoided if possible.

+

+The public interface is the

+[`derivepassphrase.exporter.vault_native.export_vault_native_data`][]

+function.  Multiple *non-public* classes are additionally documented

+here for didactical and educational reasons, but they are not part of

+the module API, are subject to change without notice (including

+removal), and should *not* be used or relied on.

+

+"""

 from __future__ import annotations

@@ -16,6 +34,7 @@ from typing import TYPE_CHECKING

 from derivepassphrase import exporter, vault

 if TYPE_CHECKING:

+    from collections.abc import Sequence

     from typing import Any

     from typing_extensions import Buffer

@@ -57,6 +76,8 @@ else:

     else:

         STUBBED = False

+__all__ = ('export_vault_native_data',)

+

 logger = logging.getLogger(__name__)

@@ -93,6 +114,11 @@ class VaultNativeConfigParser(abc.ABC):

                 If this is a text string, then the UTF-8 encoding of the

                 string is used as the binary password.

+        Warning:

+            Non-public class, provided for didactical and educational

+            purposes only. Subject to change without notice, including

+            removal.

+

         """

         if not password:

             msg = 'Password must not be empty'

@@ -237,16 +263,21 @@ class VaultNativeV03ConfigParser(VaultNativeConfigParser):

     This is the modern, pre-storeroom configuration format.

+    Warning:

+        Non-public class, provided for didactical and educational

+        purposes only. Subject to change without notice, including

+        removal.

+

     """

     KEY_SIZE = 32

-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # noqa: ANN401,D107

+    def __init__(self, *args: Any, **kwargs: Any) -> None:  # noqa: ANN401

         super().__init__(*args, **kwargs)

         self._iv_size = 16

         self._mac_size = 32

-    def __call__(self) -> Any:  # noqa: ANN401,D102

+    def __call__(self) -> Any:  # noqa: ANN401

         if self._data is self._sentinel:

             logger.info('Attempting to parse as v0.3 configuration')

             return super().__call__()

@@ -277,14 +308,19 @@ class VaultNativeV02ConfigParser(VaultNativeConfigParser):

     v0.2 configurations should be upgraded to at least v0.3 as soon as

     possible.

+    Warning:

+        Non-public class, provided for didactical and educational

+        purposes only. Subject to change without notice, including

+        removal.

+

     """

-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # noqa: ANN401,D107

+    def __init__(self, *args: Any, **kwargs: Any) -> None:  # noqa: ANN401

         super().__init__(*args, **kwargs)

         self._iv_size = 16

         self._mac_size = 64

-    def __call__(self) -> Any:  # noqa: ANN401,D102

+    def __call__(self) -> Any:  # noqa: ANN401

         if self._data is self._sentinel:

             logger.info('Attempting to parse as v0.2 configuration')

             return super().__call__()

@@ -355,6 +391,84 @@ class VaultNativeV02ConfigParser(VaultNativeConfigParser):

         ).decryptor()

+def export_vault_native_data(

+    contents: Buffer | None = None,

+    key: str | Buffer | None = None,

+    *,

+    try_formats: Sequence[str] = ('v0.3', 'v0.2'),

+) -> Any:  # noqa: ANN401

+    """Export the full configuration stored in vault native format.

+

+    Args:

+        contents:

+            The binary encrypted contents of the vault configuration

+            file.  If not given, then query

+            [`derivepassphrase.exporter.get_vault_path`][] for the

+            correct filename and read the contents from there.

+

+            Note: On disk, these are usually stored in base64-encoded

+            form, not in the "raw" form as needed here.

+        key:

+            Encryption key/password for the configuration file, usually

+            the username, or passed via the `VAULT_KEY` environment

+            variable.  If not given, then query

+            [`derivepassphrase.exporter.get_vault_key`][] for the value.

+        try_formats:

+            A sequence of formats to try out, in order.  Each key must

+            be one of `v0.2` or `v0.3`.

+

+    Returns:

+        The vault configuration, as recorded in the configuration file.

+

+        This may or may not be a valid configuration according to vault

+        or derivepassphrase.

+

+    Raises:

+        RuntimeError:

+            Something went wrong during data collection, e.g. we

+            encountered unsupported or corrupted data in the storeroom.

+        json.JSONDecodeError:

+            An internal JSON data structure failed to parse from disk.

+            The storeroom is probably corrupted.

+        ValueError:

+            The requested formats to try out are invalid, or the

+            encrypted contents aren't in any of the attempted

+            configuration formats.

+

+    """

+    if contents is None:

+        with open(exporter.get_vault_path(), 'rb') as infile:

+            contents = base64.standard_b64decode(infile.read())

+    if key is None:

+        key = exporter.get_vault_key()

+    stored_exception: Exception | None = None

+    for config_format in try_formats:

+        match config_format:

+            case 'v0.2':

+                try:

+                    return VaultNativeV02ConfigParser(contents, key)()

+                except ValueError as exc:

+                    exc.__context__ = stored_exception

+                    stored_exception = exc

+            case 'v0.3':

+                try:

+                    return VaultNativeV03ConfigParser(contents, key)()

+                except ValueError as exc:

+                    exc.__context__ = stored_exception

+                    stored_exception = exc

+            case _:  # pragma: no cover

+                msg = (

+                    f'Invalid vault native configuration format: '

+                    f'{config_format!r}'

+                )

+                raise ValueError(msg)

+    msg = (

+        f'Not a valid vault native configuration. '

+        f'(We tried: {try_formats!r}.)'

+    )

+    raise stored_exception or ValueError(msg)

+

+

 if __name__ == '__main__':

     import os

@@ -323,6 +323,19 @@ class TestVaultNativeConfig:

             == result

         )

+    def test_201_export_vault_native_data_no_arguments(

+        self, monkeypatch: pytest.MonkeyPatch

+    ) -> None:

+        runner = click.testing.CliRunner(mix_stderr=False)

+        with tests.isolated_vault_exporter_config(

+            monkeypatch=monkeypatch,

+            runner=runner,

+            vault_config=tests.VAULT_V03_CONFIG,

+            vault_key=tests.VAULT_MASTER_KEY,

+        ):

+            parsed_config = vault_native.export_vault_native_data(None)

+        assert parsed_config == tests.VAULT_V03_CONFIG_DATA

+

     @pytest.mark.parametrize(

         ['parser_class', 'config', 'result'],

         [