Marco Ricci commited on 2026-06-14 22:46:36
Zeige 12 geänderte Dateien mit 207 Einfügungen und 130 Löschungen.
Tweak the hypothesis settings for determinism and speed when running the
test suite solely for the purpose of regression testing [1], in the new
hypothesis profile "regression": test generation will be derandomized,
stateful step counts and example counts will be set very low. As all of
our slow-running tests are hypothesis-based, and all of our
hypothesis-based tests are slow because of the high example count, this
change alone immediately makes "regression-only" testing feasible for us
without having to skip hypothesis-based tests. (In fact, the hypothesis
derandomization feature appears to draw motivation from the same source
[1] as we do.)
As a counterpoint, mark test cases that are (or probably should be)
hypothesis-based, but test for completion or correctness purposes rather
than for regression purposes, with a custom marker
`correctness_focused`.
Also try to add at least one working example and one counterexample to
each hypothesis-based test, if feasible. This buys us the flexibility
(in the future) to remove test case generation from the "regression"
profile, if desired. The test cases would then still run the explicit
examples, and thus remain useful, instead of needing another separate
set of non-hypothesis tests to cover the same ground.
References:
1. Nelson Elhage: [Two kinds of testing][], particularly the section
"Test budget".
2. Alex Kladov: [How to Test][], particularly the section "Make Tests
Fast".
[Two kinds of testing]: https://blog.nelhage.com/post/two-kinds-of-testing/
[How to Test]: https://matklad.github.io/2021/05/31/how-to-test.html#Make-Tests-Fast
| ... | ... |
@@ -31,16 +31,27 @@ startup_ssh_auth_sock = os.environ.get("SSH_AUTH_SOCK", None)
|
| 31 | 31 |
|
| 32 | 32 |
|
| 33 | 33 |
def pytest_configure(config: pytest.Config) -> None: |
| 34 |
- """Configure `pytest`: add the `heavy_duty` marker.""" |
|
| 34 |
+ """Configure `pytest`: add custom markers.""" |
|
| 35 | 35 |
config.addinivalue_line( |
| 36 | 36 |
"markers", |
| 37 | 37 |
( |
| 38 | 38 |
"heavy_duty: " |
| 39 |
- "mark test as a slow, heavy-duty test (e.g., an integration test)" |
|
| 40 |
- "\n" |
|
| 39 |
+ "mark test as a slow, heavy-duty test" |
|
| 40 |
+ ) |
|
| 41 |
+ ) |
|
| 42 |
+ config.addinivalue_line( |
|
| 43 |
+ "markers", |
|
| 44 |
+ ( |
|
| 45 |
+ "correctness_focused: " |
|
| 46 |
+ "mark test as a potentially slow, correctness-focused test" |
|
| 47 |
+ ) |
|
| 48 |
+ ) |
|
| 49 |
+ config.addinivalue_line( |
|
| 50 |
+ "markers", |
|
| 51 |
+ ( |
|
| 41 | 52 |
"non_reentrant_agent: " |
| 42 | 53 |
"mark the agent (fixture output) as non-reentrant" |
| 43 |
- ), |
|
| 54 |
+ ) |
|
| 44 | 55 |
) |
| 45 | 56 |
hypothesis_machinery._hypothesis_settings_setup() |
| 46 | 57 |
|
| ... | ... |
@@ -17,7 +17,6 @@ All similar-minded code requiring only plain `pytest` lives in [the |
| 17 | 17 |
from __future__ import annotations |
| 18 | 18 |
|
| 19 | 19 |
import copy |
| 20 |
-import datetime |
|
| 21 | 20 |
import importlib |
| 22 | 21 |
import importlib.util |
| 23 | 22 |
import math |
| ... | ... |
@@ -94,36 +93,45 @@ def _hypothesis_settings_setup() -> None: |
| 94 | 93 |
and isinstance(trace_func.__self__, pytracer_class) |
| 95 | 94 |
): |
| 96 | 95 |
slowdown = 8.0 |
| 97 |
- settings = hypothesis.settings( |
|
| 96 |
+ hypothesis.settings.register_profile( |
|
| 97 |
+ "default", |
|
| 98 |
+ parent=settings, |
|
| 98 | 99 |
deadline=slowdown * settings.deadline |
| 99 |
- if slowdown |
|
| 100 |
+ if slowdown and settings.deadline is not None |
|
| 100 | 101 |
else settings.deadline, |
| 101 | 102 |
stateful_step_count=32, |
| 102 |
- suppress_health_check=(hypothesis.HealthCheck.too_slow,), |
|
| 103 |
+ suppress_health_check=[hypothesis.HealthCheck.too_slow], |
|
| 103 | 104 |
) |
| 104 |
- hypothesis.settings.register_profile("default", settings)
|
|
| 105 |
+ default_profile = hypothesis.settings.get_profile("default")
|
|
| 105 | 106 |
hypothesis.settings.register_profile( |
| 106 |
- "dev", derandomize=True, max_examples=10 |
|
| 107 |
+ "regression", |
|
| 108 |
+ parent=hypothesis.settings.get_profile("ci"),
|
|
| 109 |
+ derandomize=True, |
|
| 110 |
+ database=None, |
|
| 111 |
+ max_examples=3, |
|
| 112 |
+ stateful_step_count=3, |
|
| 113 |
+ suppress_health_check=[hypothesis.HealthCheck.too_slow], |
|
| 114 |
+ phases=[ |
|
| 115 |
+ hypothesis.Phase.explicit, |
|
| 116 |
+ hypothesis.Phase.reuse, |
|
| 117 |
+ hypothesis.Phase.generate, |
|
| 118 |
+ ], |
|
| 107 | 119 |
) |
| 108 | 120 |
hypothesis.settings.register_profile( |
| 109 | 121 |
"debug", |
| 110 |
- parent=hypothesis.settings.get_profile("dev"),
|
|
| 122 |
+ parent=default_profile, |
|
| 111 | 123 |
verbosity=hypothesis.Verbosity.verbose, |
| 112 | 124 |
) |
| 113 | 125 |
hypothesis.settings.register_profile( |
| 114 |
- "flaky", |
|
| 115 |
- deadline=( |
|
| 116 |
- settings.deadline - settings.deadline // 4 |
|
| 117 |
- if settings.deadline is not None |
|
| 118 |
- else datetime.timedelta(milliseconds=150) |
|
| 119 |
- ), |
|
| 126 |
+ "debug-regression", |
|
| 127 |
+ parent=hypothesis.settings.get_profile("regression"),
|
|
| 128 |
+ verbosity=hypothesis.Verbosity.verbose, |
|
| 120 | 129 |
) |
| 121 |
- ci_profile = hypothesis.settings.get_profile("ci")
|
|
| 122 | 130 |
hypothesis.settings.register_profile( |
| 123 |
- "intense", |
|
| 124 |
- parent=ci_profile, |
|
| 131 |
+ "correctness", |
|
| 132 |
+ parent=default_profile, |
|
| 125 | 133 |
derandomize=False, |
| 126 |
- max_examples=10 * ci_profile.max_examples, |
|
| 134 |
+ max_examples=10 * default_profile.max_examples, |
|
| 127 | 135 |
) |
| 128 | 136 |
|
| 129 | 137 |
|
| ... | ... |
@@ -161,13 +169,13 @@ def get_process_spawning_state_machine_examples_count( |
| 161 | 169 |
examples count by default, and require the user to opt-in to the |
| 162 | 170 |
original naive example count explicitly. |
| 163 | 171 |
|
| 164 |
- If the "intense" profile is in effect, or something with even higher |
|
| 165 |
- `max_examples` and `stateful_step_count`, then we return the |
|
| 172 |
+ If the "correctness" profile is in effect, or something with even |
|
| 173 |
+ higher `max_examples` and `stateful_step_count`, then we return the |
|
| 166 | 174 |
unaltered example count for the *default* profile. Otherwise, we |
| 167 | 175 |
return the square root of the `max_examples` setting (rounded down). |
| 168 |
- We *never* return a value below the "dev" profile's example count: |
|
| 169 |
- any lower computed example count is increased to the "dev" profile's |
|
| 170 |
- example count. |
|
| 176 |
+ We *never* return a value below the "regression" profile's example |
|
| 177 |
+ count: any lower computed example count is increased to the |
|
| 178 |
+ "regression" profile's example count. |
|
| 171 | 179 |
|
| 172 | 180 |
Args: |
| 173 | 181 |
settings: |
| ... | ... |
@@ -182,20 +190,22 @@ def get_process_spawning_state_machine_examples_count( |
| 182 | 190 |
_hypothesis_settings_setup() |
| 183 | 191 |
|
| 184 | 192 |
these_values = (settings.max_examples, settings.stateful_step_count) |
| 185 |
- intense_profile = hypothesis.settings.get_profile("intense")
|
|
| 186 |
- intense_values = ( |
|
| 187 |
- intense_profile.max_examples, |
|
| 188 |
- intense_profile.stateful_step_count, |
|
| 193 |
+ correctness_profile = hypothesis.settings.get_profile("correctness")
|
|
| 194 |
+ correctness_values = ( |
|
| 195 |
+ correctness_profile.max_examples, |
|
| 196 |
+ correctness_profile.stateful_step_count, |
|
| 189 | 197 |
) |
| 190 |
- min_count = hypothesis.settings.get_profile("dev").max_examples
|
|
| 198 |
+ min_count = hypothesis.settings.get_profile("regression").max_examples
|
|
| 191 | 199 |
high_count = hypothesis.settings.get_profile("default").max_examples
|
| 192 |
- we_are_intense = ( |
|
| 193 |
- these_values[0] >= intense_values[0] |
|
| 194 |
- and these_values[1] >= intense_values[1] |
|
| 200 |
+ correctness_based_testing = ( |
|
| 201 |
+ these_values[0] >= correctness_values[0] |
|
| 202 |
+ and these_values[1] >= correctness_values[1] |
|
| 195 | 203 |
) |
| 196 | 204 |
return max( |
| 197 | 205 |
min_count, |
| 198 |
- high_count if we_are_intense else math.isqrt(settings.max_examples), |
|
| 206 |
+ high_count |
|
| 207 |
+ if correctness_based_testing |
|
| 208 |
+ else math.isqrt(settings.max_examples), |
|
| 199 | 209 |
) |
| 200 | 210 |
|
| 201 | 211 |
|
| ... | ... |
@@ -143,19 +143,30 @@ def xfail_on_the_annoying_os( |
| 143 | 143 |
|
| 144 | 144 |
heavy_duty = pytest.mark.heavy_duty |
| 145 | 145 |
""" |
| 146 |
-A cached `pytest` mark indicating that this test function/class/module |
|
| 147 |
-is a slow, heavy duty test. Users who are impatient (or otherwise |
|
| 148 |
-cannot afford to wait for these tests to complete) may wish to exclude |
|
| 149 |
-these tests; this mark helps in achieving that. |
|
| 146 |
+Test functions/classes/modules with this `pytest` mark are slow, heavy |
|
| 147 |
+duty tests. Users who are impatient (or otherwise cannot afford to wait |
|
| 148 |
+for these tests to complete) may wish to exclude these tests; this mark |
|
| 149 |
+helps in achieving that. |
|
| 150 |
+""" |
|
| 150 | 151 |
|
| 151 |
-All current heavy duty tests are integration tests. |
|
| 152 |
+correctness_focused = pytest.mark.correctness_focused |
|
| 152 | 153 |
""" |
| 154 |
+Test functions/classes/modules with this `pytest` mark are designed |
|
| 155 |
+primarily to verify correctness and to find novel failure cases, even at |
|
| 156 |
+the cost of high CPU or wall clock time. |
|
| 157 |
+ |
|
| 158 |
+Contrast this with tests designed primarily to verify that the existing |
|
| 159 |
+system still behaves as before in all the important ways ("is healthy")
|
|
| 160 |
+even after introducing a small change, but before committing it. (See |
|
| 161 |
+[Two kinds of testing][TWO_KINDS] for more elaboration.) |
|
| 153 | 162 |
|
| 163 |
+[TWO_KINDS]: https://blog.nelhage.com/post/two-kinds-of-testing/ |
|
| 164 |
+""" |
|
| 154 | 165 |
|
| 155 | 166 |
non_isolated_agent_use = pytest.mark.xdist_group("non_isolated_agent_use")
|
| 156 | 167 |
""" |
| 157 |
-A cached `pytest` mark, for use with xdist's "loadgroup" scheduling, |
|
| 158 |
-that marks a test as part of the group of tests that use a potentially |
|
| 168 |
+Tests with this `pytest` mark (for use with xdist's "loadgroup" |
|
| 169 |
+scheduling), comprise the group of tests that use a potentially |
|
| 159 | 170 |
non-isolated agent, and thus need to be run serially, in the same worker |
| 160 | 171 |
(because of the setup and teardown). |
| 161 | 172 |
|
| ... | ... |
@@ -1245,7 +1245,9 @@ class TestTempdir: |
| 1245 | 1245 |
), |
| 1246 | 1246 |
), |
| 1247 | 1247 |
) |
| 1248 |
- @hypothesis.example(env_var="", suffix=".") |
|
| 1248 |
+ @hypothesis.example(env_var="", suffix=".").via( |
|
| 1249 |
+ "static example; branch coverage (test)" |
|
| 1250 |
+ ) |
|
| 1249 | 1251 |
def test_get_tempdir( |
| 1250 | 1252 |
self, |
| 1251 | 1253 |
env_var: str, |
| ... | ... |
@@ -141,6 +141,9 @@ class TestNotesPrinting: |
| 141 | 141 |
|
| 142 | 142 |
@Parametrize.NOTES_PLACEMENT |
| 143 | 143 |
@hypothesis.given(notes=Strategies.notes().filter(str.strip)) |
| 144 |
+ @hypothesis.example(notes=" ").xfail( |
|
| 145 |
+ reason="empty notes", raises=AssertionError |
|
| 146 |
+ ) |
|
| 144 | 147 |
def test_notes_placement( |
| 145 | 148 |
self, |
| 146 | 149 |
notes_placement: Literal["before", "after"], |
| ... | ... |
@@ -347,11 +350,15 @@ class TestNotesEditingValid(TestNotesEditing): |
| 347 | 350 |
], |
| 348 | 351 |
) |
| 349 | 352 |
@hypothesis.given( |
| 350 |
- notes=Strategies.notes() |
|
| 353 |
+ notes=Strategies |
|
| 354 |
+ .notes() |
|
| 351 | 355 |
.filter(str.strip) |
| 352 | 356 |
.filter(lambda notes: notes != TestNotesEditingValid.CURRENT_NOTES) |
| 353 | 357 |
) |
| 354 | 358 |
@hypothesis.example(TestNotesEditing.CURRENT_NOTES) |
| 359 |
+ @hypothesis.example(notes=" ").xfail( |
|
| 360 |
+ reason="empty notes", raises=AssertionError |
|
| 361 |
+ ) |
|
| 355 | 362 |
def test_successful_edit( |
| 356 | 363 |
self, |
| 357 | 364 |
caplog: pytest.LogCaptureFixture, |
| ... | ... |
@@ -463,6 +470,9 @@ class TestNotesEditingValid(TestNotesEditing): |
| 463 | 470 |
], |
| 464 | 471 |
) |
| 465 | 472 |
@hypothesis.given(notes=Strategies.notes().filter(str.strip)) |
| 473 |
+ @hypothesis.example(notes=" ").xfail( |
|
| 474 |
+ reason="empty notes", raises=AssertionError |
|
| 475 |
+ ) |
|
| 466 | 476 |
def test_marker_removed( |
| 467 | 477 |
self, |
| 468 | 478 |
caplog: pytest.LogCaptureFixture, |
| ... | ... |
@@ -509,7 +519,7 @@ class TestNotesEditingInvalid(TestNotesEditing): |
| 509 | 519 |
"""Tests concerning editing service notes: invalid/error calls.""" |
| 510 | 520 |
|
| 511 | 521 |
@hypothesis.given(notes=Strategies.notes()) |
| 512 |
- @hypothesis.example("")
|
|
| 522 |
+ @hypothesis.example(notes="") |
|
| 513 | 523 |
def test_abort( |
| 514 | 524 |
self, |
| 515 | 525 |
notes: str, |
| ... | ... |
@@ -548,7 +558,7 @@ class TestNotesEditingInvalid(TestNotesEditing): |
| 548 | 558 |
], |
| 549 | 559 |
) |
| 550 | 560 |
@hypothesis.given(notes=Strategies.notes()) |
| 551 |
- @hypothesis.example("")
|
|
| 561 |
+ @hypothesis.example(notes="") |
|
| 552 | 562 |
def test_fail_on_config_option_missing( |
| 553 | 563 |
self, |
| 554 | 564 |
caplog: pytest.LogCaptureFixture, |
| ... | ... |
@@ -391,6 +391,7 @@ class TestStoreroom: |
| 391 | 391 |
== data.VAULT_STOREROOM_CONFIG_DATA |
| 392 | 392 |
) |
| 393 | 393 |
|
| 394 |
+ @pytest_machinery.correctness_focused |
|
| 394 | 395 |
def test_decrypt_bucket_item_unknown_version(self) -> None: |
| 395 | 396 |
"""Fail on unknown versions of the master keys file.""" |
| 396 | 397 |
bucket_item = ( |
| ... | ... |
@@ -404,6 +405,7 @@ class TestStoreroom: |
| 404 | 405 |
with pytest.raises(ValueError, match="Cannot handle version 255"): |
| 405 | 406 |
storeroom._decrypt_bucket_item(bucket_item, master_keys) |
| 406 | 407 |
|
| 408 |
+ @pytest_machinery.correctness_focused |
|
| 407 | 409 |
@Parametrize.BAD_CONFIG |
| 408 | 410 |
def test_decrypt_bucket_file_bad_json_or_version( |
| 409 | 411 |
self, |
| ... | ... |
@@ -427,6 +429,7 @@ class TestStoreroom: |
| 427 | 429 |
with pytest.raises(ValueError, match="Invalid bucket file: "): |
| 428 | 430 |
list(storeroom._decrypt_bucket_file(p, master_keys)) |
| 429 | 431 |
|
| 432 |
+ @pytest_machinery.correctness_focused |
|
| 430 | 433 |
@Parametrize.BAD_MASTER_KEYS_DATA |
| 431 | 434 |
@Parametrize.STOREROOM_HANDLER |
| 432 | 435 |
def test_export_storeroom_data_bad_master_keys_file( |
| ... | ... |
@@ -447,6 +450,7 @@ class TestStoreroom: |
| 447 | 450 |
with pytest.raises(RuntimeError, match=err_msg): |
| 448 | 451 |
handler(format="storeroom") |
| 449 | 452 |
|
| 453 |
+ @pytest_machinery.correctness_focused |
|
| 450 | 454 |
@Parametrize.BAD_STOREROOM_CONFIG_DATA |
| 451 | 455 |
@Parametrize.STOREROOM_HANDLER |
| 452 | 456 |
def test_export_storeroom_data_bad_directory_listing( |
| ... | ... |
@@ -476,6 +480,7 @@ class TestStoreroom: |
| 476 | 480 |
with pytest.raises(RuntimeError, match=error_text): |
| 477 | 481 |
handler(format="storeroom") |
| 478 | 482 |
|
| 483 |
+ @pytest_machinery.correctness_focused |
|
| 479 | 484 |
def test_decrypt_keys_wrong_data_length(self) -> None: |
| 480 | 485 |
"""Fail on internal structural data of the wrong size. |
| 481 | 486 |
|
| ... | ... |
@@ -527,6 +532,7 @@ class TestStoreroom: |
| 527 | 532 |
), |
| 528 | 533 |
) |
| 529 | 534 |
|
| 535 |
+ @pytest_machinery.correctness_focused |
|
| 530 | 536 |
@hypothesis.given( |
| 531 | 537 |
data=strategies.binary( |
| 532 | 538 |
min_size=storeroom.MAC_SIZE, max_size=storeroom.MAC_SIZE |
| ... | ... |
@@ -193,66 +193,86 @@ class TestCLIUtilities: |
| 193 | 193 |
) |
| 194 | 194 |
yield monkeypatch |
| 195 | 195 |
|
| 196 |
+ @hypothesis.given( |
|
| 197 |
+ vault_key_env=VaultKeyEnvironment.strategy().filter( |
|
| 198 |
+ lambda env: bool(env.expected) |
|
| 199 |
+ ), |
|
| 200 |
+ ) |
|
| 201 |
+ # BEGIN pre-hypothesis parametrization examples |
|
| 196 | 202 |
@hypothesis.example( |
| 197 |
- VaultKeyEnvironment("4username", None, None, None, "4username")
|
|
| 198 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 203 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 204 |
+ "4username", None, None, None, "4username" |
|
| 205 |
+ ) |
|
| 206 |
+ ) |
|
| 199 | 207 |
@hypothesis.example( |
| 200 |
- VaultKeyEnvironment("3user", None, None, "3user", None)
|
|
| 201 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 208 |
+ vault_key_env=VaultKeyEnvironment("3user", None, None, "3user", None)
|
|
| 209 |
+ ) |
|
| 202 | 210 |
@hypothesis.example( |
| 203 |
- VaultKeyEnvironment("3user", None, None, "3user", "4username")
|
|
| 204 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 211 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 212 |
+ "3user", None, None, "3user", "4username" |
|
| 213 |
+ ) |
|
| 214 |
+ ) |
|
| 205 | 215 |
@hypothesis.example( |
| 206 |
- VaultKeyEnvironment("2logname", None, "2logname", None, None)
|
|
| 207 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 216 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 217 |
+ "2logname", None, "2logname", None, None |
|
| 218 |
+ ) |
|
| 219 |
+ ) |
|
| 208 | 220 |
@hypothesis.example( |
| 209 |
- VaultKeyEnvironment("2logname", None, "2logname", None, "4username")
|
|
| 210 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 221 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 222 |
+ "2logname", None, "2logname", None, "4username" |
|
| 223 |
+ ) |
|
| 224 |
+ ) |
|
| 211 | 225 |
@hypothesis.example( |
| 212 |
- VaultKeyEnvironment("2logname", None, "2logname", "3user", None)
|
|
| 213 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 226 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 227 |
+ "2logname", None, "2logname", "3user", None |
|
| 228 |
+ ) |
|
| 229 |
+ ) |
|
| 214 | 230 |
@hypothesis.example( |
| 215 |
- VaultKeyEnvironment("2logname", None, "2logname", "3user", "4username")
|
|
| 216 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 231 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 232 |
+ "2logname", None, "2logname", "3user", "4username" |
|
| 233 |
+ ) |
|
| 234 |
+ ) |
|
| 217 | 235 |
@hypothesis.example( |
| 218 |
- VaultKeyEnvironment("1vault_key", "1vault_key", None, None, None)
|
|
| 219 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 236 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 237 |
+ "1vault_key", "1vault_key", None, None, None |
|
| 238 |
+ ) |
|
| 239 |
+ ) |
|
| 220 | 240 |
@hypothesis.example( |
| 221 |
- VaultKeyEnvironment( |
|
| 241 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 222 | 242 |
"1vault_key", "1vault_key", None, None, "4username" |
| 223 | 243 |
) |
| 224 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 244 |
+ ) |
|
| 225 | 245 |
@hypothesis.example( |
| 226 |
- VaultKeyEnvironment("1vault_key", "1vault_key", None, "3user", None)
|
|
| 227 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 246 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 247 |
+ "1vault_key", "1vault_key", None, "3user", None |
|
| 248 |
+ ) |
|
| 249 |
+ ) |
|
| 228 | 250 |
@hypothesis.example( |
| 229 |
- VaultKeyEnvironment( |
|
| 251 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 230 | 252 |
"1vault_key", "1vault_key", None, "3user", "4username" |
| 231 | 253 |
) |
| 232 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 254 |
+ ) |
|
| 233 | 255 |
@hypothesis.example( |
| 234 |
- VaultKeyEnvironment("1vault_key", "1vault_key", "2logname", None, None)
|
|
| 235 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 256 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 257 |
+ "1vault_key", "1vault_key", "2logname", None, None |
|
| 258 |
+ ) |
|
| 259 |
+ ) |
|
| 236 | 260 |
@hypothesis.example( |
| 237 |
- VaultKeyEnvironment( |
|
| 261 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 238 | 262 |
"1vault_key", "1vault_key", "2logname", None, "4username" |
| 239 | 263 |
) |
| 240 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 264 |
+ ) |
|
| 241 | 265 |
@hypothesis.example( |
| 242 |
- VaultKeyEnvironment( |
|
| 266 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 243 | 267 |
"1vault_key", "1vault_key", "2logname", "3user", None |
| 244 | 268 |
) |
| 245 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 269 |
+ ) |
|
| 246 | 270 |
@hypothesis.example( |
| 247 |
- VaultKeyEnvironment( |
|
| 271 |
+ vault_key_env=VaultKeyEnvironment( |
|
| 248 | 272 |
"1vault_key", "1vault_key", "2logname", "3user", "4username" |
| 249 | 273 |
) |
| 250 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 251 |
- @hypothesis.given( |
|
| 252 |
- vault_key_env=VaultKeyEnvironment.strategy().filter( |
|
| 253 |
- lambda env: bool(env.expected) |
|
| 254 |
- ), |
|
| 255 | 274 |
) |
| 275 |
+ # END pre-hypothesis parametrization examples |
|
| 256 | 276 |
def test_get_vault_key( |
| 257 | 277 |
self, |
| 258 | 278 |
vault_key_env: VaultKeyEnvironment, |
| ... | ... |
@@ -164,21 +164,24 @@ class TestStaticFunctionality: |
| 164 | 164 |
) |
| 165 | 165 |
|
| 166 | 166 |
@hypothesis.given(test_case=BigEndianNumberTest.strategy()) |
| 167 |
+ # decimal example |
|
| 167 | 168 |
@hypothesis.example( |
| 168 |
- BigEndianNumberTest([1, 2, 3, 4, 5, 6], 10, 123456) |
|
| 169 |
- ).via("manual decimal example")
|
|
| 169 |
+ test_case=BigEndianNumberTest([1, 2, 3, 4, 5, 6], 10, 123456) |
|
| 170 |
+ ) |
|
| 171 |
+ # decimal example, different base |
|
| 172 |
+ @hypothesis.example( |
|
| 173 |
+ test_case=BigEndianNumberTest([1, 2, 3, 4, 5, 6], 100, 10203040506) |
|
| 174 |
+ ) |
|
| 175 |
+ # leading zeroes example |
|
| 170 | 176 |
@hypothesis.example( |
| 171 |
- BigEndianNumberTest([1, 2, 3, 4, 5, 6], 100, 10203040506) |
|
| 172 |
- ).via("manual decimal example in different base")
|
|
| 173 |
- @hypothesis.example(BigEndianNumberTest([0, 0, 1, 4, 9, 7], 10, 1497)).via( |
|
| 174 |
- "manual example with leading zeroes" |
|
| 177 |
+ test_case=BigEndianNumberTest([0, 0, 1, 4, 9, 7], 10, 1497) |
|
| 175 | 178 |
) |
| 179 |
+ # binary example |
|
| 176 | 180 |
@hypothesis.example( |
| 177 |
- BigEndianNumberTest([1, 0, 0, 1, 0, 0, 0, 0], 2, 144) |
|
| 178 |
- ).via("manual binary example")
|
|
| 179 |
- @hypothesis.example(BigEndianNumberTest([1, 7, 5, 5], 8, 0o1755)).via( |
|
| 180 |
- "manual octal example" |
|
| 181 |
+ test_case=BigEndianNumberTest([1, 0, 0, 1, 0, 0, 0, 0], 2, 144) |
|
| 181 | 182 |
) |
| 183 |
+ # octal example |
|
| 184 |
+ @hypothesis.example(test_case=BigEndianNumberTest([1, 7, 5, 5], 8, 0o1755)) |
|
| 182 | 185 |
def test_big_endian_number(self, test_case: BigEndianNumberTest) -> None: |
| 183 | 186 |
"""Conversion to big endian numbers in any base works. |
| 184 | 187 |
|
| ... | ... |
@@ -294,22 +297,28 @@ class TestSequin: |
| 294 | 297 |
) |
| 295 | 298 |
|
| 296 | 299 |
@hypothesis.given(test_case=ConstructorTestCase.strategy()) |
| 300 |
+ # bitstring example |
|
| 297 | 301 |
@hypothesis.example( |
| 298 |
- ConstructorTestCase([1, 0, 0, 1, 0, 1], True, [1, 0, 0, 1, 0, 1]) |
|
| 299 |
- ).via("manual example bitstring")
|
|
| 302 |
+ test_case=ConstructorTestCase( |
|
| 303 |
+ [1, 0, 0, 1, 0, 1], True, [1, 0, 0, 1, 0, 1] |
|
| 304 |
+ ) |
|
| 305 |
+ ) |
|
| 306 |
+ # bitstring as bytestring example |
|
| 300 | 307 |
@hypothesis.example( |
| 301 |
- ConstructorTestCase( |
|
| 308 |
+ test_case=ConstructorTestCase( |
|
| 302 | 309 |
[1, 0, 0, 1, 0, 1], |
| 303 | 310 |
False, |
| 304 | 311 |
bitseq("000000010000000000000000000000010000000000000001"),
|
| 305 | 312 |
) |
| 306 |
- ).via("manual example bitstring as byte string")
|
|
| 313 |
+ ) |
|
| 314 |
+ # true byte string example |
|
| 307 | 315 |
@hypothesis.example( |
| 308 |
- ConstructorTestCase(b"OK", False, bitseq("0100111101001011"))
|
|
| 309 |
- ).via("manual example true byte string")
|
|
| 316 |
+ test_case=ConstructorTestCase(b"OK", False, bitseq("0100111101001011"))
|
|
| 317 |
+ ) |
|
| 318 |
+ # latin1 text example |
|
| 310 | 319 |
@hypothesis.example( |
| 311 |
- ConstructorTestCase("OK", False, bitseq("0100111101001011"))
|
|
| 312 |
- ).via("manual example latin1 text")
|
|
| 320 |
+ test_case=ConstructorTestCase("OK", False, bitseq("0100111101001011"))
|
|
| 321 |
+ ) |
|
| 313 | 322 |
def test_constructor( |
| 314 | 323 |
self, |
| 315 | 324 |
test_case: ConstructorTestCase, |
| ... | ... |
@@ -355,8 +364,9 @@ class TestSequin: |
| 355 | 364 |
], |
| 356 | 365 |
) |
| 357 | 366 |
|
| 367 |
+ @hypothesis.given(sequence=GenerationSequence.strategy()) |
|
| 358 | 368 |
@hypothesis.example( |
| 359 |
- GenerationSequence( |
|
| 369 |
+ sequence=GenerationSequence( |
|
| 360 | 370 |
bitseq("110101011111001"),
|
| 361 | 371 |
[ |
| 362 | 372 |
(1, 0), |
| ... | ... |
@@ -367,8 +377,7 @@ class TestSequin: |
| 367 | 377 |
(1, sequin.SequinExhaustedError), |
| 368 | 378 |
], |
| 369 | 379 |
) |
| 370 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 371 |
- @hypothesis.given(sequence=GenerationSequence.strategy()) |
|
| 380 |
+ ) |
|
| 372 | 381 |
def test_generating(self, sequence: GenerationSequence) -> None: |
| 373 | 382 |
"""The sequin generates deterministic sequences.""" |
| 374 | 383 |
seq = sequin.Sequin(sequence.bit_sequence, is_bitstring=True) |
| ... | ... |
@@ -396,8 +405,9 @@ class TestSequin: |
| 396 | 405 |
with pytest.raises(ValueError, match="invalid target range"): |
| 397 | 406 |
seq.generate(0) |
| 398 | 407 |
|
| 408 |
+ @hypothesis.given(sequence=GenerationSequence.strategy()) |
|
| 399 | 409 |
@hypothesis.example( |
| 400 |
- GenerationSequence( |
|
| 410 |
+ sequence=GenerationSequence( |
|
| 401 | 411 |
bitseq("110101011111001"),
|
| 402 | 412 |
[ |
| 403 | 413 |
(1, 0), |
| ... | ... |
@@ -408,8 +418,7 @@ class TestSequin: |
| 408 | 418 |
(1, sequin.SequinExhaustedError), |
| 409 | 419 |
], |
| 410 | 420 |
) |
| 411 |
- ).via("manual, pre-hypothesis parametrization value")
|
|
| 412 |
- @hypothesis.given(sequence=GenerationSequence.strategy()) |
|
| 421 |
+ ) |
|
| 413 | 422 |
def test_internal_generating(self, sequence: GenerationSequence) -> None: |
| 414 | 423 |
"""The sequin internals generate deterministic sequences.""" |
| 415 | 424 |
seq = sequin.Sequin(sequence.bit_sequence, is_bitstring=True) |
| ... | ... |
@@ -495,7 +504,7 @@ class TestSequin: |
| 495 | 504 |
|
| 496 | 505 |
@hypothesis.given(sequence=ShiftSequence.strategy()) |
| 497 | 506 |
@hypothesis.example( |
| 498 |
- ShiftSequence( |
|
| 507 |
+ sequence=ShiftSequence( |
|
| 499 | 508 |
bitseq("1010010001"),
|
| 500 | 509 |
[ |
| 501 | 510 |
(3, bitseq("101"), bitseq("0010001")),
|
| ... | ... |
@@ -533,17 +533,17 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality): |
| 533 | 533 |
uint32 = ssh_agent.SSHAgentClient.uint32 |
| 534 | 534 |
assert uint32(input) == expected |
| 535 | 535 |
|
| 536 |
- @hypothesis.given(strategies.integers(min_value=0, max_value=0xFFFFFFFF)) |
|
| 537 |
- @hypothesis.example(0xDEADBEEF).via("manual, pre-hypothesis example")
|
|
| 536 |
+ @hypothesis.given( |
|
| 537 |
+ num=strategies.integers(min_value=0, max_value=0xFFFFFFFF) |
|
| 538 |
+ ) |
|
| 539 |
+ @hypothesis.example(num=0xDEADBEEF) |
|
| 538 | 540 |
def test_uint32_from_number(self, num: int) -> None: |
| 539 | 541 |
"""`uint32` encoding works, starting from numbers.""" |
| 540 | 542 |
uint32 = ssh_agent.SSHAgentClient.uint32 |
| 541 | 543 |
assert int.from_bytes(uint32(num), "big", signed=False) == num |
| 542 | 544 |
|
| 543 |
- @hypothesis.given(strategies.binary(min_size=4, max_size=4)) |
|
| 544 |
- @hypothesis.example(b"\xde\xad\xbe\xef").via( |
|
| 545 |
- "manual, pre-hypothesis example" |
|
| 546 |
- ) |
|
| 545 |
+ @hypothesis.given(bytestring=strategies.binary(min_size=4, max_size=4)) |
|
| 546 |
+ @hypothesis.example(bytestring=b"\xde\xad\xbe\xef") |
|
| 547 | 547 |
def test_uint32_from_bytestring(self, bytestring: bytes) -> None: |
| 548 | 548 |
"""`uint32` encoding works, starting from length four byte strings.""" |
| 549 | 549 |
uint32 = ssh_agent.SSHAgentClient.uint32 |
| ... | ... |
@@ -560,10 +560,9 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality): |
| 560 | 560 |
string = ssh_agent.SSHAgentClient.string |
| 561 | 561 |
assert bytes(string(input)) == expected |
| 562 | 562 |
|
| 563 |
- @hypothesis.given(strategies.binary(max_size=0x0001FFFF)) |
|
| 564 |
- @hypothesis.example(b"DEADBEEF" * 10000).via( |
|
| 565 |
- "manual, pre-hypothesis example with highest order bit set" |
|
| 566 |
- ) |
|
| 563 |
+ @hypothesis.given(bytestring=strategies.binary(max_size=0x0001FFFF)) |
|
| 564 |
+ # example with highest order bit set |
|
| 565 |
+ @hypothesis.example(bytestring=b"DEADBEEF" * 10000) |
|
| 567 | 566 |
def test_string_from_bytestring(self, bytestring: bytes) -> None: |
| 568 | 567 |
"""SSH string encoding works, starting from a byte string.""" |
| 569 | 568 |
res = ssh_agent.SSHAgentClient.string(bytestring) |
| ... | ... |
@@ -584,13 +583,11 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality): |
| 584 | 583 |
b"", |
| 585 | 584 |
) |
| 586 | 585 |
|
| 587 |
- @hypothesis.given(strategies.binary(max_size=0x00FFFFFF)) |
|
| 588 |
- @hypothesis.example(b"\x00\x00\x00\x07ssh-rsa").via( |
|
| 589 |
- "manual, pre-hypothesis example to attempt to detect double-decoding" |
|
| 590 |
- ) |
|
| 591 |
- @hypothesis.example(b"\x00\x00\x00\x01").via( |
|
| 592 |
- "detect no-op encoding via ill-formed SSH string" |
|
| 593 |
- ) |
|
| 586 |
+ @hypothesis.given(bytestring=strategies.binary(max_size=0x00FFFFFF)) |
|
| 587 |
+ # should not double-decode |
|
| 588 |
+ @hypothesis.example(bytestring=b"\x00\x00\x00\x07ssh-rsa") |
|
| 589 |
+ # should not choke on ill-formed SSH string payloads |
|
| 590 |
+ @hypothesis.example(bytestring=b"\x00\x00\x00\x01") |
|
| 594 | 591 |
def test_unstring_of_string_of_data(self, bytestring: bytes) -> None: |
| 595 | 592 |
"""SSH string decoding of encoded SSH strings works. |
| 596 | 593 |
|
| ... | ... |
@@ -612,7 +609,7 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality): |
| 612 | 609 |
assert unstring_prefix(encoded2) == (bytestring, trailing_data) |
| 613 | 610 |
|
| 614 | 611 |
@hypothesis.given( |
| 615 |
- strategies.binary(max_size=0x00FFFFFF).map( |
|
| 612 |
+ encoded=strategies.binary(max_size=0x00FFFFFF).map( |
|
| 616 | 613 |
# Scoping issues, and the fact that staticmethod objects |
| 617 | 614 |
# (before class finalization) are not callable, necessitate |
| 618 | 615 |
# wrapping this staticmethod call in a lambda. |
| ... | ... |
@@ -70,9 +70,7 @@ def js_nested_strategy(draw: strategies.DrawFn) -> Any: |
| 70 | 70 |
|
| 71 | 71 |
|
| 72 | 72 |
@hypothesis.given(value=js_nested_strategy()) |
| 73 |
-@hypothesis.example(float("nan")).via(
|
|
| 74 |
- "for branch coverage in the implementation" |
|
| 75 |
-) |
|
| 73 |
+@hypothesis.example(value=float("nan")).via("branch coverage (implementation)")
|
|
| 76 | 74 |
def test_js_truthiness(value: Any) -> None: |
| 77 | 75 |
"""Determine the truthiness of a value according to JavaScript. |
| 78 | 76 |
|
| ... | ... |
@@ -303,8 +303,7 @@ class TestPhraseDependence: |
| 303 | 303 |
service=Strategies.text_strategy(), |
| 304 | 304 |
) |
| 305 | 305 |
@hypothesis.example(phrases=[b"\x00", b"\x00\x00"], service="0").xfail( |
| 306 |
- reason="phrases are interchangable", |
|
| 307 |
- raises=AssertionError, |
|
| 306 |
+ reason="phrases are interchangable", raises=AssertionError |
|
| 308 | 307 |
) |
| 309 | 308 |
def test_small(self, phrases: Sequence[bytes], service: str) -> None: |
| 310 | 309 |
"""The internal hash is dependent on the master passphrase. |
| ... | ... |
@@ -320,6 +319,10 @@ class TestPhraseDependence: |
| 320 | 319 |
), |
| 321 | 320 |
service=Strategies.text_strategy(), |
| 322 | 321 |
) |
| 322 |
+ @hypothesis.example( |
|
| 323 |
+ phrases=[b"\x01" * DIGEST_SIZE, b"\x01" * DIGEST_SIZE], |
|
| 324 |
+ service="service", |
|
| 325 |
+ ).xfail(reason="phrases are interchangable", raises=AssertionError) |
|
| 323 | 326 |
def test_medium(self, phrases: Sequence[bytes], service: str) -> None: |
| 324 | 327 |
"""The internal hash is dependent on the master passphrase. |
| 325 | 328 |
|
| ... | ... |
@@ -172,14 +172,14 @@ class TestDebugTranslations: |
| 172 | 172 |
assert not suffix or suffix.startswith("(")
|
| 173 | 173 |
|
| 174 | 174 |
@hypothesis.given(value=Strategies.printable_strings()) |
| 175 |
- @hypothesis.example("{")
|
|
| 175 |
+ @hypothesis.example(value="{")
|
|
| 176 | 176 |
def test_get_str(self, value: str) -> None: |
| 177 | 177 |
"""Translating a raw string object does nothing.""" |
| 178 | 178 |
translated = msg.translation.gettext(value) |
| 179 | 179 |
assert translated == value |
| 180 | 180 |
|
| 181 | 181 |
@hypothesis.given(value=Strategies.printable_strings()) |
| 182 |
- @hypothesis.example("{")
|
|
| 182 |
+ @hypothesis.example(value="{")
|
|
| 183 | 183 |
def test_get_ts_str(self, value: str) -> None: |
| 184 | 184 |
"""Translating a constant TranslatableString does nothing.""" |
| 185 | 185 |
translated = msg.TranslatedString.constant(value) |
| 186 | 186 |