Ensure better test suite behavior for regression-only testing
Marco Ricci

Marco Ricci commited on 2026-06-14 22:46:36
Zeige 12 geänderte Dateien mit 207 Einfügungen und 130 Löschungen.


Tweak the hypothesis settings for determinism and speed when running the
test suite solely for the purpose of regression testing [1], in the new
hypothesis profile "regression": test generation will be derandomized,
stateful step counts and example counts will be set very low.  As all of
our slow-running tests are hypothesis-based, and all of our
hypothesis-based tests are slow because of the high example count, this
change alone immediately makes "regression-only" testing feasible for us
without having to skip hypothesis-based tests.  (In fact, the hypothesis
derandomization feature appears to draw motivation from the same source
[1] as we do.)

As a counterpoint, mark test cases that are (or probably should be)
hypothesis-based, but test for completion or correctness purposes rather
than for regression purposes, with a custom marker
`correctness_focused`.

Also try to add at least one working example and one counterexample to
each hypothesis-based test, if feasible.  This buys us the flexibility
(in the future) to remove test case generation from the "regression"
profile, if desired.  The test cases would then still run the explicit
examples, and thus remain useful, instead of needing another separate
set of non-hypothesis tests to cover the same ground.

References:

 1. Nelson Elhage: [Two kinds of testing][], particularly the section
    "Test budget".
 2. Alex Kladov: [How to Test][], particularly the section "Make Tests
    Fast".

[Two kinds of testing]: https://blog.nelhage.com/post/two-kinds-of-testing/
[How to Test]: https://matklad.github.io/2021/05/31/how-to-test.html#Make-Tests-Fast
... ...
@@ -31,16 +31,27 @@ startup_ssh_auth_sock = os.environ.get("SSH_AUTH_SOCK", None)
31 31
 
32 32
 
33 33
 def pytest_configure(config: pytest.Config) -> None:
34
-    """Configure `pytest`: add the `heavy_duty` marker."""
34
+    """Configure `pytest`: add custom markers."""
35 35
     config.addinivalue_line(
36 36
         "markers",
37 37
         (
38 38
             "heavy_duty: "
39
-            "mark test as a slow, heavy-duty test (e.g., an integration test)"
40
-            "\n"
39
+            "mark test as a slow, heavy-duty test"
40
+        )
41
+    )
42
+    config.addinivalue_line(
43
+        "markers",
44
+        (
45
+            "correctness_focused: "
46
+            "mark test as a potentially slow, correctness-focused test"
47
+        )
48
+    )
49
+    config.addinivalue_line(
50
+        "markers",
51
+        (
41 52
             "non_reentrant_agent: "
42 53
             "mark the agent (fixture output) as non-reentrant"
43
-        ),
54
+        )
44 55
     )
45 56
     hypothesis_machinery._hypothesis_settings_setup()
46 57
 
... ...
@@ -17,7 +17,6 @@ All similar-minded code requiring only plain `pytest` lives in [the
17 17
 from __future__ import annotations
18 18
 
19 19
 import copy
20
-import datetime
21 20
 import importlib
22 21
 import importlib.util
23 22
 import math
... ...
@@ -94,36 +93,45 @@ def _hypothesis_settings_setup() -> None:
94 93
             and isinstance(trace_func.__self__, pytracer_class)
95 94
         ):
96 95
             slowdown = 8.0
97
-    settings = hypothesis.settings(
96
+    hypothesis.settings.register_profile(
97
+        "default",
98
+        parent=settings,
98 99
         deadline=slowdown * settings.deadline
99
-        if slowdown
100
+        if slowdown and settings.deadline is not None
100 101
         else settings.deadline,
101 102
         stateful_step_count=32,
102
-        suppress_health_check=(hypothesis.HealthCheck.too_slow,),
103
+        suppress_health_check=[hypothesis.HealthCheck.too_slow],
103 104
     )
104
-    hypothesis.settings.register_profile("default", settings)
105
+    default_profile = hypothesis.settings.get_profile("default")
105 106
     hypothesis.settings.register_profile(
106
-        "dev", derandomize=True, max_examples=10
107
+        "regression",
108
+        parent=hypothesis.settings.get_profile("ci"),
109
+        derandomize=True,
110
+        database=None,
111
+        max_examples=3,
112
+        stateful_step_count=3,
113
+        suppress_health_check=[hypothesis.HealthCheck.too_slow],
114
+        phases=[
115
+            hypothesis.Phase.explicit,
116
+            hypothesis.Phase.reuse,
117
+            hypothesis.Phase.generate,
118
+        ],
107 119
     )
108 120
     hypothesis.settings.register_profile(
109 121
         "debug",
110
-        parent=hypothesis.settings.get_profile("dev"),
122
+        parent=default_profile,
111 123
         verbosity=hypothesis.Verbosity.verbose,
112 124
     )
113 125
     hypothesis.settings.register_profile(
114
-        "flaky",
115
-        deadline=(
116
-            settings.deadline - settings.deadline // 4
117
-            if settings.deadline is not None
118
-            else datetime.timedelta(milliseconds=150)
119
-        ),
126
+        "debug-regression",
127
+        parent=hypothesis.settings.get_profile("regression"),
128
+        verbosity=hypothesis.Verbosity.verbose,
120 129
     )
121
-    ci_profile = hypothesis.settings.get_profile("ci")
122 130
     hypothesis.settings.register_profile(
123
-        "intense",
124
-        parent=ci_profile,
131
+        "correctness",
132
+        parent=default_profile,
125 133
         derandomize=False,
126
-        max_examples=10 * ci_profile.max_examples,
134
+        max_examples=10 * default_profile.max_examples,
127 135
     )
128 136
 
129 137
 
... ...
@@ -161,13 +169,13 @@ def get_process_spawning_state_machine_examples_count(
161 169
     examples count by default, and require the user to opt-in to the
162 170
     original naive example count explicitly.
163 171
 
164
-    If the "intense" profile is in effect, or something with even higher
165
-    `max_examples` and `stateful_step_count`, then we return the
172
+    If the "correctness" profile is in effect, or something with even
173
+    higher `max_examples` and `stateful_step_count`, then we return the
166 174
     unaltered example count for the *default* profile.  Otherwise, we
167 175
     return the square root of the `max_examples` setting (rounded down).
168
-    We *never* return a value below the "dev" profile's example count:
169
-    any lower computed example count is increased to the "dev" profile's
170
-    example count.
176
+    We *never* return a value below the "regression" profile's example
177
+    count: any lower computed example count is increased to the
178
+    "regression" profile's example count.
171 179
 
172 180
     Args:
173 181
         settings:
... ...
@@ -182,20 +190,22 @@ def get_process_spawning_state_machine_examples_count(
182 190
     _hypothesis_settings_setup()
183 191
 
184 192
     these_values = (settings.max_examples, settings.stateful_step_count)
185
-    intense_profile = hypothesis.settings.get_profile("intense")
186
-    intense_values = (
187
-        intense_profile.max_examples,
188
-        intense_profile.stateful_step_count,
193
+    correctness_profile = hypothesis.settings.get_profile("correctness")
194
+    correctness_values = (
195
+        correctness_profile.max_examples,
196
+        correctness_profile.stateful_step_count,
189 197
     )
190
-    min_count = hypothesis.settings.get_profile("dev").max_examples
198
+    min_count = hypothesis.settings.get_profile("regression").max_examples
191 199
     high_count = hypothesis.settings.get_profile("default").max_examples
192
-    we_are_intense = (
193
-        these_values[0] >= intense_values[0]
194
-        and these_values[1] >= intense_values[1]
200
+    correctness_based_testing = (
201
+        these_values[0] >= correctness_values[0]
202
+        and these_values[1] >= correctness_values[1]
195 203
     )
196 204
     return max(
197 205
         min_count,
198
-        high_count if we_are_intense else math.isqrt(settings.max_examples),
206
+        high_count
207
+        if correctness_based_testing
208
+        else math.isqrt(settings.max_examples),
199 209
     )
200 210
 
201 211
 
... ...
@@ -143,19 +143,30 @@ def xfail_on_the_annoying_os(
143 143
 
144 144
 heavy_duty = pytest.mark.heavy_duty
145 145
 """
146
-A cached `pytest` mark indicating that this test function/class/module
147
-is a slow, heavy duty test.  Users who are impatient (or otherwise
148
-cannot afford to wait for these tests to complete) may wish to exclude
149
-these tests; this mark helps in achieving that.
146
+Test functions/classes/modules with this `pytest` mark are slow, heavy
147
+duty tests.  Users who are impatient (or otherwise cannot afford to wait
148
+for these tests to complete) may wish to exclude these tests; this mark
149
+helps in achieving that.
150
+"""
150 151
 
151
-All current heavy duty tests are integration tests.
152
+correctness_focused = pytest.mark.correctness_focused
152 153
 """
154
+Test functions/classes/modules with this `pytest` mark are designed
155
+primarily to verify correctness and to find novel failure cases, even at
156
+the cost of high CPU or wall clock time.
157
+
158
+Contrast this with tests designed primarily to verify that the existing
159
+system still behaves as before in all the important ways ("is healthy")
160
+even after introducing a small change, but before committing it.  (See
161
+[Two kinds of testing][TWO_KINDS] for more elaboration.)
153 162
 
163
+[TWO_KINDS]: https://blog.nelhage.com/post/two-kinds-of-testing/
164
+"""
154 165
 
155 166
 non_isolated_agent_use = pytest.mark.xdist_group("non_isolated_agent_use")
156 167
 """
157
-A cached `pytest` mark, for use with xdist's "loadgroup" scheduling,
158
-that marks a test as part of the group of tests that use a potentially
168
+Tests with this `pytest` mark (for use with xdist's "loadgroup"
169
+scheduling), comprise the group of tests that use a potentially
159 170
 non-isolated agent, and thus need to be run serially, in the same worker
160 171
 (because of the setup and teardown).
161 172
 
... ...
@@ -1245,7 +1245,9 @@ class TestTempdir:
1245 1245
             ),
1246 1246
         ),
1247 1247
     )
1248
-    @hypothesis.example(env_var="", suffix=".")
1248
+    @hypothesis.example(env_var="", suffix=".").via(
1249
+        "static example; branch coverage (test)"
1250
+    )
1249 1251
     def test_get_tempdir(
1250 1252
         self,
1251 1253
         env_var: str,
... ...
@@ -141,6 +141,9 @@ class TestNotesPrinting:
141 141
 
142 142
     @Parametrize.NOTES_PLACEMENT
143 143
     @hypothesis.given(notes=Strategies.notes().filter(str.strip))
144
+    @hypothesis.example(notes="  ").xfail(
145
+        reason="empty notes", raises=AssertionError
146
+    )
144 147
     def test_notes_placement(
145 148
         self,
146 149
         notes_placement: Literal["before", "after"],
... ...
@@ -347,11 +350,15 @@ class TestNotesEditingValid(TestNotesEditing):
347 350
         ],
348 351
     )
349 352
     @hypothesis.given(
350
-        notes=Strategies.notes()
353
+        notes=Strategies
354
+        .notes()
351 355
         .filter(str.strip)
352 356
         .filter(lambda notes: notes != TestNotesEditingValid.CURRENT_NOTES)
353 357
     )
354 358
     @hypothesis.example(TestNotesEditing.CURRENT_NOTES)
359
+    @hypothesis.example(notes="  ").xfail(
360
+        reason="empty notes", raises=AssertionError
361
+    )
355 362
     def test_successful_edit(
356 363
         self,
357 364
         caplog: pytest.LogCaptureFixture,
... ...
@@ -463,6 +470,9 @@ class TestNotesEditingValid(TestNotesEditing):
463 470
         ],
464 471
     )
465 472
     @hypothesis.given(notes=Strategies.notes().filter(str.strip))
473
+    @hypothesis.example(notes="  ").xfail(
474
+        reason="empty notes", raises=AssertionError
475
+    )
466 476
     def test_marker_removed(
467 477
         self,
468 478
         caplog: pytest.LogCaptureFixture,
... ...
@@ -509,7 +519,7 @@ class TestNotesEditingInvalid(TestNotesEditing):
509 519
     """Tests concerning editing service notes: invalid/error calls."""
510 520
 
511 521
     @hypothesis.given(notes=Strategies.notes())
512
-    @hypothesis.example("")
522
+    @hypothesis.example(notes="")
513 523
     def test_abort(
514 524
         self,
515 525
         notes: str,
... ...
@@ -548,7 +558,7 @@ class TestNotesEditingInvalid(TestNotesEditing):
548 558
         ],
549 559
     )
550 560
     @hypothesis.given(notes=Strategies.notes())
551
-    @hypothesis.example("")
561
+    @hypothesis.example(notes="")
552 562
     def test_fail_on_config_option_missing(
553 563
         self,
554 564
         caplog: pytest.LogCaptureFixture,
... ...
@@ -391,6 +391,7 @@ class TestStoreroom:
391 391
                 == data.VAULT_STOREROOM_CONFIG_DATA
392 392
             )
393 393
 
394
+    @pytest_machinery.correctness_focused
394 395
     def test_decrypt_bucket_item_unknown_version(self) -> None:
395 396
         """Fail on unknown versions of the master keys file."""
396 397
         bucket_item = (
... ...
@@ -404,6 +405,7 @@ class TestStoreroom:
404 405
         with pytest.raises(ValueError, match="Cannot handle version 255"):
405 406
             storeroom._decrypt_bucket_item(bucket_item, master_keys)
406 407
 
408
+    @pytest_machinery.correctness_focused
407 409
     @Parametrize.BAD_CONFIG
408 410
     def test_decrypt_bucket_file_bad_json_or_version(
409 411
         self,
... ...
@@ -427,6 +429,7 @@ class TestStoreroom:
427 429
             with pytest.raises(ValueError, match="Invalid bucket file: "):
428 430
                 list(storeroom._decrypt_bucket_file(p, master_keys))
429 431
 
432
+    @pytest_machinery.correctness_focused
430 433
     @Parametrize.BAD_MASTER_KEYS_DATA
431 434
     @Parametrize.STOREROOM_HANDLER
432 435
     def test_export_storeroom_data_bad_master_keys_file(
... ...
@@ -447,6 +450,7 @@ class TestStoreroom:
447 450
             with pytest.raises(RuntimeError, match=err_msg):
448 451
                 handler(format="storeroom")
449 452
 
453
+    @pytest_machinery.correctness_focused
450 454
     @Parametrize.BAD_STOREROOM_CONFIG_DATA
451 455
     @Parametrize.STOREROOM_HANDLER
452 456
     def test_export_storeroom_data_bad_directory_listing(
... ...
@@ -476,6 +480,7 @@ class TestStoreroom:
476 480
             with pytest.raises(RuntimeError, match=error_text):
477 481
                 handler(format="storeroom")
478 482
 
483
+    @pytest_machinery.correctness_focused
479 484
     def test_decrypt_keys_wrong_data_length(self) -> None:
480 485
         """Fail on internal structural data of the wrong size.
481 486
 
... ...
@@ -527,6 +532,7 @@ class TestStoreroom:
527 532
                 ),
528 533
             )
529 534
 
535
+    @pytest_machinery.correctness_focused
530 536
     @hypothesis.given(
531 537
         data=strategies.binary(
532 538
             min_size=storeroom.MAC_SIZE, max_size=storeroom.MAC_SIZE
... ...
@@ -193,66 +193,86 @@ class TestCLIUtilities:
193 193
             )
194 194
             yield monkeypatch
195 195
 
196
+    @hypothesis.given(
197
+        vault_key_env=VaultKeyEnvironment.strategy().filter(
198
+            lambda env: bool(env.expected)
199
+        ),
200
+    )
201
+    # BEGIN pre-hypothesis parametrization examples
196 202
     @hypothesis.example(
197
-        VaultKeyEnvironment("4username", None, None, None, "4username")
198
-    ).via("manual, pre-hypothesis parametrization value")
203
+        vault_key_env=VaultKeyEnvironment(
204
+            "4username", None, None, None, "4username"
205
+        )
206
+    )
199 207
     @hypothesis.example(
200
-        VaultKeyEnvironment("3user", None, None, "3user", None)
201
-    ).via("manual, pre-hypothesis parametrization value")
208
+        vault_key_env=VaultKeyEnvironment("3user", None, None, "3user", None)
209
+    )
202 210
     @hypothesis.example(
203
-        VaultKeyEnvironment("3user", None, None, "3user", "4username")
204
-    ).via("manual, pre-hypothesis parametrization value")
211
+        vault_key_env=VaultKeyEnvironment(
212
+            "3user", None, None, "3user", "4username"
213
+        )
214
+    )
205 215
     @hypothesis.example(
206
-        VaultKeyEnvironment("2logname", None, "2logname", None, None)
207
-    ).via("manual, pre-hypothesis parametrization value")
216
+        vault_key_env=VaultKeyEnvironment(
217
+            "2logname", None, "2logname", None, None
218
+        )
219
+    )
208 220
     @hypothesis.example(
209
-        VaultKeyEnvironment("2logname", None, "2logname", None, "4username")
210
-    ).via("manual, pre-hypothesis parametrization value")
221
+        vault_key_env=VaultKeyEnvironment(
222
+            "2logname", None, "2logname", None, "4username"
223
+        )
224
+    )
211 225
     @hypothesis.example(
212
-        VaultKeyEnvironment("2logname", None, "2logname", "3user", None)
213
-    ).via("manual, pre-hypothesis parametrization value")
226
+        vault_key_env=VaultKeyEnvironment(
227
+            "2logname", None, "2logname", "3user", None
228
+        )
229
+    )
214 230
     @hypothesis.example(
215
-        VaultKeyEnvironment("2logname", None, "2logname", "3user", "4username")
216
-    ).via("manual, pre-hypothesis parametrization value")
231
+        vault_key_env=VaultKeyEnvironment(
232
+            "2logname", None, "2logname", "3user", "4username"
233
+        )
234
+    )
217 235
     @hypothesis.example(
218
-        VaultKeyEnvironment("1vault_key", "1vault_key", None, None, None)
219
-    ).via("manual, pre-hypothesis parametrization value")
236
+        vault_key_env=VaultKeyEnvironment(
237
+            "1vault_key", "1vault_key", None, None, None
238
+        )
239
+    )
220 240
     @hypothesis.example(
221
-        VaultKeyEnvironment(
241
+        vault_key_env=VaultKeyEnvironment(
222 242
             "1vault_key", "1vault_key", None, None, "4username"
223 243
         )
224
-    ).via("manual, pre-hypothesis parametrization value")
244
+    )
225 245
     @hypothesis.example(
226
-        VaultKeyEnvironment("1vault_key", "1vault_key", None, "3user", None)
227
-    ).via("manual, pre-hypothesis parametrization value")
246
+        vault_key_env=VaultKeyEnvironment(
247
+            "1vault_key", "1vault_key", None, "3user", None
248
+        )
249
+    )
228 250
     @hypothesis.example(
229
-        VaultKeyEnvironment(
251
+        vault_key_env=VaultKeyEnvironment(
230 252
             "1vault_key", "1vault_key", None, "3user", "4username"
231 253
         )
232
-    ).via("manual, pre-hypothesis parametrization value")
254
+    )
233 255
     @hypothesis.example(
234
-        VaultKeyEnvironment("1vault_key", "1vault_key", "2logname", None, None)
235
-    ).via("manual, pre-hypothesis parametrization value")
256
+        vault_key_env=VaultKeyEnvironment(
257
+            "1vault_key", "1vault_key", "2logname", None, None
258
+        )
259
+    )
236 260
     @hypothesis.example(
237
-        VaultKeyEnvironment(
261
+        vault_key_env=VaultKeyEnvironment(
238 262
             "1vault_key", "1vault_key", "2logname", None, "4username"
239 263
         )
240
-    ).via("manual, pre-hypothesis parametrization value")
264
+    )
241 265
     @hypothesis.example(
242
-        VaultKeyEnvironment(
266
+        vault_key_env=VaultKeyEnvironment(
243 267
             "1vault_key", "1vault_key", "2logname", "3user", None
244 268
         )
245
-    ).via("manual, pre-hypothesis parametrization value")
269
+    )
246 270
     @hypothesis.example(
247
-        VaultKeyEnvironment(
271
+        vault_key_env=VaultKeyEnvironment(
248 272
             "1vault_key", "1vault_key", "2logname", "3user", "4username"
249 273
         )
250
-    ).via("manual, pre-hypothesis parametrization value")
251
-    @hypothesis.given(
252
-        vault_key_env=VaultKeyEnvironment.strategy().filter(
253
-            lambda env: bool(env.expected)
254
-        ),
255 274
     )
275
+    # END pre-hypothesis parametrization examples
256 276
     def test_get_vault_key(
257 277
         self,
258 278
         vault_key_env: VaultKeyEnvironment,
... ...
@@ -164,21 +164,24 @@ class TestStaticFunctionality:
164 164
             )
165 165
 
166 166
     @hypothesis.given(test_case=BigEndianNumberTest.strategy())
167
+    # decimal example
167 168
     @hypothesis.example(
168
-        BigEndianNumberTest([1, 2, 3, 4, 5, 6], 10, 123456)
169
-    ).via("manual decimal example")
169
+        test_case=BigEndianNumberTest([1, 2, 3, 4, 5, 6], 10, 123456)
170
+    )
171
+    # decimal example, different base
172
+    @hypothesis.example(
173
+        test_case=BigEndianNumberTest([1, 2, 3, 4, 5, 6], 100, 10203040506)
174
+    )
175
+    # leading zeroes example
170 176
     @hypothesis.example(
171
-        BigEndianNumberTest([1, 2, 3, 4, 5, 6], 100, 10203040506)
172
-    ).via("manual decimal example in different base")
173
-    @hypothesis.example(BigEndianNumberTest([0, 0, 1, 4, 9, 7], 10, 1497)).via(
174
-        "manual example with leading zeroes"
177
+        test_case=BigEndianNumberTest([0, 0, 1, 4, 9, 7], 10, 1497)
175 178
     )
179
+    # binary example
176 180
     @hypothesis.example(
177
-        BigEndianNumberTest([1, 0, 0, 1, 0, 0, 0, 0], 2, 144)
178
-    ).via("manual binary example")
179
-    @hypothesis.example(BigEndianNumberTest([1, 7, 5, 5], 8, 0o1755)).via(
180
-        "manual octal example"
181
+        test_case=BigEndianNumberTest([1, 0, 0, 1, 0, 0, 0, 0], 2, 144)
181 182
     )
183
+    # octal example
184
+    @hypothesis.example(test_case=BigEndianNumberTest([1, 7, 5, 5], 8, 0o1755))
182 185
     def test_big_endian_number(self, test_case: BigEndianNumberTest) -> None:
183 186
         """Conversion to big endian numbers in any base works.
184 187
 
... ...
@@ -294,22 +297,28 @@ class TestSequin:
294 297
             )
295 298
 
296 299
     @hypothesis.given(test_case=ConstructorTestCase.strategy())
300
+    # bitstring example
297 301
     @hypothesis.example(
298
-        ConstructorTestCase([1, 0, 0, 1, 0, 1], True, [1, 0, 0, 1, 0, 1])
299
-    ).via("manual example bitstring")
302
+        test_case=ConstructorTestCase(
303
+            [1, 0, 0, 1, 0, 1], True, [1, 0, 0, 1, 0, 1]
304
+        )
305
+    )
306
+    # bitstring as bytestring example
300 307
     @hypothesis.example(
301
-        ConstructorTestCase(
308
+        test_case=ConstructorTestCase(
302 309
             [1, 0, 0, 1, 0, 1],
303 310
             False,
304 311
             bitseq("000000010000000000000000000000010000000000000001"),
305 312
         )
306
-    ).via("manual example bitstring as byte string")
313
+    )
314
+    # true byte string example
307 315
     @hypothesis.example(
308
-        ConstructorTestCase(b"OK", False, bitseq("0100111101001011"))
309
-    ).via("manual example true byte string")
316
+        test_case=ConstructorTestCase(b"OK", False, bitseq("0100111101001011"))
317
+    )
318
+    # latin1 text example
310 319
     @hypothesis.example(
311
-        ConstructorTestCase("OK", False, bitseq("0100111101001011"))
312
-    ).via("manual example latin1 text")
320
+        test_case=ConstructorTestCase("OK", False, bitseq("0100111101001011"))
321
+    )
313 322
     def test_constructor(
314 323
         self,
315 324
         test_case: ConstructorTestCase,
... ...
@@ -355,8 +364,9 @@ class TestSequin:
355 364
                 ],
356 365
             )
357 366
 
367
+    @hypothesis.given(sequence=GenerationSequence.strategy())
358 368
     @hypothesis.example(
359
-        GenerationSequence(
369
+        sequence=GenerationSequence(
360 370
             bitseq("110101011111001"),
361 371
             [
362 372
                 (1, 0),
... ...
@@ -367,8 +377,7 @@ class TestSequin:
367 377
                 (1, sequin.SequinExhaustedError),
368 378
             ],
369 379
         )
370
-    ).via("manual, pre-hypothesis parametrization value")
371
-    @hypothesis.given(sequence=GenerationSequence.strategy())
380
+    )
372 381
     def test_generating(self, sequence: GenerationSequence) -> None:
373 382
         """The sequin generates deterministic sequences."""
374 383
         seq = sequin.Sequin(sequence.bit_sequence, is_bitstring=True)
... ...
@@ -396,8 +405,9 @@ class TestSequin:
396 405
         with pytest.raises(ValueError, match="invalid target range"):
397 406
             seq.generate(0)
398 407
 
408
+    @hypothesis.given(sequence=GenerationSequence.strategy())
399 409
     @hypothesis.example(
400
-        GenerationSequence(
410
+        sequence=GenerationSequence(
401 411
             bitseq("110101011111001"),
402 412
             [
403 413
                 (1, 0),
... ...
@@ -408,8 +418,7 @@ class TestSequin:
408 418
                 (1, sequin.SequinExhaustedError),
409 419
             ],
410 420
         )
411
-    ).via("manual, pre-hypothesis parametrization value")
412
-    @hypothesis.given(sequence=GenerationSequence.strategy())
421
+    )
413 422
     def test_internal_generating(self, sequence: GenerationSequence) -> None:
414 423
         """The sequin internals generate deterministic sequences."""
415 424
         seq = sequin.Sequin(sequence.bit_sequence, is_bitstring=True)
... ...
@@ -495,7 +504,7 @@ class TestSequin:
495 504
 
496 505
     @hypothesis.given(sequence=ShiftSequence.strategy())
497 506
     @hypothesis.example(
498
-        ShiftSequence(
507
+        sequence=ShiftSequence(
499 508
             bitseq("1010010001"),
500 509
             [
501 510
                 (3, bitseq("101"), bitseq("0010001")),
... ...
@@ -533,17 +533,17 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality):
533 533
         uint32 = ssh_agent.SSHAgentClient.uint32
534 534
         assert uint32(input) == expected
535 535
 
536
-    @hypothesis.given(strategies.integers(min_value=0, max_value=0xFFFFFFFF))
537
-    @hypothesis.example(0xDEADBEEF).via("manual, pre-hypothesis example")
536
+    @hypothesis.given(
537
+        num=strategies.integers(min_value=0, max_value=0xFFFFFFFF)
538
+    )
539
+    @hypothesis.example(num=0xDEADBEEF)
538 540
     def test_uint32_from_number(self, num: int) -> None:
539 541
         """`uint32` encoding works, starting from numbers."""
540 542
         uint32 = ssh_agent.SSHAgentClient.uint32
541 543
         assert int.from_bytes(uint32(num), "big", signed=False) == num
542 544
 
543
-    @hypothesis.given(strategies.binary(min_size=4, max_size=4))
544
-    @hypothesis.example(b"\xde\xad\xbe\xef").via(
545
-        "manual, pre-hypothesis example"
546
-    )
545
+    @hypothesis.given(bytestring=strategies.binary(min_size=4, max_size=4))
546
+    @hypothesis.example(bytestring=b"\xde\xad\xbe\xef")
547 547
     def test_uint32_from_bytestring(self, bytestring: bytes) -> None:
548 548
         """`uint32` encoding works, starting from length four byte strings."""
549 549
         uint32 = ssh_agent.SSHAgentClient.uint32
... ...
@@ -560,10 +560,9 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality):
560 560
         string = ssh_agent.SSHAgentClient.string
561 561
         assert bytes(string(input)) == expected
562 562
 
563
-    @hypothesis.given(strategies.binary(max_size=0x0001FFFF))
564
-    @hypothesis.example(b"DEADBEEF" * 10000).via(
565
-        "manual, pre-hypothesis example with highest order bit set"
566
-    )
563
+    @hypothesis.given(bytestring=strategies.binary(max_size=0x0001FFFF))
564
+    # example with highest order bit set
565
+    @hypothesis.example(bytestring=b"DEADBEEF" * 10000)
567 566
     def test_string_from_bytestring(self, bytestring: bytes) -> None:
568 567
         """SSH string encoding works, starting from a byte string."""
569 568
         res = ssh_agent.SSHAgentClient.string(bytestring)
... ...
@@ -584,13 +583,11 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality):
584 583
             b"",
585 584
         )
586 585
 
587
-    @hypothesis.given(strategies.binary(max_size=0x00FFFFFF))
588
-    @hypothesis.example(b"\x00\x00\x00\x07ssh-rsa").via(
589
-        "manual, pre-hypothesis example to attempt to detect double-decoding"
590
-    )
591
-    @hypothesis.example(b"\x00\x00\x00\x01").via(
592
-        "detect no-op encoding via ill-formed SSH string"
593
-    )
586
+    @hypothesis.given(bytestring=strategies.binary(max_size=0x00FFFFFF))
587
+    # should not double-decode
588
+    @hypothesis.example(bytestring=b"\x00\x00\x00\x07ssh-rsa")
589
+    # should not choke on ill-formed SSH string payloads
590
+    @hypothesis.example(bytestring=b"\x00\x00\x00\x01")
594 591
     def test_unstring_of_string_of_data(self, bytestring: bytes) -> None:
595 592
         """SSH string decoding of encoded SSH strings works.
596 593
 
... ...
@@ -612,7 +609,7 @@ class TestSSHProtocolDatatypes(TestStaticFunctionality):
612 609
         assert unstring_prefix(encoded2) == (bytestring, trailing_data)
613 610
 
614 611
     @hypothesis.given(
615
-        strategies.binary(max_size=0x00FFFFFF).map(
612
+        encoded=strategies.binary(max_size=0x00FFFFFF).map(
616 613
             # Scoping issues, and the fact that staticmethod objects
617 614
             # (before class finalization) are not callable, necessitate
618 615
             # wrapping this staticmethod call in a lambda.
... ...
@@ -70,9 +70,7 @@ def js_nested_strategy(draw: strategies.DrawFn) -> Any:
70 70
 
71 71
 
72 72
 @hypothesis.given(value=js_nested_strategy())
73
-@hypothesis.example(float("nan")).via(
74
-    "for branch coverage in the implementation"
75
-)
73
+@hypothesis.example(value=float("nan")).via("branch coverage (implementation)")
76 74
 def test_js_truthiness(value: Any) -> None:
77 75
     """Determine the truthiness of a value according to JavaScript.
78 76
 
... ...
@@ -303,8 +303,7 @@ class TestPhraseDependence:
303 303
         service=Strategies.text_strategy(),
304 304
     )
305 305
     @hypothesis.example(phrases=[b"\x00", b"\x00\x00"], service="0").xfail(
306
-        reason="phrases are interchangable",
307
-        raises=AssertionError,
306
+        reason="phrases are interchangable", raises=AssertionError
308 307
     )
309 308
     def test_small(self, phrases: Sequence[bytes], service: str) -> None:
310 309
         """The internal hash is dependent on the master passphrase.
... ...
@@ -320,6 +319,10 @@ class TestPhraseDependence:
320 319
         ),
321 320
         service=Strategies.text_strategy(),
322 321
     )
322
+    @hypothesis.example(
323
+        phrases=[b"\x01" * DIGEST_SIZE, b"\x01" * DIGEST_SIZE],
324
+        service="service",
325
+    ).xfail(reason="phrases are interchangable", raises=AssertionError)
323 326
     def test_medium(self, phrases: Sequence[bytes], service: str) -> None:
324 327
         """The internal hash is dependent on the master passphrase.
325 328
 
... ...
@@ -172,14 +172,14 @@ class TestDebugTranslations:
172 172
         assert not suffix or suffix.startswith("(")
173 173
 
174 174
     @hypothesis.given(value=Strategies.printable_strings())
175
-    @hypothesis.example("{")
175
+    @hypothesis.example(value="{")
176 176
     def test_get_str(self, value: str) -> None:
177 177
         """Translating a raw string object does nothing."""
178 178
         translated = msg.translation.gettext(value)
179 179
         assert translated == value
180 180
 
181 181
     @hypothesis.given(value=Strategies.printable_strings())
182
-    @hypothesis.example("{")
182
+    @hypothesis.example(value="{")
183 183
     def test_get_ts_str(self, value: str) -> None:
184 184
         """Translating a constant TranslatableString does nothing."""
185 185
         translated = msg.TranslatedString.constant(value)
186 186