Centralize settings for hypothesis deadline management
Marco Ricci commited on 2024-10-08 11:43:10
Zeige 3 geänderte Dateien mit 31 Einfügungen und 31 Löschungen.
Our unit tests run in multiple, very different environments, which leads to drastically different execution times, up to a slowdown factor of roughly 40 (test coverage, "timid" Python tracer). The `hypothesis` library however runs timing checks on each of its tests, indepedent of the available processing power and coverage instrumentation. As a result, some benign tests time out under these circumstances regardless. In the past, I've raised their execution deadline in an ad-hoc manner whenever this happens (or fixed the tests, if they weren't so benign). But instead of littering the test suite with one-time adjustments of deadlines, a more sensible approach is to use a test decorator that ensures a common extended deadline for tests that need it, only if they need it (i.e. run under coverage). So do that. (Sadly, because of how the settings decorator works, this must be applied function-wise, and cannot be stacked with other settings decorators.) Finally, if this deadline extension still doesn't help, then this usually means we are generating huge or expensive-to-evaluate inputs. So limit the size of some of the inputs (string length, recursion depth, size of passphrases to derive) to keep execution times better constrained.
- tests/__init__.py ceb12ba..f3c6742
- tests/test_derivepassphrase_types.py 4559fdd..977c3a7
- tests/test_derivepassphrase_vault.py aae6808..5f89ff7
| ... | ... |
@@ -13,6 +13,7 @@ import json |
| 13 | 13 |
import os |
| 14 | 14 |
import shlex |
| 15 | 15 |
import stat |
| 16 |
+import sys |
|
| 16 | 17 |
import tempfile |
| 17 | 18 |
import zipfile |
| 18 | 19 |
from typing import TYPE_CHECKING |
| ... | ... |
@@ -1079,6 +1080,22 @@ skip_if_no_cryptography_support = pytest.mark.skipif( |
| 1079 | 1080 |
reason='no "cryptography" support', |
| 1080 | 1081 |
) |
| 1081 | 1082 |
|
| 1083 |
+hypothesis_settings_coverage_compatible = ( |
|
| 1084 |
+ hypothesis.settings( |
|
| 1085 |
+ # Running under coverage with the Python tracer increases |
|
| 1086 |
+ # running times 40-fold, on my machines. Sadly, not every |
|
| 1087 |
+ # Python version offers the C tracer, so sometimes the Python |
|
| 1088 |
+ # tracer is used anyway. |
|
| 1089 |
+ deadline=( |
|
| 1090 |
+ 40 * deadline |
|
| 1091 |
+ if (deadline := hypothesis.settings().deadline) is not None |
|
| 1092 |
+ else None |
|
| 1093 |
+ ), |
|
| 1094 |
+ ) |
|
| 1095 |
+ if sys.gettrace() is not None |
|
| 1096 |
+ else hypothesis.settings() |
|
| 1097 |
+) |
|
| 1098 |
+ |
|
| 1082 | 1099 |
|
| 1083 | 1100 |
def list_keys(self: Any = None) -> list[_types.KeyCommentPair]: |
| 1084 | 1101 |
del self # Unused. |
| ... | ... |
@@ -15,6 +15,7 @@ from hypothesis import strategies |
| 15 | 15 |
from typing_extensions import TypeAlias, TypeVar |
| 16 | 16 |
|
| 17 | 17 |
import derivepassphrase |
| 18 |
+import tests |
|
| 18 | 19 |
|
| 19 | 20 |
if TYPE_CHECKING: |
| 20 | 21 |
from collections.abc import Iterator |
| ... | ... |
@@ -297,20 +298,8 @@ def vault_config(draw: strategies.DrawFn) -> dict[str, int]: |
| 297 | 298 |
} |
| 298 | 299 |
|
| 299 | 300 |
|
| 300 |
-# TODO(@the-13th-letter): Since all tests in this class manipulate the |
|
| 301 |
-# hypothesis deadline setting, perhaps it is more sensible to move this |
|
| 302 |
-# manipulation into a separate decorator, or a fixture. |
|
| 303 | 301 |
class TestHypotheses: |
| 304 |
- # This test tends to time out when using coverage without the |
|
| 305 |
- # C tracer, which in my testing leads to a roughly 40-fold execution |
|
| 306 |
- # time. So reset the deadline accordingly. |
|
| 307 |
- @hypothesis.settings( |
|
| 308 |
- deadline=( |
|
| 309 |
- 40 * deadline # type: ignore[name-defined] |
|
| 310 |
- if (deadline := hypothesis.settings().deadline) is not None |
|
| 311 |
- else None |
|
| 312 |
- ) |
|
| 313 |
- ) |
|
| 302 |
+ @tests.hypothesis_settings_coverage_compatible |
|
| 314 | 303 |
@hypothesis.given( |
| 315 | 304 |
phrase=strategies.one_of( |
| 316 | 305 |
strategies.binary(min_size=1), strategies.text(min_size=1) |
| ... | ... |
@@ -408,26 +397,18 @@ class TestHypotheses: |
| 408 | 397 |
len(set(snippet)) > 1 |
| 409 | 398 |
), 'Password does not satisfy character repeat constraints.' |
| 410 | 399 |
|
| 411 |
- # This test tends to time out when using coverage without the |
|
| 412 |
- # C tracer, which in my testing leads to a roughly 40-fold execution |
|
| 413 |
- # time. So reset the deadline accordingly. |
|
| 414 |
- @hypothesis.settings( |
|
| 415 |
- deadline=( |
|
| 416 |
- 40 * deadline # type: ignore[name-defined] |
|
| 417 |
- if (deadline := hypothesis.settings().deadline) is not None |
|
| 418 |
- else None |
|
| 419 |
- ) |
|
| 420 |
- ) |
|
| 400 |
+ @tests.hypothesis_settings_coverage_compatible |
|
| 421 | 401 |
@hypothesis.given( |
| 422 | 402 |
phrase=strategies.one_of( |
| 423 |
- strategies.binary(min_size=1), |
|
| 403 |
+ strategies.binary(min_size=1, max_size=100), |
|
| 424 | 404 |
strategies.text( |
| 425 | 405 |
min_size=1, |
| 406 |
+ max_size=100, |
|
| 426 | 407 |
alphabet=strategies.characters(max_codepoint=255), |
| 427 | 408 |
), |
| 428 | 409 |
), |
| 429 |
- length=strategies.integers(min_value=1, max_value=1000), |
|
| 430 |
- service=strategies.text(min_size=1), |
|
| 410 |
+ length=strategies.integers(min_value=1, max_value=200), |
|
| 411 |
+ service=strategies.text(min_size=1, max_size=100), |
|
| 431 | 412 |
) |
| 432 | 413 |
def test_101_password_with_length( |
| 433 | 414 |
self, |
| ... | ... |
@@ -439,19 +420,20 @@ class TestHypotheses: |
| 439 | 420 |
assert len(password) == length |
| 440 | 421 |
|
| 441 | 422 |
# This test has time complexity `O(length * repeat)`, both of which |
| 442 |
- # are chosen by hypothesis. So disable the deadline. |
|
| 423 |
+ # are chosen by hypothesis and thus outside our control. |
|
| 443 | 424 |
@hypothesis.settings(deadline=None) |
| 444 | 425 |
@hypothesis.given( |
| 445 | 426 |
phrase=strategies.one_of( |
| 446 |
- strategies.binary(min_size=1), |
|
| 427 |
+ strategies.binary(min_size=1, max_size=100), |
|
| 447 | 428 |
strategies.text( |
| 448 | 429 |
min_size=1, |
| 430 |
+ max_size=100, |
|
| 449 | 431 |
alphabet=strategies.characters(max_codepoint=255), |
| 450 | 432 |
), |
| 451 | 433 |
), |
| 452 |
- length=strategies.integers(min_value=2, max_value=1000), |
|
| 453 |
- repeat=strategies.integers(min_value=1, max_value=1000), |
|
| 454 |
- service=strategies.text(min_size=1), |
|
| 434 |
+ length=strategies.integers(min_value=2, max_value=200), |
|
| 435 |
+ repeat=strategies.integers(min_value=1, max_value=200), |
|
| 436 |
+ service=strategies.text(min_size=1, max_size=1000), |
|
| 455 | 437 |
) |
| 456 | 438 |
def test_102_password_with_repeat( |
| 457 | 439 |
self, |
| 458 | 440 |