From 21e4342c73fd02be6458b9b2353e9f63fa1824a0 Mon Sep 17 00:00:00 2001 From: "Nor.na" <2573438329@qq.com> Date: Fri, 2 Oct 2026 22:39:42 +0800 Subject: [PATCH] Preserve externally compiled regex flags during backend selection --- README-dist.rst | 7 + README.rst | 7 + pathspec/_backends/_utils.py | 42 +++- pathspec/_backends/agg.py | 6 + pathspec/_backends/hyperscan/pathspec.py | 6 +- pathspec/_backends/re2/pathspec.py | 6 +- pathspec/pathspec.py | 12 +- tests/test_07_regex_flags.py | 235 +++++++++++++++++++++++ 8 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 tests/test_07_regex_flags.py diff --git a/README-dist.rst b/README-dist.rst index 7dcca83..72b067e 100644 --- a/README-dist.rst +++ b/README-dist.rst @@ -116,6 +116,13 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the be significantly faster than "simple", and 3 times faster than "hyperscan" at high pattern counts. +When an active ``RegexPattern`` wraps a standard-library ``re.Pattern`` with +compile flags that are not encoded in its expression (for example, ``re.compile('secret', +re.IGNORECASE)``), "best" uses "simple" to preserve those flags. Explicitly +selecting "re2" or "hyperscan" raises ``ValueError`` for these patterns; use +``backend='simple'`` instead. Inline flags remain subject to the selected +backend's regular expression syntax and semantics. + See `benchmarks_backends.md`_ for comparisons between native Python regular expressions and the optional backends. diff --git a/README.rst b/README.rst index 9824fb9..330a071 100644 --- a/README.rst +++ b/README.rst @@ -116,6 +116,13 @@ The "re2" backend uses the `google-re2`_ library (not to be confused with the be significantly faster than "simple", and 3 times faster than "hyperscan" at high pattern counts. +When an active ``RegexPattern`` wraps a standard-library ``re.Pattern`` with +compile flags that are not encoded in its expression (for example, ``re.compile('secret', +re.IGNORECASE)``), "best" uses "simple" to preserve those flags. Explicitly +selecting "re2" or "hyperscan" raises ``ValueError`` for these patterns; use +``backend='simple'`` instead. Inline flags remain subject to the selected +backend's regular expression syntax and semantics. + See `benchmarks_backends.md`_ for comparisons between native Python regular expressions and the optional backends. diff --git a/pathspec/_backends/_utils.py b/pathspec/_backends/_utils.py index 77c7cd9..46e7fa8 100644 --- a/pathspec/_backends/_utils.py +++ b/pathspec/_backends/_utils.py @@ -5,17 +5,57 @@ contents and structure are likely to change. """ +import re +import warnings from collections.abc import ( Iterable) from typing import ( TypeVar) from pathspec.pattern import ( - Pattern) + Pattern, + RegexPattern) TPattern = TypeVar("TPattern", bound=Pattern) +def has_regex_flags(patterns: Iterable[Pattern]) -> bool: + """ + Check for active Python regexes with flags not encoded in the expression. + + Native backends recompile the expression, losing external compile flags. + Inline flags are already part of the expression and need no translation. + """ + for pattern in patterns: + if pattern.include is None or not isinstance(pattern, RegexPattern): + continue + + regex = pattern.regex + if not isinstance(regex, re.Pattern): + continue + + # UNICODE is the default for str regexes. Avoid recompiling ordinary + # patterns (including the built-in gitignore patterns). + if not regex.flags & ~int(re.UNICODE): + continue + + try: + # This expression is only a probe, not the regex used for matching. + # Do not expose warnings from, e.g., VERBOSE comment contents. + with warnings.catch_warnings(): + warnings.simplefilter('ignore', FutureWarning) + inline_flags = re.compile(regex.pattern).flags + except re.error: + # The expression may only be valid with external flags, e.g., a + # VERBOSE comment containing an unmatched bracket. + return True + + if regex.flags != inline_flags: + return True + + return False + + def enumerate_patterns( patterns: Iterable[TPattern], filter: bool, diff --git a/pathspec/_backends/agg.py b/pathspec/_backends/agg.py index c387146..5401db9 100644 --- a/pathspec/_backends/agg.py +++ b/pathspec/_backends/agg.py @@ -18,6 +18,8 @@ Pattern, RegexPattern) +from ._utils import ( + has_regex_flags) from .hyperscan.base import ( hyperscan_error) from .hyperscan.gitignore import ( @@ -65,6 +67,8 @@ def make_gitignore_backend( """ if name == 'best': name = _BEST_BACKEND + if name != 'simple' and has_regex_flags(patterns): + name = 'simple' if name == 'hyperscan': return HyperscanGiBackend(cast(Sequence[RegexPattern], patterns)) @@ -93,6 +97,8 @@ def make_pathspec_backend( """ if name == 'best': name = _BEST_BACKEND + if name != 'simple' and has_regex_flags(patterns): + name = 'simple' if name == 'hyperscan': return HyperscanPsBackend(cast(Sequence[RegexPattern], patterns)) diff --git a/pathspec/_backends/hyperscan/pathspec.py b/pathspec/_backends/hyperscan/pathspec.py index f7a8ddc..8e64670 100644 --- a/pathspec/_backends/hyperscan/pathspec.py +++ b/pathspec/_backends/hyperscan/pathspec.py @@ -26,7 +26,8 @@ override) # Added in 3.12. from .._utils import ( - enumerate_patterns) + enumerate_patterns, + has_regex_flags) from .base import ( hyperscan_error) @@ -63,6 +64,9 @@ def __init__( if patterns and not isinstance(patterns[0], RegexPattern): raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.") + if has_regex_flags(patterns): + raise ValueError("The hyperscan backend cannot preserve regex compile flags. Use backend='simple'.") + use_patterns = enumerate_patterns( patterns, filter=True, reverse=False, ) diff --git a/pathspec/_backends/re2/pathspec.py b/pathspec/_backends/re2/pathspec.py index ca994b5..070aa41 100644 --- a/pathspec/_backends/re2/pathspec.py +++ b/pathspec/_backends/re2/pathspec.py @@ -25,7 +25,8 @@ override) # Added in 3.12. from .._utils import ( - enumerate_patterns) + enumerate_patterns, + has_regex_flags) from .base import ( re2_error) @@ -60,6 +61,9 @@ def __init__( if patterns and not isinstance(patterns[0], RegexPattern): raise TypeError(f"{patterns[0]=!r} must be a RegexPattern.") + if has_regex_flags(patterns): + raise ValueError("The re2 backend cannot preserve regex compile flags. Use backend='simple'.") + use_patterns = dict(enumerate_patterns( patterns, filter=True, reverse=False, )) diff --git a/pathspec/pathspec.py b/pathspec/pathspec.py index 6db7fff..d398391 100644 --- a/pathspec/pathspec.py +++ b/pathspec/pathspec.py @@ -75,7 +75,11 @@ def __init__( *backend* (:class:`str` or :data:`None`) is the pattern (regular expression) matching backend to use. Default is :data:`None` for "best" to use the best available backend. Priority of backends is: "re2", "hyperscan", "simple". - The "simple" backend is always available. + The "simple" backend is always available. "best" falls back to "simple" + for active :class:`re.Pattern` objects with compile flags not encoded in + the expression. + Explicitly selecting a native backend raises :exc:`ValueError` for those + patterns. Use ``backend='simple'`` to preserve their compile flags. """ if isinstance(patterns, Sequence): use_patterns = patterns @@ -141,8 +145,10 @@ def __iadd__(self: Self, other: PathSpec) -> Self: # type: ignore[misc] (:class:`PathSpec`) to this instance. """ if isinstance(other, PathSpec): - self.patterns = [*self.patterns, *other.patterns] - self._backend = self._make_backend(self._backend_name, self.patterns) + use_patterns = [*self.patterns, *other.patterns] + use_backend = self._make_backend(self._backend_name, use_patterns) + self.patterns = use_patterns + self._backend = use_backend return self else: return NotImplemented diff --git a/tests/test_07_regex_flags.py b/tests/test_07_regex_flags.py new file mode 100644 index 0000000..d64768d --- /dev/null +++ b/tests/test_07_regex_flags.py @@ -0,0 +1,235 @@ +""" +This script tests backend selection for compiled regular expression flags. +""" + +import copy +import re +import unittest +import warnings + +from pathspec import ( + GitIgnoreSpec, + PathSpec, + RegexPattern) +from pathspec._backends import agg +from pathspec._backends._utils import ( + has_regex_flags) +from pathspec._backends.hyperscan.gitignore import ( + HyperscanGiBackend) +from pathspec._backends.hyperscan.pathspec import ( + HyperscanPsBackend) +from pathspec._backends.re2.gitignore import ( + Re2GiBackend) +from pathspec._backends.re2.pathspec import ( + Re2PsBackend) +from pathspec._backends.simple.gitignore import ( + SimpleGiBackend) +from pathspec._backends.simple.pathspec import ( + SimplePsBackend) +from pathspec.patterns.gitignore.spec import ( + GitIgnoreSpecPattern) + +from .util import ( + require_backend) + + +class RegexFlagsTest(unittest.TestCase): + """ + Compiled flags must not be silently discarded by native backends. + """ + + def test_best_preserves_flags(self): + """ + Automatic selection preserves the original Python regex semantics. + """ + cases = [ + ('secret', re.IGNORECASE, 'SECRET', 'public'), + ('^secret$', re.MULTILINE, 'x\nsecret\ny', 'xsecret'), + ('^a.b$', re.DOTALL, 'a\nb', 'ab'), + ('s e c r e t # note', re.VERBOSE, 'secret', 's e c r e t # note'), + ('secret # [', re.VERBOSE, 'secret', 'public'), + ('^[a-z]+$', re.IGNORECASE, '\u0130\u0131\u017f\u212a', '!'), + ('^\\w+$', re.ASCII, 'secret', '\u00e9'), + ('^[a-z]+$', re.ASCII | re.IGNORECASE, 'SECRET', '\u0130'), + ('^secret.*end$', re.IGNORECASE | re.MULTILINE | re.DOTALL, + 'x\nSECRET\nEND\ny', 'SECRET'), + ('(?i)^secret$', re.MULTILINE, 'x\nSECRET\ny', 'xsecret'), + ] + for spec_cls, simple_cls in ( + (PathSpec, SimplePsBackend), (GitIgnoreSpec, SimpleGiBackend), + ): + for backend in (None, 'best', 'simple'): + for expression, flags, match, no_match in cases: + with self.subTest(spec=spec_cls, backend=backend, flags=flags, expression=expression): + pattern = RegexPattern(re.compile(expression, flags), include=True) + spec = spec_cls([pattern], backend=backend) + self.assertTrue(spec.match_file(match)) + self.assertFalse(spec.match_file(no_match)) + self.assertIsInstance(spec._backend, simple_cls) + + def test_explicit_native_rejects_flags(self): + """ + Explicit native selection reports unsupported compiled flags. + """ + for backend, ps_backend, gi_backend in ( + ('re2', Re2PsBackend, Re2GiBackend), + ('hyperscan', HyperscanPsBackend, HyperscanGiBackend), + ): + with self.subTest(backend=backend): + require_backend(backend) + for flags in (re.I, re.M, re.S, re.X, re.A, re.I | re.M | re.S): + for factory in ( + lambda patterns: PathSpec(patterns, backend=backend), + lambda patterns: GitIgnoreSpec(patterns, backend=backend), + ps_backend, gi_backend, + ): + with self.subTest(flags=flags, factory=factory): + patterns = [RegexPattern(re.compile('secret', flags), include=True)] + with self.assertRaisesRegex(ValueError, "backend='simple'"): + factory(patterns) + + def test_rejected_iadd_keeps_state(self): + """ + Rejecting flags during an in-place addition leaves the spec unchanged. + """ + for backend in ('re2', 'hyperscan'): + with self.subTest(backend=backend): + require_backend(backend) + for spec_cls in (PathSpec, GitIgnoreSpec): + with self.subTest(spec=spec_cls): + spec = spec_cls([RegexPattern(re.compile('secret'), include=True)], backend=backend) + patterns, matcher = spec.patterns, spec._backend + other = spec_cls([RegexPattern(re.compile('secret', re.I), include=False)], backend='simple') + with self.assertRaisesRegex(ValueError, "backend='simple'"): + spec += other + self.assertIs(spec.patterns, patterns) + self.assertIs(spec._backend, matcher) + self.assertEqual(len(spec), 1) + self.assertTrue(spec.match_file('secret')) + self.assertFalse(spec.match_file('SECRET')) + + def test_flag_detection(self): + """ + Detect external flags without translating bytes or unknown Python flags. + """ + for expression, flags, expected in ( + ('secret', 0, False), + ('secret', re.UNICODE, False), + ('(?i)secret', re.I, False), + ('(?i)secret', re.M, True), + ('secret', 1 << 20, True), + (b'secret', 0, False), + (b'secret', re.I, True), + (b'secret', re.LOCALE, True), + (b'(?i)secret', re.I, False), + ): + with self.subTest(expression=expression, flags=flags): + pattern = RegexPattern(re.compile(expression, flags), include=True) + self.assertEqual(has_regex_flags([pattern]), expected) + pattern.include = None + self.assertFalse(has_regex_flags([pattern])) + self.assertFalse(has_regex_flags([])) + self.assertFalse(has_regex_flags([RegexPattern(None)])) + + def test_probe_does_not_expose_warnings(self): + """ + Compiling a probe does not leak warnings or change the outer filters. + """ + for expression in ('secret # [a&&b]', 'secret # [[a]', 'secret # [a--b]'): + pattern = RegexPattern(re.compile(expression, re.X), include=True) + for backend in (None, 'best', 'simple', 're2', 'hyperscan'): + with self.subTest(expression=expression, backend=backend): + require_backend(backend) + for action in ('always', 'error'): + with self.subTest(action=action): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter(action, FutureWarning) + filters = warnings.filters[:] + re.purge() + self.assertTrue(has_regex_flags([pattern])) + re.purge() + if backend in ('re2', 'hyperscan'): + with self.assertRaisesRegex(ValueError, "backend='simple'"): + PathSpec([pattern], backend=backend) + else: + self.assertTrue(PathSpec([pattern], backend=backend).match_file('secret')) + self.assertEqual(warnings.filters, filters) + self.assertEqual(caught, []) + + def test_mixed_flags_and_rebuild(self): + """ + Fallback keeps pattern indices and precedence after copying or adding. + """ + for spec_cls in (PathSpec, GitIgnoreSpec): + with self.subTest(spec=spec_cls): + spec = spec_cls([ + RegexPattern(None), + RegexPattern(re.compile('secret', re.I), include=True), + RegexPattern(re.compile('secret'), include=False), + ]) + for current in (spec, copy.copy(spec), spec + spec_cls([])): + upper = current.check_file('SECRET') + lower = current.check_file('secret') + self.assertEqual((upper.include, upper.index), (True, 1)) + self.assertEqual((lower.include, lower.index), (False, 2)) + self.assertEqual(list(current.match_files(['SECRET', 'secret', 'public'])), ['SECRET']) + base = spec_cls([RegexPattern(re.compile('secret'), include=False)]) + base += spec_cls([RegexPattern(re.compile('secret', re.I), include=True)]) + result = base.check_file('SECRET') + self.assertEqual((result.include, result.index), (True, 1)) + + def test_gitignore_directory_priority(self): + """ + GitIgnoreSpec fallback retains its ancestor-directory exclusion rules. + """ + patterns = [] + for line in ('logs/', '!logs/KEEP'): + pattern = GitIgnoreSpecPattern(line) + assert pattern.regex is not None, pattern + patterns.append(GitIgnoreSpecPattern( + re.compile(pattern.regex.pattern, re.I), pattern.include, + )) + for backend in (None, 'best', 'simple'): + with self.subTest(backend=backend): + spec = GitIgnoreSpec(patterns, backend=backend) + self.assertIsInstance(spec._backend, SimpleGiBackend) + result = spec.check_file('LOGS/KEEP') + self.assertEqual((result.include, result.index), (True, 0)) + plain_result = PathSpec(patterns, backend=backend).check_file('LOGS/KEEP') + self.assertEqual((plain_result.include, plain_result.index), (False, 1)) + + def test_native_controls(self): + """ + Plain regexes, inline flags, and no-op patterns keep native selection. + """ + for backend in ('best', 're2', 'hyperscan'): + with self.subTest(backend=backend): + require_backend(backend) + for spec_cls in (PathSpec, GitIgnoreSpec): + for expression, file in ( + ('secret', 'secret'), + ('(?i)secret', 'SECRET'), + ('(?m)^secret$', 'x\nsecret\ny'), + ('(?s)^a.b$', 'a\nb'), + ): + with self.subTest(spec=spec_cls, expression=expression): + patterns = [ + RegexPattern(re.compile('ignored', re.I), include=None), + RegexPattern(re.compile(expression), include=True), + ] + spec = spec_cls(patterns, backend=backend) + expected = agg._BEST_BACKEND if backend == 'best' else backend + self.assertEqual(type(spec._backend).__name__, { + ('re2', PathSpec): 'Re2PsBackend', + ('re2', GitIgnoreSpec): 'Re2GiBackend', + ('hyperscan', PathSpec): 'HyperscanPsBackend', + ('hyperscan', GitIgnoreSpec): 'HyperscanGiBackend', + ('simple', PathSpec): 'SimplePsBackend', + ('simple', GitIgnoreSpec): 'SimpleGiBackend', + }[expected, spec_cls]) + self.assertTrue(spec.match_file(file)) + self.assertFalse(spec.match_file('public')) + + +if __name__ == '__main__': + unittest.main()