diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 32704f5..e1d7f30 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,17 +3,65 @@ name: Publish to crates.io on: push: tags: - - 'v*' + - "v*" + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUSTDOCFLAGS: -D warnings jobs: - build: + publish: runs-on: ubuntu-latest + # Gate publishing behind a protected GitHub environment so the crates.io + # token is only exposed to reviewed, tag-triggered runs. Configure required + # reviewers on this environment in the repository settings. + environment: crates-io steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master with: - toolchain: stable - override: true - - uses: katyo/publish-crates@v2 + toolchain: stable + components: rustfmt, clippy + + # Never publish a crate whose version disagrees with the pushed tag. + - name: Verify tag matches Cargo.toml version + run: | + crate_version="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')" + tag_version="${GITHUB_REF_NAME#v}" + if [ "$crate_version" != "$tag_version" ]; then + echo "::error::Tag ${GITHUB_REF_NAME} (${tag_version}) does not match Cargo.toml version ${crate_version}" + exit 1 + fi + + - name: Format + run: cargo fmt --all -- --check + - name: Lint + run: | + cargo clippy --locked --all-targets -- -D warnings + cargo clippy --locked --no-default-features --all-targets -- -D warnings + - name: Test + run: | + cargo test --locked + cargo test --locked --no-default-features + - name: Docs + run: cargo doc --locked --no-deps + + # Also verify the declared MSRV before publishing, like CI does. + - name: Read MSRV from Cargo.toml + id: msrv + run: echo "msrv=$(grep -m1 '^rust-version' Cargo.toml | cut -d'"' -f2)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master with: - registry-token: ${{ secrets.CRATES_API_TOKEN }} + toolchain: ${{ steps.msrv.outputs.msrv }} + - name: Check on MSRV + run: | + cargo +${{ steps.msrv.outputs.msrv }} check --locked --all-features + cargo +${{ steps.msrv.outputs.msrv }} check --locked --no-default-features + + - name: Publish + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_API_TOKEN }} + run: cargo +stable publish --locked diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7730802..3be6c62 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,23 +2,68 @@ name: Rust on: push: - branches: [ "main" ] + branches: ["main", "v1"] pull_request: - branches: [ "main" ] + workflow_dispatch: + +permissions: + contents: read env: CARGO_TERM_COLOR: always + RUSTDOCFLAGS: -D warnings jobs: build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master + with: + toolchain: stable + components: rustfmt, clippy + - name: Format + run: cargo fmt --all -- --check + - name: Build + run: cargo build --locked --verbose + - name: Test & Lint + run: | + cargo test --locked + cargo clippy --locked --all-targets -- -D warnings + - name: Test & Lint (no default features) + run: | + cargo test --locked --no-default-features + cargo clippy --locked --no-default-features --all-targets -- -D warnings + - name: Docs + run: cargo doc --locked --no-deps + + # Keep the declared `rust-version` (MSRV) honest, for both feature configs. + msrv: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + # Single source of truth: the MSRV comes from Cargo.toml. + - name: Read MSRV from Cargo.toml + id: msrv + run: echo "msrv=$(grep -m1 '^rust-version' Cargo.toml | cut -d'"' -f2)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master + with: + toolchain: ${{ steps.msrv.outputs.msrv }} + - name: Check on MSRV + run: | + cargo check --locked --all-features + cargo check --locked --no-default-features + audit: runs-on: ubuntu-latest + permissions: + contents: read + issues: write # audit-check files an issue when a vulnerability is found steps: - - uses: actions/checkout@v4 - - name: Build - run: cargo build --all-features --verbose - - name: Test & Lint - run: | - cargo test --all-features - cargo clippy --all-features + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index d01bd1a..40cf6e9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,10 +3,6 @@ debug/ target/ -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk @@ -18,4 +14,7 @@ Cargo.lock # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +# cargo mutants output +mutants.out*/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..167674a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,160 @@ +# Changelog + +All notable changes to this crate are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2026-08-08 + +This is a major redesign of the public API around the `Term` enum (wrapping +either a `RegularExpression` or a `FastAutomaton`), which dispatches each +operation to the cheaper representation when possible. See the crate-level +docs for the architecture. Almost the entire public surface changed; highlights +below. + +### Added +- New `Term` constructors and conversions: `new_empty`, `new_total`, + `new_empty_string`, `from_pattern`, `from_regex(RegularExpression)`, + `from_automaton(FastAutomaton)`, plus `From`, + `From`, `FromStr`, `Display`, and `Default` (= `new_empty`). +- New `Term` operations: `concat`, `complement`, `determinize`, `minimize`, + `matches`, `is_deterministic`, `is_minimal`, `is_finite`, `to_pattern`, and + `iter_strings` (lazy `StringGenerator` iterator). +- `RegularExpression::MAX_NESTING_DEPTH` and a depth check in `to_automaton` + that returns the new `EngineError::RegexTooDeeplyNested` for hand-built trees + nested past the limit, instead of overflowing the stack. +- `to_automaton` returns `Cow` to avoid unnecessary cloning; `to_regex` + returns `Result, EngineError>` (see Changed — it is now fallible). +- `FastAutomaton` gained corresponding low-level constructors/operations + (`new_empty`, `new_total`, `new_empty_string`, `determinize`, `minimize` + using Hopcroft's algorithm, `is_minimal`, `unaccept`, `print_dot`, + `try_add_transition`) and inspection helpers (`states`, `direct_states`, + `transitions_from`, `transitions_to_vec`, `has_transition`, ...). +- New `EngineError` variants: `InvalidRepetitionBounds`, + `IncompatibleSpanningSet`, `DeterministicAutomatonRequired`, + `UnsupportedRegexFeature`, `RegexTooDeeplyNested`; the enum is now + `#[non_exhaustive]`. +- `tracing` instrumentation on the core `Term`, `FastAutomaton`, and + `RegularExpression` operations (concat, union, intersection, difference, + complement, repeat, determinize, minimize, equivalence/subset checks, + string generation, conversions). No-op unless a `tracing` subscriber is + installed. +- Parallel (Rayon-backed) variants of union/intersection for >3 operands, + gated behind the default-on `parallel` feature, with sequential fallbacks + for `--no-default-features`. +- `cargo fmt --check`, `cargo clippy -- -D warnings`, `cargo doc --no-deps` + (with `RUSTDOCFLAGS=-D warnings`), and a dependency vulnerability audit + (`rustsec/audit-check`) to CI. +- `regex_charclass` is re-exported at the crate root, so automata can be + built by hand (`Char`, `CharRange`) without adding a separately + version-matched dependency. +- `NoHashHasher`, the crate-owned no-op hasher behind the `IntSet` state-id + sets (replaces the `nohash-hasher` dependency, keeping 0.x types out of + the public API). +- `EngineError` implements `Clone`; `StringGenerator` implements `Debug` and + `FusedIterator`; `ExecutionProfileBuilder` implements `Debug` and `Clone`. +- `PathOrder` and `CharacterOrder`, the two independent axes + `generate_strings`/`iter_strings` enumerate a language along. + `PathOrder::Sweep` is the previous behaviour: shortest strings first, one + path expanded in full before the next. `PathOrder::Interleave` covers + every shape the automaton holds before asking any of them for a second + string, so `.*abc.*` yields `abc`, `abc\u{0}`, `\u{0}abc`, ... instead of + a million variations of `abc\u{0}`. `PathOrder::Shuffled` interleaves and + additionally visits same-length shapes in a seed-drawn order, so *which* + shapes a small `limit` reaches looks random too. + `CharacterOrder::Ascending` (the default) expands each position from the + low end of its character range; `CharacterOrder::Shuffled` draws each + path's combinations through a seeded permutation instead — `[a-z]{8}` + yields something like `sjtwsive` rather than `aaaaaaaa` (the exact + shuffled sequence is implementation-defined). Every combination of the axes + enumerates the same strings, stays deterministic (the seed defaults to 0), + and pages with `offset` the same way; both axes shuffled is the mode to + derive realistic test cases from a pattern. +- `GenerationOptions`, what `generate_strings`/`iter_strings` may generate: + the two axes, the seed behind the `Shuffled` modes of both + (`with_seed(u64)`), an optional charset (`with_charset(CharRange)`) that + keeps generation to a set of characters — only strings made entirely of + them come out; a path needing a ruled-out character is dropped whole, + never shortened, so `.*abc.*` over `[ -~]` yields `abc`, `abc `, `abc!`, + ... — plus optional length bounds (`with_min_length`/`with_max_length`) + that confine the enumeration to a band of string lengths, `offset` never + counting the strings outside it. Without a max, a deep `offset` into a + looping language like `.*` pages into arbitrarily long strings. + +### Changed +- `Term::to_regex`/`to_pattern` and `FastAutomaton::to_regex` are now fallible + (`Result<_, EngineError>`) and honor the `ExecutionProfile` timeout, since + state elimination can grow super-polynomially on adversarial automata. + `Term`'s `Display` therefore renders a pattern only for regex-backed terms; + automaton-backed terms display as Graphviz DOT (use `to_pattern` for a + parseable pattern). +- `ExecutionProfile` redesigned as an immutable, thread-local-aware config + built via the new `ExecutionProfileBuilder`, governing execution timeouts, + state-count limits, and an `implicit_determinization` toggle. +- `union`/`intersection`/`concat` now take + `impl IntoIterator>` instead of `&[Term]`, so + `&[a, b]`, `[&a, &b]`, and `Vec` all work without cloning. +- `repeat` now takes `impl RangeBounds` (e.g. `3..6`, `..=2`) instead of + explicit min/max parameters. +- `generate_strings` now takes `(limit, offset, options)`: pagination instead + of a single `count`, plus the `GenerationOptions` to generate under (the + two enumeration axes, a seed, and optionally a charset and length bounds). + A `PathOrder`, a `CharacterOrder`, or a `(PathOrder, CharacterOrder)` pair + converts into options, so any of them can be passed on its own. + `iter_strings` takes the same `options`. +- `is_empty`, `is_total`, and `is_empty_string` now return + `Result` instead of `bool`. +- `are_equivalent`/`is_subset_of` renamed to `equivalent`/`subset`. +- `subtraction` renamed to `difference`, kept single-operand by design. +- Renamed `FastAutomaton::as_dot` to `to_dot` (old printing `to_dot` is now + `print_dot`), matching the crate's `to_*` convention for allocating + conversions (`to_pattern`, `to_regex`, `to_automaton`, `to_range`). +- Renamed `get_*` accessors to drop the `get_` prefix, per the Rust API + Guidelines' C-GETTER convention: `Term::get_length` to `length`, + `Term::get_cardinality` to `cardinality`, `FastAutomaton::get_length` to + `length`, `FastAutomaton::get_cardinality` to `cardinality`, + `FastAutomaton::get_number_of_states` to `number_of_states`, + `FastAutomaton::get_condition` to `condition`, + `FastAutomaton::get_start_state` to `start_state`, + `FastAutomaton::get_accept_states` to `accept_states`, + `FastAutomaton::get_spanning_set` to `spanning_set`, + `FastAutomaton::get_live_states` to `live_states`, + `FastAutomaton::get_spanning_bases` to `spanning_bases`, + `RegularExpression::get_length` to `length`, + `RegularExpression::get_cardinality` to `cardinality`, + `SpanningSet::get_spanning_ranges` to `spanning_ranges`, + `SpanningSet::get_number_of_spanning_ranges` to + `number_of_spanning_ranges`, `SpanningSet::get_spanning_range` to + `spanning_range`, `SpanningSet::get_rest` to `rest`, + `Condition::get_cardinality` to `cardinality`, + `Condition::get_binary_representation` to `binary_representation`, + `ConditionConverter::get_from_spanning_set`/`get_to_spanning_set` to + `from_spanning_set`/`to_spanning_set`. +- Error messages follow the std convention (lowercase, no trailing + punctuation) so they compose cleanly when wrapped by callers. +- Edition bumped to 2024 and `Cargo.toml` metadata (`description`, + `categories`) updated. + +### Removed +- The `serde` feature and all serialization, FAIR (base85) encoding, + encryption, and compression support (`serde`, `ciborium`, `z85`, + `aes-gcm-siv`, `sha2`, `flate2` dependencies). +- `Term::get_details` and the `Details` type. +- The `tokenizer` module. +- Unused `log`, `rand`, and `lazy_static` dependencies, and the `regex` + crate dependency (now dev-only, used by integration tests). +- The `nohash-hasher` dependency (replaced by the crate-owned + `NoHashHasher`; see Added). +- `EngineError` variants `AutomatonShouldBeDeterministic`, `TooMuchTerms`, + `ConditionIndexOutOfBound`, `TokenError`, and the `is_server_error` method. +- The `max_number_of_terms` execution-profile limit (no longer enforced). + +## Earlier releases + +Releases prior to 1.0.0 (`v0.1.0` through `v0.3.1`) predate this changelog; +see the [GitHub tags](https://github.com/RegexSolver/regexsolver/tags) and +commit history for details. + +[Unreleased]: https://github.com/RegexSolver/regexsolver/compare/v1.0.0...HEAD +[1.0.0]: https://github.com/RegexSolver/regexsolver/compare/v0.3.1...v1.0.0 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..a398ab9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,975 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "irange" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de8eb46ec263fe6dd2dea011ecc4cfbf23ef16be6ec5984ac1b75d8edad19d6" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-charclass" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fe903a137888ed31ff2b00dd49871885cb9abede0a42887dd82f4080aad2ad7" +dependencies = [ + "irange", + "once_cell", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "regexsolver" +version = "1.0.0" +dependencies = [ + "ahash", + "bit-set", + "criterion", + "indexmap", + "proptest", + "rayon", + "regex", + "regex-charclass", + "regex-syntax", + "tracing", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index cd03087..958b154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,50 +1,37 @@ [package] name = "regexsolver" -version = "0.3.1" -edition = "2021" +version = "1.0.0" +edition = "2024" +rust-version = "1.88" authors = ["Alexandre van Beurden"] repository = "https://github.com/RegexSolver/regexsolver" +homepage = "https://github.com/RegexSolver/regexsolver" +documentation = "https://docs.rs/regexsolver" license = "MIT" keywords = ["automaton", "intersection", "union", "difference", "regex"] -description = "Manipulate regex and automaton as if they were sets." +categories = ["text-processing", "mathematics", "algorithms"] +description = "High-performance Rust library for building, combining, and analyzing regular expressions and finite automata" readme = "README.md" +exclude = ["/assets"] [dependencies] -serde = { version = "1.0", features = ["derive"], optional = true } -ciborium = { version = "0.2.2", optional = true } -z85 = { version = "3.0.5", optional = true } -aes-gcm-siv = { version = "0.11.1", optional = true } -sha2 = { version = "0.10.8", optional = true } -flate2 = { version = "1.0.30", features = [ - "zlib-ng", -], default-features = false, optional = true } -nohash-hasher = "0.2" +tracing = "0.1" ahash = "0.8.11" -log = "0.4.21" -rand = "0.8.5" -lazy_static = "1.4.0" -regex = "1.10.3" regex-syntax = "0.8.5" regex-charclass = { version = "1.0.3" } - -[dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } -env_logger = "0.11.3" -serde_json = "1.0.114" - +rayon = { version = "1.10.0", optional = true } +bit-set = "0.8.0" +indexmap = "2.13.0" [features] -default = ["serde"] -serde = [ - "regex-charclass/serde", - "dep:serde", - "dep:ciborium", - "dep:z85", - "dep:aes-gcm-siv", - "dep:sha2", - "dep:flate2", -] +default = ["parallel"] +parallel = ["dep:rayon"] + +[dev-dependencies] +criterion = { version = "0.8", features = ["html_reports"] } +proptest = "1" +regex = "1.10.3" [[bench]] -name = "my_benchmark" -harness = false \ No newline at end of file +name = "operations" +harness = false diff --git a/LICENSE b/LICENSE index 5745dae..d98b41b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 RegexSolver +Copyright (c) 2024-2026 RegexSolver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index dcb0b47..2f05f23 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,246 @@ # RegexSolver [![Crates.io Version](https://img.shields.io/crates/v/regexsolver)](https://crates.io/crates/regexsolver) +[![docs.rs](https://img.shields.io/docsrs/regexsolver)](https://docs.rs/regexsolver) +[![CI](https://github.com/RegexSolver/regexsolver/actions/workflows/rust.yml/badge.svg)](https://github.com/RegexSolver/regexsolver/actions/workflows/rust.yml) +[![License: MIT](https://img.shields.io/crates/l/regexsolver)](LICENSE) -This repository contains the code of [RegexSolver](https://regexsolver.com/) engine. +The `regex` crate tells you whether a *string* matches a pattern. **RegexSolver treats patterns as the sets of strings they match** — so you can intersect, subtract, compare, complement, and enumerate them, and get the result back as a regex. -For more information, you can check the library's [documentation](https://docs.rs/regexsolver/latest/regexsolver/). +```rust +use regexsolver::{Term, fast_automaton::PathOrder}; -If you want to use this library with other programming languages, we provide a wide range of wrappers: +let a: Term = "(ab|xy){2}".parse()?; +let b: Term = ".*xy".parse()?; -- [regexsolver-java](https://github.com/RegexSolver/regexsolver-java) -- [regexsolver-js](https://github.com/RegexSolver/regexsolver-js) -- [regexsolver-python](https://github.com/RegexSolver/regexsolver-python) +// Which strings match BOTH patterns? Get the answer as a regex: +let both = a.intersection([&b])?; +assert_eq!(both.to_pattern()?, "(ab|xy)xy"); + +// Test a concrete string against the result (matching is anchored): +assert!(both.matches("abxy")?); + +// ...and sample them: +assert_eq!(both.generate_strings(2, 0, PathOrder::Sweep)?, ["xyxy", "abxy"]); +``` -For more information about how to use the wrappers, you can refer to our [getting started guide](https://docs.regexsolver.com/getting-started.html). +## What would you use this for? -## Installation +- **Safe migrations** - `old_rule.subset(&new_rule)?`: does the new validation pattern accept *everything* the old one did? +- **Test-data generation** - `term.generate_strings(100, 0, (PathOrder::Shuffled, CharacterOrder::Shuffled))?`: produce realistic-looking strings matching any pattern, spread over the cases the pattern allows, reproducible by seed, restricted to the characters you can use (`with_charset`) and to a band of lengths (`with_min_length`/`with_max_length`), with pagination. +- **Rule analysis**: find shadowed or overlapping routes, firewall rules, and validators with `intersection` / `difference`. +- **Equivalence proofs** - `a.equivalent(&b)?`: show that two differently-written patterns match exactly the same strings. +- **Pattern simplification**: every operation returns a `Term` you can turn back into a regex pattern with `to_pattern()`. -Add the following line in your `Cargo.toml`: +Under the hood, every pattern compiles to a finite automaton: + +

the minimal automaton for text containing abc then def

+

The minimal automaton for .*abc.*def.* (text containing abc then def), rendered from this library's to_dot().

+ +## Try it + +```bash +git clone https://github.com/RegexSolver/regexsolver && cd regexsolver + +# How do two patterns relate? (equivalence, subsets, intersection, differences) +cargo run --example relate -- "(ab|xy){2}" ".*xy" + +# Generate n sample strings matching a pattern +cargo run --example generate -- "[a-z]{2}[0-9]" 20 +``` + +Or in your own project: + +```bash +cargo add regexsolver +``` + +By default the `parallel` feature is enabled: automaton-backed unions/intersections with more than 3 argument operands and parts of the automaton-to-regex conversion run on [rayon](https://crates.io/crates/rayon) (purely regex-backed operations stay sequential). Disable it for a leaner dependency tree on single-threaded workloads: ```toml -[dependencies] -regexsolver = "0.3" +regexsolver = { version = "1", default-features = false } ``` -## Examples +## Semantics in 30 seconds + +RegexSolver implements **pure regular languages**, which differs from typical regex engines in two ways: + +- **Everything is anchored**: `abc` matches the string "abc", not "xabc" or "abcx". Patterns describe *whole strings*. +- **`.` matches any character**, including line feed (`\n`). + +The rest follows from regular-language theory: + +- **Backreferences** (`\1`, `\2`, ...) go beyond regular languages and return an error, as do **lookahead/lookbehind** assertions (`(?=...)`, `(?<=...)`). +- **Anchors and word boundaries**: since matching is already full-string, a leading `^`/`\A` and a trailing `$`/`\z` are accepted as redundant no-ops. Anchors anywhere else, and word boundaries (`\b`, `\B`), would constrain matching in ways a pure regular language can't express, so they return `EngineError::UnsupportedRegexFeature` rather than silently changing the language. +- **Inline flags** (`(?i)`, `(?m)`, `(?s)`, `(?x)`) return `EngineError::UnsupportedRegexFeature`: the engine matches character ranges uniformly and can't honor them, and silently dropping them would diverge from standard regex semantics (e.g. `(?i)abc` would no longer match `ABC`). +- **All quantifiers are greedy**: ungreedy markers (`*?`, `+?`, `??`) are ignored as *sets of strings*, `a*` and `a*?` are the same language. +- **The empty language** (matches no string at all) is written `[]` (empty character class). This is distinct from the empty string `""`. + +RegexSolver is based on the [regex-syntax](https://docs.rs/regex-syntax/0.8.5/regex_syntax/) library for parsing patterns. Features that don't affect the language as a set of strings (such as ungreedy markers) are accepted and ignored; features that would change matching in a way the engine can't represent (backreferences, lookaround, inline flags, and unsupported anchor/boundary positions) return an `EngineError` instead of being applied incorrectly. + +## A tour of the API -### Union +[`Term`](https://docs.rs/regexsolver/latest/regexsolver/enum.Term.html) is the type you'll interact with: it wraps either a regular expression or an automaton and picks the best representation for each operation. The essentials: + +| Method | Description | +| -------- | ------- | +| `Term::from_pattern(pattern)` | Parses a pattern into a term. | +| `intersection(&self, terms)` / `union(&self, terms)` | Set operations over any number of terms. | +| `difference(&self, other)` / `complement(&self)` | What `self` matches and `other` doesn't / everything `self` doesn't match. | +| `concat(&self, terms)` / `repeat(&self, range)` | Sequence and repeat languages; `range` is any Rust range expression (`2..=5`, `1..`, `..3`, ...). | +| `equivalent(&self, other)` / `subset(&self, other)` | Compare languages. | +| `is_empty()` / `is_total()` / `length()` / `cardinality()` | Analyze a language: matches nothing? everything? string lengths? how many strings? | +| `generate_strings(limit, offset, options)` | Enumerate matching strings eagerly (call `determinize()` or `minimize()` once first when paginating). | +| `iter_strings(options)` | Lazy iterator equivalent; computes the deterministic automaton once and yields strings in batches. `options.with_min_length(n)`/`.with_max_length(n)` confine the walk to a band of lengths — with a max, even an infinite language yields a finite iterator. | +| `to_pattern()` / `to_automaton()` / `to_regex()` | Convert back out. | + +All fallible operations return `Result<_, EngineError>`. + +### Building automata by hand + +`FastAutomaton` is used to directly build, manipulate and analyze automata. To convert an automaton to a `RegularExpression` the method `to_regex()` can be used. + +States are created with `new_state()` and transitions with `add_transition_from_range`, which labels the transition with a plain `CharRange`: ```rust -use regexsolver::Term; +use regexsolver::CharRange; +use regexsolver::fast_automaton::FastAutomaton; +use regexsolver::regex_charclass::char::Char; + +// Build an automaton matching "[a-c][0-9]*" by hand: +let mut automaton = FastAutomaton::new_empty(); +let s1 = automaton.new_state(); +automaton.accept(s1); + +let a_to_c = CharRange::new_from_range(Char::new('a')..=Char::new('c')); +let digits = CharRange::new_from_range(Char::new('0')..=Char::new('9')); +automaton.add_transition_from_range(0, s1, &a_to_c)?; +automaton.add_transition_from_range(s1, s1, &digits)?; + +assert!(automaton.is_match("b42")); +assert!(!automaton.is_match("4b")); +assert_eq!(automaton.to_regex()?.to_string(), "[a-c][0-9]*"); +``` -let term1 = Term::from_regex("abc").unwrap(); -let term2 = Term::from_regex("de").unwrap(); -let term3 = Term::from_regex("fghi").unwrap(); +Internally, transition labels are bitvector `Condition`s over the automaton's `SpanningSet` of disjoint character ranges, that is what makes label union/intersection/complement O(1) ([article](https://alexvbrdn.me/post/optimizing-transition-conditions-automaton-representation)). `add_transition_from_range` maintains that representation for you; for full manual control over conditions and spanning sets, see the [`add_transition` documentation](https://docs.rs/regexsolver/latest/regexsolver/fast_automaton/struct.FastAutomaton.html#method.add_transition). -let union = term1.union(&[term2, term3]).unwrap(); +Everything `Term` does is also available directly on [`FastAutomaton`](https://docs.rs/regexsolver/latest/regexsolver/fast_automaton/struct.FastAutomaton.html), including `determinize`, `minimize`, the set operations, `equivalent`/`subset`, the analyses, `generate_strings`, `to_regex`, plus low-level construction (`new_state`, `accept`, `add_epsilon_transition`, ...) and inspection (`states`, `transitions_from`, `to_dot`, ...). -if let Term::RegularExpression(regex) = union { - println!("{}", regex.to_string()); // (abc|de|fghi) +### Working with patterns as ASTs + +`RegularExpression` is the parsed pattern itself: a plain AST enum (`Character` / `Repetition` / `Concat` / `Alternation`) you can analyze and walk directly. Set operations like intersection and difference live on `FastAutomaton` (or, more conveniently, on `Term`); convert with `to_automaton()`. + +```rust +use regexsolver::regex::RegularExpression; + +// A validation pattern for an order id, e.g. "ORD-2024-12345". +let pattern = RegularExpression::new("ORD-20[0-9]{2}-[0-9]{4,6}")?; + +// How long can matching ids get? Size your database column accordingly. +assert_eq!(pattern.length(), (Some(13), Some(15))); + +// The AST is a plain enum: walk it to lint patterns, e.g. reject +// validation rules that accept unboundedly long input. +fn has_unbounded_repetition(regex: &RegularExpression) -> bool { + match regex { + RegularExpression::Character(_) => false, + RegularExpression::Repetition(inner, _, max) => { + max.is_none() || has_unbounded_repetition(inner) + } + RegularExpression::Concat(parts) => parts.iter().any(has_unbounded_repetition), + RegularExpression::Alternation(parts) => parts.iter().any(has_unbounded_repetition), + } } +assert!(!has_unbounded_repetition(&pattern)); +assert!(has_unbounded_repetition(&RegularExpression::new(".*@example\\.com")?)); ``` -### Intersection +The variants are freely constructible too; a hand-built repetition whose maximum is below its minimum denotes no valid language and is rejected with `EngineError::InvalidRepetitionBounds` when converted by `to_automaton()`. + +Parsing (`new`, `parse`), the simplifying combinators (`concat`, `union`, `repeat`, `simplify`) and the analyses (`length`, `cardinality`, `evaluate_complexity`) are documented on [`RegularExpression`](https://docs.rs/regexsolver/latest/regexsolver/regex/enum.RegularExpression.html). + +## Bound Execution + +Automaton operations can blow up on adversarial inputs, so the engine is built to run untrusted patterns safely: a thread-local `ExecutionProfile` caps runtime and state explosion, and controls when the engine may determinize or minimize on its own. Hitting a limit returns a specific `EngineError` instead of hanging or panicking. + +### Time-Bounded Execution ```rust -use regexsolver::Term; +use regexsolver::{Term, execution_profile::{ExecutionProfile, ExecutionProfileBuilder}, error::EngineError, fast_automaton::GenerationOptions}; -let term1 = Term::from_regex("(abc|de){2}").unwrap(); -let term2 = Term::from_regex("de.*").unwrap(); -let term3 = Term::from_regex(".*abc").unwrap(); +let term = Term::from_pattern(".*abc.*cdef.*sqdsqf.*")?; -let intersection = term1.intersection(&[term2, term3]).unwrap(); +let execution_profile = ExecutionProfileBuilder::new() + .execution_timeout(50) // limit in milliseconds + .build(); -if let Term::RegularExpression(regex) = intersection { - println!("{}", regex.to_string()); // deabc -} +// Asking for 100 million strings cannot finish within the budget, so the +// generation aborts instead of running to completion. +execution_profile.run(|| { + assert_eq!(EngineError::OperationTimeOutError, term.generate_strings(100_000_000, 0, GenerationOptions::new()).unwrap_err()); +}); +``` + +### State-Limited Execution + +```rust +use regexsolver::{Term, execution_profile::{ExecutionProfile, ExecutionProfileBuilder}, error::EngineError}; + +let term1 = Term::from_pattern(".*abcdef.*")?; +let term2 = Term::from_pattern(".*defabc.*")?; + +let execution_profile = ExecutionProfileBuilder::new() + .max_number_of_states(5) // we set the limit + .build(); + +// We run the operation with the defined limitation +execution_profile.run(|| { + assert_eq!(EngineError::AutomatonHasTooManyStates, term1.intersection(&[term2]).unwrap_err()); +}); ``` -### Difference/Subtraction +### Disabling Implicit Determinization + +`FastAutomaton` operations that require a deterministic automaton (`minimize`, `complement`, `difference`, `equivalent`, `subset`, `cardinality`, ...) determinize a non-deterministic input on their own by default. Since subset construction can blow up exponentially, this can be disabled: those operations then return `EngineError::DeterministicAutomatonRequired` instead, and determinization only happens through an explicit `determinize()` call. Deterministic inputs are always accepted, and the whole `Term` API keeps working since that layer manages the underlying representation itself, so its determinizations count as explicit. ```rust use regexsolver::Term; +use regexsolver::execution_profile::ExecutionProfileBuilder; +use regexsolver::error::EngineError; -let term1 = Term::from_regex("(abc|de)").unwrap(); -let term2 = Term::from_regex("de").unwrap(); +// Any non-deterministic FastAutomaton; ".*abc" compiles to one. +let nfa = Term::from_pattern(".*abc")?.to_automaton()?.into_owned(); +assert!(!nfa.is_deterministic()); -let subtraction = term1.subtraction(&term2).unwrap(); +let execution_profile = ExecutionProfileBuilder::new() + .implicit_determinization(false) // default is true + .build(); -if let Term::RegularExpression(regex) = subtraction { - println!("{}", regex.to_string()); // abc -} +execution_profile.run(|| { + let mut cannot_minimize = nfa.clone(); + assert_eq!(EngineError::DeterministicAutomatonRequired, cannot_minimize.minimize().unwrap_err()); + + // Determinizing explicitly is always allowed. + let mut dfa = nfa.determinize().unwrap().into_owned(); + assert!(dfa.minimize().is_ok()); +}); ``` + +## How it works + +- Patterns are parsed with [regex-syntax](https://docs.rs/regex-syntax/latest/regex_syntax/) and simplified into a small regular-expression AST; set operations run on finite automata; results convert back to patterns via state elimination. +- Transition labels are bitvectors over a per-automaton "spanning set" of disjoint character ranges, making label union/intersection/complement O(1): see [Optimizing Automaton Representation with Transition Conditions](https://alexvbrdn.me/post/optimizing-transition-conditions-automaton-representation). +- Correctness is cross-validated against the `regex` crate and exercised by property-based tests over randomly generated automata and expressions, with brute-force oracles for the analyses. + +## Cross-Language Support + +If you want to use this library with other programming languages, we provide a wide range of wrappers: +- [regexsolver-java](https://github.com/RegexSolver/regexsolver-java) +- [regexsolver-js](https://github.com/RegexSolver/regexsolver-js) +- [regexsolver-python](https://github.com/RegexSolver/regexsolver-python) + +For more information about how to use the wrappers, you can refer to our [guide](https://docs.regexsolver.com/getting-started.html). + +## License + +This project is licensed under the MIT License. diff --git a/assets/automaton.svg b/assets/automaton.svg new file mode 100644 index 0000000..d20e3e6 --- /dev/null +++ b/assets/automaton.svg @@ -0,0 +1,186 @@ + + + + + + +Automaton + + + +initial + + + + +0 + +0 + + + +initial->0 + + + + + +6 + + +6 + + + +6->6 + + +. + + + +0->0 + + +[^a] + + + +1 + +1 + + + +0->1 + + +a + + + +1->0 + + +[^ab] + + + +1->1 + + +a + + + +2 + +2 + + + +1->2 + + +b + + + +2->0 + + +[^ac] + + + +2->1 + + +a + + + +3 + +3 + + + +2->3 + + +c + + + +3->3 + + +[^d] + + + +4 + +4 + + + +3->4 + + +d + + + +4->3 + + +[^de] + + + +4->4 + + +d + + + +5 + +5 + + + +4->5 + + +e + + + +5->6 + + +f + + + +5->3 + + +[^df] + + + +5->4 + + +d + + + diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs deleted file mode 100644 index f2f9fdc..0000000 --- a/benches/my_benchmark.rs +++ /dev/null @@ -1,87 +0,0 @@ -use ahash::AHashSet; -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use regexsolver::{fast_automaton::FastAutomaton, regex::RegularExpression}; - -fn parse_regex(regex: &str) -> RegularExpression { - RegularExpression::new(regex).unwrap() -} - -fn to_regex(automaton: &FastAutomaton) -> RegularExpression { - automaton.to_regex().unwrap() -} - -fn determinize(automaton: &FastAutomaton) -> FastAutomaton { - automaton.determinize().unwrap() -} - -fn intersection(automaton_1: &FastAutomaton, automaton_2: &FastAutomaton) -> FastAutomaton { - automaton_1.intersection(automaton_2).unwrap() -} - -fn generate_strings(automaton: &FastAutomaton) -> AHashSet { - automaton.generate_strings(2000).unwrap() -} - -fn criterion_benchmark(c: &mut Criterion) { - { - c.bench_function("parse_regex", |b| { - b.iter(|| parse_regex(black_box("a(bcfe|bcdg|mkv)*(abc){2,3}(abc){2}"))) - }); - } - - { - let input_regex = RegularExpression::new("a(bcfe|bcdg|mkv)*(abc){2,3}").unwrap(); - let input_automaton = input_regex.to_automaton().unwrap(); - - c.bench_function("to_regex", |b| { - b.iter(|| to_regex(black_box(&input_automaton))) - }); - } - - { - let input_regex = RegularExpression::new( - "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q)", - ) - .unwrap(); - let input_automaton = input_regex.to_automaton().unwrap(); - - c.bench_function("determinize", |b| { - b.iter(|| determinize(black_box(&input_automaton))) - }); - } - - /*{ - let input_regex = RegularExpression::new("((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,5}").unwrap(); - let input_automaton = input_regex.to_automaton().unwrap(); - - c.bench_function("test_determinize", |b| { - b.iter(|| determinize(black_box(&input_automaton))) - }); - }*/ - - { - let automaton1 = RegularExpression::new("a(bcfe|bcdg|mkv)*(abc){1,3}") - .unwrap() - .to_automaton().unwrap(); - let automaton2 = RegularExpression::new("a(bcfe|mkv|opr)*(abc){2,4}") - .unwrap() - .to_automaton().unwrap(); - - c.bench_function("intersection", |b| { - b.iter(|| intersection(black_box(&automaton1), black_box(&automaton2))) - }); - } - - { - let automaton = RegularExpression::new("a(bcfe|bcdg|mkv)*(abc){1,3}") - .unwrap() - .to_automaton().unwrap(); - - c.bench_function("generate_strings", |b| { - b.iter(|| generate_strings(black_box(&automaton))) - }); - } -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/benches/operations.rs b/benches/operations.rs new file mode 100644 index 0000000..4526716 --- /dev/null +++ b/benches/operations.rs @@ -0,0 +1,324 @@ +//! Benchmarks covering the main operation families of the library. +//! +//! Inputs come in named sizes so numbers stay comparable across versions: +//! +//! * `small` / `medium` / `large` — realistic patterns of increasing size. +//! * `blowup_N` — the classic `(a|b)*a(a|b){N}` family whose minimal DFA has +//! 2^N states: the worst case of subset construction. +//! +//! Mutating operations (`minimize`, `complement`) are measured with +//! `iter_batched` on a fresh clone per iteration, so flag short-circuits +//! (e.g. `minimize` early-returning on an already-minimal automaton) don't +//! skew the numbers. + +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use regex_charclass::char::Char; +use regexsolver::fast_automaton::{CharacterOrder, FastAutomaton, GenerationOptions, PathOrder}; +use regexsolver::regex::RegularExpression; +use regexsolver::{CharRange, Term}; +use std::hint::black_box; + +const SMALL: (&str, &str) = ("small", "(abc|de){2}"); +const MEDIUM: (&str, &str) = ("medium", "a(bcfe|bcdg|mkv)*(abc){2,3}(abc){2}"); +const LARGE: (&str, &str) = ( + "large", + "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q)", +); + +fn automaton(pattern: &str) -> FastAutomaton { + RegularExpression::new(pattern) + .unwrap() + .to_automaton() + .unwrap() +} + +fn dfa(pattern: &str) -> FastAutomaton { + automaton(pattern).determinize().unwrap().into_owned() +} + +/// `(a|b)*a(a|b){n}`: an n+2-state NFA whose minimal DFA has 2^(n+1) states. +fn blowup_pattern(n: usize) -> String { + format!("(a|b)*a(a|b){{{n}}}") +} + +fn bench_parse(c: &mut Criterion) { + let mut group = c.benchmark_group("parse"); + for (name, pattern) in [SMALL, MEDIUM, LARGE] { + group.bench_with_input(BenchmarkId::from_parameter(name), pattern, |b, pattern| { + b.iter(|| RegularExpression::new(black_box(pattern)).unwrap()) + }); + } + group.finish(); +} + +fn bench_to_automaton(c: &mut Criterion) { + let mut group = c.benchmark_group("to_automaton"); + for (name, pattern) in [SMALL, MEDIUM, LARGE] { + let regex = RegularExpression::new(pattern).unwrap(); + group.bench_with_input(BenchmarkId::from_parameter(name), ®ex, |b, regex| { + b.iter(|| black_box(regex).to_automaton().unwrap()) + }); + } + group.finish(); +} + +fn bench_determinize(c: &mut Criterion) { + let mut group = c.benchmark_group("determinize"); + for n in [5, 10] { + let nfa = automaton(&blowup_pattern(n)); + group.bench_with_input(BenchmarkId::new("blowup", n), &nfa, |b, nfa| { + b.iter(|| black_box(nfa).determinize().unwrap().into_owned()) + }); + } + let nfa = automaton(LARGE.1); + group.bench_with_input(BenchmarkId::from_parameter("large"), &nfa, |b, nfa| { + b.iter(|| black_box(nfa).determinize().unwrap().into_owned()) + }); + group.finish(); +} + +fn bench_minimize(c: &mut Criterion) { + let mut group = c.benchmark_group("minimize"); + for n in [5, 10] { + let blowup_dfa = dfa(&blowup_pattern(n)); + group.bench_with_input(BenchmarkId::new("blowup", n), &blowup_dfa, |b, dfa| { + b.iter_batched( + || dfa.clone(), + |mut automaton| { + automaton.minimize().unwrap(); + automaton + }, + BatchSize::SmallInput, + ) + }); + } + let large_dfa = dfa(LARGE.1); + group.bench_with_input( + BenchmarkId::from_parameter("large"), + &large_dfa, + |b, dfa| { + b.iter_batched( + || dfa.clone(), + |mut automaton| { + automaton.minimize().unwrap(); + automaton + }, + BatchSize::SmallInput, + ) + }, + ); + group.finish(); +} + +fn bench_set_operations(c: &mut Criterion) { + let mut group = c.benchmark_group("set_operations"); + + let a = automaton("a(bcfe|bcdg|mkv)*(abc){1,3}"); + let b_op = automaton("a(bcfe|mkv|opr)*(abc){2,4}"); + group.bench_function("intersection", |b| { + b.iter(|| black_box(&a).intersection(black_box(&b_op)).unwrap()) + }); + group.bench_function("union", |b| { + b.iter(|| black_box(&a).union(black_box(&b_op)).unwrap()) + }); + + let minuend = automaton(".*abc.*"); + let subtrahend = automaton(".*def.*"); + group.bench_function("difference", |b| { + b.iter(|| { + black_box(&minuend) + .difference(black_box(&subtrahend)) + .unwrap() + }) + }); + + let complement_input = dfa(".*abc.*"); + group.bench_function("complement", |b| { + b.iter_batched( + || complement_input.clone(), + |mut automaton| { + automaton.complement().unwrap(); + automaton + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +fn bench_decision(c: &mut Criterion) { + let mut group = c.benchmark_group("decision"); + + // Same language, structurally different automata: the `self == other` + // shortcut cannot fire, forcing the full check in both directions. + let left_form = automaton("(a|b)*abc(a|b)*"); + let right_form = automaton("(a*b*)*abc(b*a*)*"); + assert_ne!(left_form, right_form); + assert!(left_form.equivalent(&right_form).unwrap()); + group.bench_function("equivalent", |b| { + b.iter(|| { + black_box(&left_form) + .equivalent(black_box(&right_form)) + .unwrap() + }) + }); + + let smaller = automaton("abc(de|fg){1,3}"); + let bigger = automaton("abc.*"); + group.bench_function("subset", |b| { + b.iter(|| black_box(&smaller).subset(black_box(&bigger)).unwrap()) + }); + + let left = automaton(".*abc.*"); + let right = automaton(".*cba.*"); + group.bench_function("has_intersection", |b| { + b.iter(|| { + black_box(&left) + .has_intersection(black_box(&right)) + .unwrap() + }) + }); + + group.finish(); +} + +fn bench_analyze(c: &mut Criterion) { + let mut group = c.benchmark_group("analyze"); + + let finite = dfa("[a-z]{1,6}"); + group.bench_function("length/finite", |b| b.iter(|| black_box(&finite).length())); + group.bench_function("cardinality/finite", |b| { + b.iter(|| black_box(&finite).cardinality().unwrap()) + }); + + let infinite = automaton(LARGE.1); + group.bench_function("length/large", |b| b.iter(|| black_box(&infinite).length())); + + group.finish(); +} + +fn bench_to_regex(c: &mut Criterion) { + let mut group = c.benchmark_group("to_regex"); + + let nfa = automaton(MEDIUM.1); + group.bench_function("nfa", |b| b.iter(|| black_box(&nfa).to_regex())); + + let medium_dfa = dfa(MEDIUM.1); + group.bench_function("dfa", |b| b.iter(|| black_box(&medium_dfa).to_regex())); + + group.finish(); +} + +fn bench_generate_strings(c: &mut Criterion) { + let mut group = c.benchmark_group("generate_strings"); + + let automaton = dfa("[a-z]{1,4}"); + group.bench_function("first_2000", |b| { + b.iter(|| { + black_box(&automaton) + .generate_strings(2000, 0, PathOrder::Sweep) + .unwrap() + }) + }); + + // The offset fast-skips whole subtrees by counting paths. + let deep = dfa("[a-z]{1,10}"); + group.bench_function("deep_offset", |b| { + b.iter(|| { + black_box(&deep) + .generate_strings(100, 1_000_000, PathOrder::Sweep) + .unwrap() + }) + }); + + // Interleaving walks the automaton once per pass instead of settling on + // one path, so it pays for the paths it spreads over. + group.bench_function("interleave_2000", |b| { + b.iter(|| { + black_box(&automaton) + .generate_strings(2000, 0, PathOrder::Interleave) + .unwrap() + }) + }); + + // Shuffling adds a Feistel permutation per string and a seeded tie-break + // per queued path on top of that. + group.bench_function("shuffled_2000", |b| { + b.iter(|| { + black_box(&automaton) + .generate_strings(2000, 0, (PathOrder::Shuffled, CharacterOrder::Shuffled)) + .unwrap() + }) + }); + + // A charset costs one intersection per transition condition, up front. + let printable = CharRange::new_from_range(Char::new(' ')..=Char::new('~')); + let options = GenerationOptions::from(PathOrder::Sweep).with_charset(printable); + group.bench_function("charset_2000", |b| { + b.iter(|| { + black_box(&automaton) + .generate_strings(2000, 0, options.clone()) + .unwrap() + }) + }); + + group.finish(); +} + +fn bench_construction(c: &mut Criterion) { + let mut group = c.benchmark_group("construction"); + + // A 64-transition chain over a growing alphabet: every few transitions + // extend the spanning set and re-project the existing conditions. + group.bench_function("add_transition_from_range/chain_64", |b| { + b.iter(|| { + let mut automaton = FastAutomaton::new_empty(); + let mut previous = 0; + for i in 0..64u8 { + let next = automaton.new_state(); + let character = Char::new(char::from(b'a' + (i % 26))); + let range = CharRange::new_from_range(character..=character); + automaton + .add_transition_from_range(previous, next, &range) + .unwrap(); + previous = next; + } + automaton.accept(previous); + automaton + }) + }); + + group.finish(); +} + +fn bench_end_to_end(c: &mut Criterion) { + let mut group = c.benchmark_group("end_to_end"); + + // The front-page scenario: parse two patterns, intersect, print back. + group.bench_function("intersection_to_pattern", |b| { + b.iter(|| { + let a = Term::from_pattern(black_box("(ab|xy){2}")).unwrap(); + let b_term = Term::from_pattern(black_box(".*xy")).unwrap(); + a.intersection(&[b_term]).unwrap().to_pattern() + }) + }); + + group.finish(); +} + +criterion_group!( + benches, + bench_parse, + bench_to_automaton, + bench_determinize, + bench_minimize, + bench_set_operations, + bench_decision, + bench_analyze, + bench_to_regex, + bench_generate_strings, + bench_construction, + bench_end_to_end, +); +criterion_main!(benches); diff --git a/examples/generate.rs b/examples/generate.rs new file mode 100644 index 0000000..6dd5760 --- /dev/null +++ b/examples/generate.rs @@ -0,0 +1,61 @@ +//! Generate strings matching a regex pattern. +//! +//! ```text +//! cargo run --example generate -- "[a-z]{2}[0-9]" 20 +//! cargo run --example generate -- "[a-z]{2}[0-9]" 20 interleave +//! cargo run --example generate -- "[A-Z][a-z]+ [0-9]{4}" 20 shuffled +//! cargo run --example generate -- ".{4}" 20 interleave "[ -~]" +//! ``` + +use regexsolver::regex::RegularExpression; +use regexsolver::{ + Term, + fast_automaton::{CharacterOrder, GenerationOptions, PathOrder}, +}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let Some(pattern) = args.next() else { + eprintln!( + "Usage: cargo run --example generate -- [count] [sweep|interleave|shuffled] [charset]" + ); + std::process::exit(2); + }; + let count: usize = args.next().map(|c| c.parse()).transpose()?.unwrap_or(10); + + // `sweep` walks the language in order, `interleave` spreads the strings + // over the shapes the pattern allows, and `shuffled` draws both the + // shapes and their characters by seed (see `PathOrder`/`CharacterOrder`). + let axes = match args.next().as_deref() { + None | Some("sweep") => (PathOrder::Sweep, CharacterOrder::Ascending), + Some("interleave") => (PathOrder::Interleave, CharacterOrder::Ascending), + Some("shuffled") => (PathOrder::Shuffled, CharacterOrder::Shuffled), + Some(other) => { + eprintln!("Unknown order {other:?}, expected `sweep`, `interleave` or `shuffled`"); + std::process::exit(2); + } + }; + + let mut options = GenerationOptions::from(axes); + + // A charset is a plain character class, e.g. "[ -~]" for printable ASCII: + // only the strings made entirely of its characters are generated. + if let Some(charset) = args.next() { + match RegularExpression::new(&charset)? { + RegularExpression::Character(charset) => options = options.with_charset(charset), + _ => { + eprintln!("The charset has to be a single character class, e.g. \"[ -~]\""); + std::process::exit(2); + } + } + } + + // Minimize once: pagination over the same minimized term yields + // disjoint, consistent pages (see `Term::generate_strings`). + let term = Term::from_pattern(&pattern)?.minimize()?; + for string in term.generate_strings(count, 0, options)? { + println!("{string:?}"); + } + + Ok(()) +} diff --git a/examples/relate.rs b/examples/relate.rs new file mode 100644 index 0000000..1adb6d8 --- /dev/null +++ b/examples/relate.rs @@ -0,0 +1,54 @@ +//! Explore how two regex patterns relate as languages. +//! +//! ```text +//! cargo run --example relate -- "(abc|de){2}" ".*xy" +//! ``` + +use regexsolver::{Term, fast_automaton::PathOrder}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let (Some(a), Some(b)) = (args.next(), args.next()) else { + eprintln!("Usage: cargo run --example relate -- "); + std::process::exit(2); + }; + + let a_term = Term::from_pattern(&a)?; + let b_term = Term::from_pattern(&b)?; + + println!("a = {a}"); + println!("b = {b}"); + println!(); + + if a_term.equivalent(&b_term)? { + println!("a and b match exactly the same strings."); + return Ok(()); + } + println!("equivalent: no"); + println!("a subset of b: {}", a_term.subset(&b_term)?); + println!("b subset of a: {}", b_term.subset(&a_term)?); + println!(); + + let intersection = a_term.intersection([&b_term])?; + if intersection.is_empty()? { + println!("a ∩ b = [] (no string matches both)"); + } else { + println!("a ∩ b = {}", intersection.to_pattern()?); + println!( + " e.g. {:?}", + intersection.generate_strings(5, 0, PathOrder::Interleave)? + ); + } + + let pattern_or_empty = |term: Term| -> Result> { + Ok(if term.is_empty()? { + "[]".to_string() + } else { + term.to_pattern()? + }) + }; + println!("a - b = {}", pattern_or_empty(a_term.difference(&b_term)?)?); + println!("b - a = {}", pattern_or_empty(b_term.difference(&a_term)?)?); + + Ok(()) +} diff --git a/proptest-regressions/regex/analyze/number_of_states.txt b/proptest-regressions/regex/analyze/number_of_states.txt new file mode 100644 index 0000000..205171e --- /dev/null +++ b/proptest-regressions/regex/analyze/number_of_states.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc c974b0759317710e3901bbc0b13a84ff1cd218b3abbaae98f4fe0ab7bc03505a # shrinks to regex = Alternation([Character(RangeSet([Char('a'), Char('a')])), Concat([Repetition(Character(RangeSet([Char('a'), Char('a')])), 0, None), Character(RangeSet([Char('a'), Char('a')]))])]) diff --git a/src/cardinality/mod.rs b/src/cardinality/mod.rs index 08131e0..d2e054c 100644 --- a/src/cardinality/mod.rs +++ b/src/cardinality/mod.rs @@ -1,10 +1,5 @@ -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; - -/// Represent a number. -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +/// Represents a cardinality: either a specific integer, a number too large to represent exactly, or infinite. #[derive(PartialEq, Eq, Debug, Clone)] -#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))] pub enum Cardinality { /// An infinite number. Infinite, diff --git a/src/error/mod.rs b/src/error/mod.rs index 6447ebe..75b5e08 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -1,62 +1,73 @@ use std::fmt::{self}; -use crate::tokenizer::token::TokenError; - /// An error thrown by the engine. -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] pub enum EngineError { /// Invalid character used in regex. InvalidCharacterInRegex, /// The operation took too much time. OperationTimeOutError, - /// The given automaton should be deterministic. - AutomatonShouldBeDeterministic, /// The automaton has too many states. AutomatonHasTooManyStates, - /// The regular expression can not be parsed. + /// The regular expression cannot be parsed. RegexSyntaxError(String), - /// Too many terms are used in the operation. - TooMuchTerms(usize, usize), - /// The provided range can not be built from the spanning set. + /// The provided range cannot be built from the spanning set. ConditionInvalidRange, - /// The provided index is out of bound of the condition. - ConditionIndexOutOfBound, - /// There is an error with one of the token. - TokenError(TokenError), + /// The repetition bounds are invalid: the maximum is below the minimum. + InvalidRepetitionBounds(u32, u32), + /// The condition does not match the spanning set it is evaluated against. + IncompatibleSpanningSet, + /// The operation requires a deterministic automaton, and implicit + /// determinization is disabled by the execution profile. + DeterministicAutomatonRequired, + /// The pattern uses a regex feature the engine cannot represent (an + /// unsupported anchor/boundary position, or inline flags). The string + /// describes the specific feature. + UnsupportedRegexFeature(String), + /// A directly-constructed [`RegularExpression`](crate::regex::RegularExpression) + /// tree nests deeper than the engine converts safely (the payload is the + /// limit). Parsed patterns never hit this; it only guards against + /// stack-overflowing on pathologically deep hand-built trees. + RegexTooDeeplyNested(usize), } impl fmt::Display for EngineError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - EngineError::InvalidCharacterInRegex => write!(f, "Invalid character used in regex."), - EngineError::OperationTimeOutError => write!(f, "The operation took too much time."), - EngineError::AutomatonShouldBeDeterministic => write!(f, "The given automaton should be deterministic."), - EngineError::AutomatonHasTooManyStates => write!(f, "The automaton has too many states."), - EngineError::RegexSyntaxError(err) => write!(f, "{err}."), - EngineError::TooMuchTerms(max, got) => write!(f, "Too many terms are used in this operation, the maximum allowed for your plan is {max} and you used {got}."), - EngineError::TokenError(err) => write!(f, "{err}."), - EngineError::ConditionInvalidRange => write!(f, "The provided range can not be built from the spanning set."), - EngineError::ConditionIndexOutOfBound => write!(f, "The provided index is out of bound of the condition."), + EngineError::InvalidCharacterInRegex => { + write!(f, "invalid character used in regex") + } + EngineError::OperationTimeOutError => write!(f, "the operation timed out"), + EngineError::AutomatonHasTooManyStates => { + write!(f, "the automaton has too many states") + } + EngineError::RegexSyntaxError(err) => write!(f, "invalid regex syntax: {err}"), + EngineError::ConditionInvalidRange => write!( + f, + "the provided range cannot be built from the spanning set" + ), + EngineError::InvalidRepetitionBounds(min, max) => write!( + f, + "the repetition maximum ({max}) is below its minimum ({min})" + ), + EngineError::IncompatibleSpanningSet => write!( + f, + "the condition does not match the spanning set it is evaluated against" + ), + EngineError::DeterministicAutomatonRequired => write!( + f, + "the operation requires a deterministic automaton, and implicit determinization is disabled by the execution profile" + ), + EngineError::UnsupportedRegexFeature(feature) => { + write!(f, "unsupported regex feature: {feature}") + } + EngineError::RegexTooDeeplyNested(limit) => write!( + f, + "the regular expression is nested more than {limit} levels deep" + ), } } } impl std::error::Error for EngineError {} - -impl EngineError { - /// Determine if the error is a server error. - /// A server error should not be shown to the end user. - pub fn is_server_error(&self) -> bool { - match self { - EngineError::InvalidCharacterInRegex => false, - EngineError::OperationTimeOutError => false, - EngineError::AutomatonShouldBeDeterministic => true, - EngineError::AutomatonHasTooManyStates => false, - EngineError::RegexSyntaxError(_) => false, - EngineError::TooMuchTerms(_, _) => false, - EngineError::TokenError(_) => false, - EngineError::ConditionInvalidRange => true, - EngineError::ConditionIndexOutOfBound => true, - } - } -} diff --git a/src/execution_profile.rs b/src/execution_profile.rs index 2ae8e2b..c5e5ac1 100644 --- a/src/execution_profile.rs +++ b/src/execution_profile.rs @@ -1,105 +1,141 @@ -use std::{cell::RefCell, time::SystemTime}; +use std::{ + cell::RefCell, + time::{Duration, Instant}, +}; use crate::error::EngineError; -/// Hold settings about limitations and constraints of operations execution within the engine. +/// Holds settings that constrain how operations execute within the engine. /// -/// To apply the settings on the current thread you need to call the following function: -/// ``` -/// use regexsolver::execution_profile::{ExecutionProfile, ThreadLocalParams}; -/// -/// let execution_profile = ExecutionProfile { -/// max_number_of_states: 1, -/// start_execution_time: None, -/// execution_timeout: 1000, -/// max_number_of_terms: 10, -/// }; -/// -/// // Store the settings on the current thread. -/// ThreadLocalParams::init_profile(&execution_profile); -/// ``` -/// -/// # Examples: +/// # Examples /// /// ## Limiting the number of states /// ``` -/// use regexsolver::{Term, execution_profile::{ExecutionProfile, ThreadLocalParams}, error::EngineError}; +/// use regexsolver::{Term, execution_profile::{ExecutionProfile, ExecutionProfileBuilder}, error::EngineError, fast_automaton::GenerationOptions}; /// -/// let term1 = Term::from_regex(".*abc.*").unwrap(); -/// let term2 = Term::from_regex(".*def.*").unwrap(); +/// let term1 = Term::from_pattern(".*abcdef.*").unwrap(); +/// let term2 = Term::from_pattern(".*defabc.*").unwrap(); /// -/// let execution_profile = ExecutionProfile { -/// max_number_of_states: 1, -/// start_execution_time: None, -/// execution_timeout: 1000, -/// max_number_of_terms: 10, -/// }; -/// ThreadLocalParams::init_profile(&execution_profile); +/// let execution_profile = ExecutionProfileBuilder::new() +/// .max_number_of_states(5) +/// .build(); /// -/// assert_eq!(EngineError::AutomatonHasTooManyStates, term1.intersection(&[term2]).unwrap_err()); +/// execution_profile.run(|| { +/// assert_eq!(EngineError::AutomatonHasTooManyStates, term1.intersection(&[term2]).unwrap_err()); +/// }); /// ``` /// -/// ## Limiting the number of terms +/// ## Limiting the execution time /// ``` -/// use regexsolver::{Term, execution_profile::{ExecutionProfile, ThreadLocalParams}, error::EngineError}; +/// use regexsolver::{Term, execution_profile::{ExecutionProfile, ExecutionProfileBuilder}, error::EngineError, fast_automaton::GenerationOptions}; /// -/// let term1 = Term::from_regex(".*abc.*").unwrap(); -/// let term2 = Term::from_regex(".*def.*").unwrap(); -/// let term3 = Term::from_regex(".*hij.*").unwrap(); +/// let term = Term::from_pattern(".*abc.*cdef.*sqdsqf.*").unwrap(); /// -/// let execution_profile = ExecutionProfile { -/// max_number_of_states: 8192, -/// start_execution_time: None, -/// execution_timeout: 1000, -/// max_number_of_terms: 2, -/// }; -/// ThreadLocalParams::init_profile(&execution_profile); +/// let execution_profile = ExecutionProfileBuilder::new() +/// .execution_timeout(5) // 5ms +/// .build(); /// -/// assert_eq!(EngineError::TooMuchTerms(2,3), term1.intersection(&[term2, term3]).unwrap_err()); +/// execution_profile.run(|| { +/// assert_eq!(EngineError::OperationTimeOutError, term.generate_strings(100_000_000, 0, GenerationOptions::new()).unwrap_err()); +/// }); /// ``` /// -/// ## Limiting the execution time +/// ## Disabling implicit determinization +/// +/// [`FastAutomaton`](crate::fast_automaton::FastAutomaton) operations that +/// require a deterministic automaton (`minimize`, `complement`, +/// `difference`, `equivalent`, `subset`, `cardinality`, ...) +/// determinize a non-deterministic input on their own by default. Since +/// subset construction can blow up exponentially, this can be disabled; +/// those operations then fail fast and determinization only happens through +/// an explicit `determinize()` call. [`Term`](crate::Term) methods are not +/// affected: that layer manages the underlying representation itself, so +/// its determinizations count as explicit. +/// /// ``` -/// use regexsolver::{Term, execution_profile::{ExecutionProfile, ThreadLocalParams}, error::EngineError}; -/// use std::time::SystemTime; +/// use regexsolver::CharRange; +/// use regexsolver::fast_automaton::FastAutomaton; +/// use regexsolver::execution_profile::ExecutionProfileBuilder; +/// use regexsolver::error::EngineError; +/// +/// // Two overlapping transitions from the start state: non-deterministic. +/// let mut nfa = FastAutomaton::new_empty(); +/// let s1 = nfa.new_state(); +/// let s2 = nfa.new_state(); +/// nfa.add_transition_from_range(0, s1, &CharRange::total()).unwrap(); +/// nfa.add_transition_from_range(0, s2, &CharRange::total()).unwrap(); +/// nfa.accept(s1); /// -/// let term = Term::from_regex(".*abc.*cdef.*sqdsqf.*").unwrap(); +/// let execution_profile = ExecutionProfileBuilder::new() +/// .implicit_determinization(false) +/// .build(); /// -/// let execution_profile = ExecutionProfile { -/// max_number_of_states: 8192, -/// start_execution_time: Some(SystemTime::now()), -/// execution_timeout: 1, -/// max_number_of_terms: 50, -/// }; -/// ThreadLocalParams::init_profile(&execution_profile); +/// execution_profile.run(|| { +/// // `minimize` requires a DFA and refuses to determinize on its own. +/// assert_eq!( +/// EngineError::DeterministicAutomatonRequired, +/// nfa.clone().minimize().unwrap_err() +/// ); /// -/// assert_eq!(EngineError::OperationTimeOutError, term.generate_strings(100).unwrap_err()); +/// // Determinizing explicitly is always allowed. +/// let mut dfa = nfa.determinize().unwrap().into_owned(); +/// assert!(dfa.minimize().is_ok()); +/// }); /// ``` +#[derive(Clone, Debug)] pub struct ExecutionProfile { /// The maximum number of states that a non-determinitic finite automaton can hold, this is checked during the convertion of regular expression to automaton. - pub max_number_of_states: usize, - /// Timestamp of when the execution has started, if this value is not set the operations will never timeout. - pub start_execution_time: Option, + max_number_of_states: Option, /// The longest time in milliseconds that an operation execution can last, there are no guaranties that the exact time will be respected. - pub execution_timeout: u128, - /// The maximum number of terms that an operation can have. - pub max_number_of_terms: usize, + execution_timeout: Option, + /// The time after when a [`EngineError::OperationTimeOutError`] should be thrown. + execution_deadline: Option, + /// Whether [`FastAutomaton`](crate::fast_automaton::FastAutomaton) + /// operations that require a deterministic automaton may determinize a + /// non-deterministic input on their own (the default). When `false`, + /// those operations return + /// [`EngineError::DeterministicAutomatonRequired`] instead, so that the + /// potentially exponential subset construction only ever happens through + /// an explicit `determinize()` call. [`Term`](crate::Term) methods + /// always work: that layer manages the representation itself. + implicit_determinization: bool, +} + +/// Equality compares the *configuration* (state limit, timeout, implicit +/// determinization) and deliberately ignores `execution_deadline`: two +/// profiles built alike compare equal whether or not one is currently +/// installed and running. +impl PartialEq for ExecutionProfile { + fn eq(&self, other: &ExecutionProfile) -> bool { + self.max_number_of_states == other.max_number_of_states + && self.execution_timeout == other.execution_timeout + && self.implicit_determinization == other.implicit_determinization + } } impl ExecutionProfile { + /// Retrieves the current thread-local execution profile. + pub fn get() -> ExecutionProfile { + ThreadLocalParams::get_execution_profile() + } + + /// Whether a execution deadline is configured. When it is not, + /// [`assert_not_timed_out`](Self::assert_not_timed_out) can be + /// skipped entirely instead of being computed for a check that + /// cannot fail. + #[inline] + pub fn limits_execution_time(&self) -> bool { + self.execution_deadline.is_some() + } + /// Assert that `execution_timeout` is not exceeded. /// - /// Return empty if `execution_timeout` is not exceeded or if `start_execution_time` is not set. + /// Return empty if `execution_timeout` is not exceeded. /// /// Return [`EngineError::OperationTimeOutError`] otherwise. pub fn assert_not_timed_out(&self) -> Result<(), EngineError> { - if let Some(start) = self.start_execution_time { - let run_duration = SystemTime::now() - .duration_since(start) - .expect("Time went backwards") - .as_millis(); - - if run_duration > self.execution_timeout { + if let Some(execution_deadline) = self.execution_deadline { + if Instant::now() > execution_deadline { Err(EngineError::OperationTimeOutError) } else { Ok(()) @@ -108,186 +144,524 @@ impl ExecutionProfile { Ok(()) } } + + /// Whether a maximum number of states is configured. When it is not, the + /// state-count heuristics feeding + /// [`assert_max_number_of_states`](Self::assert_max_number_of_states) can + /// be skipped entirely instead of being computed for a check that cannot + /// fail. + #[inline] + pub fn limits_number_of_states(&self) -> bool { + self.max_number_of_states.is_some() + } + + /// Assert that `max_number_of_states` is not exceeded. + /// + /// `max_number_of_states` is the largest number of states an automaton may + /// hold, so `number_of_states == max_number_of_states` is allowed and only + /// strictly exceeding it returns [`EngineError::AutomatonHasTooManyStates`]. + pub fn assert_max_number_of_states(&self, number_of_states: usize) -> Result<(), EngineError> { + if let Some(max_number_of_states) = self.max_number_of_states + && number_of_states > max_number_of_states + { + return Err(EngineError::AutomatonHasTooManyStates); + } + Ok(()) + } + + /// Assert that implicit determinization is allowed. + /// + /// Return empty if it is. + /// + /// Return [`EngineError::DeterministicAutomatonRequired`] otherwise. + pub fn assert_implicit_determinization_allowed(&self) -> Result<(), EngineError> { + if self.implicit_determinization { + Ok(()) + } else { + Err(EngineError::DeterministicAutomatonRequired) + } + } + + /// Returns a copy of this profile with the execution timeout set to + /// `execution_timeout_in_ms` milliseconds. Use these `with_*` methods to + /// derive a variant of an existing profile (e.g. one from + /// [`get`](Self::get)); to build one from scratch, prefer + /// [`ExecutionProfileBuilder`]. See + /// [`ExecutionProfileBuilder::execution_timeout`]. + pub fn with_execution_timeout(mut self, execution_timeout_in_ms: u64) -> Self { + self.execution_timeout = Some(execution_timeout_in_ms); + self + } + + /// Returns a copy of this profile with the maximum number of states set to + /// `max_number_of_states`. See + /// [`ExecutionProfileBuilder::max_number_of_states`]. + pub fn with_max_number_of_states(mut self, max_number_of_states: usize) -> Self { + self.max_number_of_states = Some(max_number_of_states); + self + } + + /// Returns a copy of this profile with implicit determinization enabled or + /// disabled. See [`ExecutionProfileBuilder::implicit_determinization`]. + pub fn with_implicit_determinization(mut self, allowed: bool) -> Self { + self.implicit_determinization = allowed; + self + } + + /// Runs the given closure with this profile installed for the current thread, setting its start time to now. + pub fn run(&self, f: F) -> R + where + F: FnOnce() -> R, + { + let _guard = ProfileRestoreGuard::install(); + + let mut execution_profile = self.clone(); + if let Some(execution_timeout) = execution_profile.execution_timeout { + // `Instant + Duration` overflow behavior is platform-dependent; a + // timeout so large the deadline is unrepresentable is equivalent + // to no deadline at all. + execution_profile.execution_deadline = + Instant::now().checked_add(Duration::from_millis(execution_timeout)); + } + + ThreadLocalParams::set_execution_profile(&execution_profile); + f() + } + + /// Runs the closure like [`run`](Self::run), but does not reset the start time. Use this to propagate an already-started profile to worker threads without restarting the clock. + pub fn apply(&self, f: F) -> R + where + F: FnOnce() -> R, + { + let _guard = ProfileRestoreGuard::install(); + + ThreadLocalParams::set_execution_profile(self); + f() + } +} + +/// Restores the thread-local execution profile captured at construction when +/// dropped, including on panic unwind. Keeps [`ExecutionProfile::run`] and +/// [`ExecutionProfile::apply`] panic-safe so a panicking closure cannot leak a +/// temporary profile onto a (possibly pooled) thread. +struct ProfileRestoreGuard { + previous: ExecutionProfile, +} + +impl ProfileRestoreGuard { + fn install() -> Self { + ProfileRestoreGuard { + previous: ThreadLocalParams::get_execution_profile(), + } + } } -/// Hold [`ExecutionProfile`] on the current thread. -/// -/// The default [`ExecutionProfile`] is the following: -/// ``` -/// use regexsolver::execution_profile::ExecutionProfile; -/// -/// ExecutionProfile { -/// max_number_of_states: 8192, -/// start_execution_time: None, -/// execution_timeout: 1500, -/// max_number_of_terms: 50, -/// }; -/// ``` -pub struct ThreadLocalParams; +impl Drop for ProfileRestoreGuard { + fn drop(&mut self) { + ThreadLocalParams::set_execution_profile(&self.previous); + } +} + +/// Builder for an [`ExecutionProfile`]. Start from [`new`](Self::new), set the +/// limits you want, and [`build`](Self::build) the immutable profile. +#[derive(Clone, Debug)] +pub struct ExecutionProfileBuilder { + /// The maximum number of states that a non-determinitic finite automaton can hold, this is checked during the convertion of regular expression to automaton. + max_number_of_states: Option, + /// The longest time in milliseconds that an operation execution can last, there are no guaranties that the exact time will be respected. + execution_timeout: Option, + /// Whether operations requiring a deterministic automaton may determinize + /// a non-deterministic input on their own. Defaults to `true`. + implicit_determinization: bool, +} +impl Default for ExecutionProfileBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ExecutionProfileBuilder { + /// Creates a builder with no limits set and implicit determinization + /// enabled (i.e. the defaults, equivalent to the ambient profile when none + /// has been installed). + pub fn new() -> Self { + Self { + max_number_of_states: None, + execution_timeout: None, + implicit_determinization: true, + } + } + + /// Sets the longest time, in milliseconds, that an operation may run before + /// it aborts with [`EngineError::OperationTimeOutError`]. Enforcement is + /// best-effort (checked between internal steps), so the exact deadline is + /// not guaranteed. Unset by default (no timeout). + pub fn execution_timeout(mut self, execution_timeout_in_ms: u64) -> Self { + self.execution_timeout = Some(execution_timeout_in_ms); + self + } + + /// Caps the number of states an automaton may reach; operations that would + /// exceed it abort with [`EngineError::AutomatonHasTooManyStates`]. This + /// bounds the exponential blow-up of conversions such as determinization. + /// Unset by default (no cap). + pub fn max_number_of_states(mut self, max_number_of_states: usize) -> Self { + self.max_number_of_states = Some(max_number_of_states); + self + } + + /// Whether [`FastAutomaton`](crate::fast_automaton::FastAutomaton) + /// operations that require a deterministic automaton may determinize a + /// non-deterministic input on their own (the default). When set to + /// `false`, those operations return + /// [`EngineError::DeterministicAutomatonRequired`] instead; explicit + /// `determinize()` calls and [`Term`](crate::Term) methods (which + /// manage the representation themselves) are always allowed. + pub fn implicit_determinization(mut self, allowed: bool) -> Self { + self.implicit_determinization = allowed; + self + } + + /// Builds the [`ExecutionProfile`]. Install it around a unit of work with + /// [`ExecutionProfile::run`]. + pub fn build(self) -> ExecutionProfile { + ExecutionProfile { + max_number_of_states: self.max_number_of_states, + execution_timeout: self.execution_timeout, + execution_deadline: None, + implicit_determinization: self.implicit_determinization, + } + } +} + +struct ThreadLocalParams; impl ThreadLocalParams { thread_local! { - static MAX_NUMBER_OF_STATES: RefCell = const { RefCell::new(8192) }; - static START_EXECUTION_TIME: RefCell> = const { RefCell::new(None) }; - static EXECUTION_TIMEOUT: RefCell = const { RefCell::new(1500) }; - static MAX_NUMBER_OF_TERMS: RefCell = const { RefCell::new(50) }; + static MAX_NUMBER_OF_STATES: RefCell> = const { RefCell::new(None) }; + static EXECUTION_DEADLINE: RefCell> = const { RefCell::new(None) }; + static EXECUTION_TIMEOUT: RefCell> = const { RefCell::new(None) }; + static IMPLICIT_DETERMINIZATION: RefCell = const { RefCell::new(true) }; } /// Store on the current thread [`ExecutionProfile`]. - pub fn init_profile(profile: &ExecutionProfile) { + fn set_execution_profile(profile: &ExecutionProfile) { ThreadLocalParams::MAX_NUMBER_OF_STATES.with(|cell| { *cell.borrow_mut() = profile.max_number_of_states; }); - ThreadLocalParams::START_EXECUTION_TIME.with(|cell| { - *cell.borrow_mut() = profile.start_execution_time; + ThreadLocalParams::EXECUTION_DEADLINE.with(|cell| { + *cell.borrow_mut() = profile.execution_deadline; }); ThreadLocalParams::EXECUTION_TIMEOUT.with(|cell| { *cell.borrow_mut() = profile.execution_timeout; }); - ThreadLocalParams::MAX_NUMBER_OF_TERMS.with(|cell| { - *cell.borrow_mut() = profile.max_number_of_terms; + ThreadLocalParams::IMPLICIT_DETERMINIZATION.with(|cell| { + *cell.borrow_mut() = profile.implicit_determinization; }); } - pub fn get_max_number_of_states() -> usize { + fn get_max_number_of_states() -> Option { ThreadLocalParams::MAX_NUMBER_OF_STATES.with(|cell| *cell.borrow()) } - pub fn get_start_execution_time() -> Option { - ThreadLocalParams::START_EXECUTION_TIME.with(|cell| *cell.borrow()) + fn get_execution_deadline() -> Option { + ThreadLocalParams::EXECUTION_DEADLINE.with(|cell| *cell.borrow()) } - pub fn get_execution_timeout() -> u128 { + fn get_execution_timeout() -> Option { ThreadLocalParams::EXECUTION_TIMEOUT.with(|cell| *cell.borrow()) } - pub fn get_max_number_of_terms() -> usize { - ThreadLocalParams::MAX_NUMBER_OF_TERMS.with(|cell| *cell.borrow()) + fn get_implicit_determinization() -> bool { + ThreadLocalParams::IMPLICIT_DETERMINIZATION.with(|cell| *cell.borrow()) } /// Return the [`ExecutionProfile`] stored on the current thread. - pub fn get_execution_profile() -> ExecutionProfile { + fn get_execution_profile() -> ExecutionProfile { ExecutionProfile { max_number_of_states: Self::get_max_number_of_states(), - start_execution_time: Self::get_start_execution_time(), + execution_deadline: Self::get_execution_deadline(), execution_timeout: Self::get_execution_timeout(), - max_number_of_terms: Self::get_max_number_of_terms(), + implicit_determinization: Self::get_implicit_determinization(), } } } #[cfg(test)] mod tests { - use crate::{regex::RegularExpression, Term}; + use crate::{Term, fast_automaton::GenerationOptions, regex::RegularExpression}; use super::*; - #[test] - fn test_execution() -> Result<(), String> { - let execution_profile = ExecutionProfile { - max_number_of_states: 1, - start_execution_time: None, - execution_timeout: 1000, - max_number_of_terms: 10, - }; - ThreadLocalParams::init_profile(&execution_profile); - - let regex = RegularExpression::new("test").unwrap(); + fn assert_send() {} + fn assert_sync() {} - assert!(regex.to_automaton().is_err()); + // `max_number_of_states(N)` allows exactly N states and only rejects N+1, + // matching the documented "maximum an automaton may hold". + #[test] + fn max_number_of_states_allows_exactly_the_limit() { + let profile = ExecutionProfileBuilder::new() + .max_number_of_states(3) + .build(); + assert!(profile.assert_max_number_of_states(2).is_ok()); + assert!(profile.assert_max_number_of_states(3).is_ok()); assert_eq!( - EngineError::AutomatonHasTooManyStates, - regex.to_automaton().unwrap_err() + profile.assert_max_number_of_states(4).unwrap_err(), + EngineError::AutomatonHasTooManyStates ); + } + + #[test] + fn test_traits() -> Result<(), String> { + assert_send::(); + assert_sync::(); Ok(()) } + // `run`/`apply` must restore the previous thread profile even when the + // closure panics — a leaked temporary profile would permanently poison + // pooled (e.g. rayon) threads. #[test] - fn test_execution_timeout_generate_strings() -> Result<(), String> { - let term = Term::from_regex(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz").unwrap(); + fn run_restores_previous_profile_on_panic() { + let outer = ExecutionProfileBuilder::new() + .max_number_of_states(123) + .build(); + outer.run(|| { + let inner = ExecutionProfileBuilder::new() + .max_number_of_states(1) + .build(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + inner.run(|| panic!("intentional test panic")); + })); + assert!(result.is_err()); + assert_eq!(outer, ExecutionProfile::get()); + }); + } - let start_time = SystemTime::now(); - let execution_profile = ExecutionProfile { - max_number_of_states: 8192, - start_execution_time: Some(start_time), - execution_timeout: 100, - max_number_of_terms: 50, - }; - ThreadLocalParams::init_profile(&execution_profile); + #[test] + fn test_execution_get() -> Result<(), String> { + let execution_profile = ExecutionProfileBuilder::new() + .execution_timeout(1000) + .max_number_of_states(8192) + .build(); + + execution_profile.run(|| { + assert_eq!(execution_profile, ExecutionProfile::get()); + }); - assert_eq!( - EngineError::OperationTimeOutError, - term.generate_strings(100).unwrap_err() - ); + Ok(()) + } - let run_duration = SystemTime::now() - .duration_since(start_time) - .expect("Time went backwards") - .as_millis(); + #[test] + fn test_execution() -> Result<(), String> { + ExecutionProfileBuilder::new() + .max_number_of_states(1) + .build() + .run(|| { + let regex = RegularExpression::new("test").unwrap(); + + assert!(regex.to_automaton().is_err()); + assert_eq!( + EngineError::AutomatonHasTooManyStates, + regex.to_automaton().unwrap_err() + ); + }); - println!("{run_duration}"); - assert!(run_duration <= execution_profile.execution_timeout + 50); Ok(()) } + /// A two-way acyclic automaton with overlapping transitions: the + /// smallest shape that is non-deterministic and reaches the + /// determinization paths of every DFA-requiring operation. + fn nondeterministic_automaton() -> crate::fast_automaton::FastAutomaton { + use crate::fast_automaton::FastAutomaton; + use crate::fast_automaton::condition::Condition; + + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.add_transition(0, s1, &cond); + a.add_transition(0, s2, &cond); + a.accept(s1); + a.accept(s2); + assert!(!a.is_deterministic()); + a + } + #[test] - fn test_execution_timeout_difference() -> Result<(), String> { - let term1 = Term::from_regex(".*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); - let term2 = Term::from_regex(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz").unwrap(); + fn test_implicit_determinization_disabled() { + let nfa = nondeterministic_automaton(); + let dfa = nfa.determinize().unwrap().into_owned(); + + ExecutionProfileBuilder::new() + .implicit_determinization(false) + .build() + .run(|| { + let err = EngineError::DeterministicAutomatonRequired; + + // Every DFA-requiring operation refuses to determinize a + // non-deterministic input on its own... + assert_eq!(nfa.clone().minimize().unwrap_err(), err); + assert_eq!(nfa.clone().complement().unwrap_err(), err); + assert_eq!(dfa.difference(&nfa).unwrap_err(), err); + assert_eq!(nfa.equivalent(&dfa).unwrap_err(), err); + assert_eq!(dfa.subset(&nfa).unwrap_err(), err); + assert_eq!(nfa.cardinality().unwrap_err(), err); + + // ...but operations that work on NFAs directly are unaffected + // (difference only determinizes the subtrahend)... + assert!(nfa.difference(&dfa).is_ok()); + + // ...deterministic inputs keep working... + assert!(dfa.clone().minimize().is_ok()); + assert!(dfa.clone().complement().is_ok()); + assert!(dfa.cardinality().is_ok()); + assert!(dfa.equivalent(&dfa).is_ok()); + + // ...and explicit determinization is always allowed. + assert!(nfa.determinize().is_ok()); + }); + } - let start_time = SystemTime::now(); - let execution_profile = ExecutionProfile { - max_number_of_states: 8192, - start_execution_time: Some(start_time), - execution_timeout: 100, - max_number_of_terms: 50, - }; - ThreadLocalParams::init_profile(&execution_profile); + /// The `implicit_determinization` knob targets direct `FastAutomaton` + /// usage; `Term` manages the underlying representation itself, so its + /// whole public API must keep working when the knob is off. + #[test] + fn test_term_api_works_without_implicit_determinization() { + let term = Term::from_automaton(nondeterministic_automaton()); + let other = Term::from_pattern("a*").unwrap(); + + ExecutionProfileBuilder::new() + .implicit_determinization(false) + .build() + .run(|| { + // Methods that need a DFA internally determinize on Term's + // behalf (an explicit choice of the Term layer)... + assert!(term.difference(&other).is_ok()); + assert!(other.difference(&term).is_ok()); + assert!(term.complement().is_ok()); + assert!(term.equivalent(&other).is_ok()); + assert!(term.subset(&other).is_ok()); + assert!(other.subset(&term).is_ok()); + assert!(term.is_total().is_ok()); + assert!(term.cardinality().is_ok()); + assert!(term.minimize().is_ok()); + assert!( + term.generate_strings(5, 0, GenerationOptions::new()) + .is_ok() + ); + + // ...and the rest of the API never needed one. + assert!(term.concat(std::slice::from_ref(&other)).is_ok()); + assert!(term.union(std::slice::from_ref(&other)).is_ok()); + assert!(term.intersection(std::slice::from_ref(&other)).is_ok()); + assert!(term.repeat(0..=2).is_ok()); + assert!(term.is_empty().is_ok()); + assert!(term.is_empty_string().is_ok()); + let _ = term.length(); + let _ = term.to_regex(); + let _ = term.to_pattern(); + assert!(term.to_automaton().is_ok()); + + // The override is scoped: direct FastAutomaton usage stays + // gated afterwards. + assert_eq!( + nondeterministic_automaton().minimize().unwrap_err(), + EngineError::DeterministicAutomatonRequired + ); + }); + } - assert_eq!( - EngineError::OperationTimeOutError, - term1.difference(&term2).unwrap_err() - ); + #[test] + fn test_implicit_determinization_default() { + let nfa = nondeterministic_automaton(); + + // Without the profile knob the historical behavior is unchanged. + assert!(nfa.clone().minimize().is_ok()); + assert!(nfa.clone().complement().is_ok()); + assert!(nfa.cardinality().is_ok()); + assert!(nfa.equivalent(&nfa.clone()).is_ok()); + } - let run_duration = SystemTime::now() - .duration_since(start_time) - .expect("Time went backwards") - .as_millis(); + #[test] + fn test_execution_timeout_generate_strings() -> Result<(), String> { + let term = Term::from_pattern(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz").unwrap(); + + let execution_timeout_in_ms = 10; + let start_time = Instant::now(); + ExecutionProfileBuilder::new() + .execution_timeout(execution_timeout_in_ms) + .build() + .run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + term.generate_strings(100_000_000, 1_000_000, GenerationOptions::new()) + .unwrap_err() + ); + + let run_duration = Instant::now().duration_since(start_time).as_millis(); + + println!("{run_duration}"); + assert!(run_duration <= (execution_timeout_in_ms + 50) as u128); + }); - println!("{run_duration}"); - assert!(run_duration <= execution_profile.execution_timeout + 50); Ok(()) } #[test] - fn test_execution_timeout_intersection() -> Result<(), String> { - let term1 = Term::from_regex(".*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); - let term2 = Term::from_regex(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); + fn test_execution_timeout_difference() -> Result<(), String> { + let term1 = Term::from_pattern(".*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); + let term2 = Term::from_pattern(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz").unwrap(); + + let execution_timeout_in_ms = 0; + let start_time = Instant::now(); + ExecutionProfileBuilder::new() + .execution_timeout(execution_timeout_in_ms) + .build() + .run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + term1.difference(&term2).unwrap_err() + ); + + let run_duration = Instant::now().duration_since(start_time).as_millis(); + + println!("{run_duration}"); + assert!(run_duration <= (execution_timeout_in_ms + 1000) as u128); + }); - let start_time = SystemTime::now(); - let execution_profile = ExecutionProfile { - max_number_of_states: 8192, - start_execution_time: Some(start_time), - execution_timeout: 100, - max_number_of_terms: 50, - }; - ThreadLocalParams::init_profile(&execution_profile); + Ok(()) + } - assert_eq!( - EngineError::OperationTimeOutError, - term1.intersection(&[term2]).unwrap_err() - ); + /*#[test] + fn test_execution_timeout_intersection() -> Result<(), String> { + let term1 = Term::from_pattern(".*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); + let term2 = Term::from_pattern(".*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdsqd.*sqdsqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz.*abc.*def.*qdqd.*qsdsqdsqdz").unwrap(); - let run_duration = SystemTime::now() - .duration_since(start_time) - .expect("Time went backwards") - .as_millis(); + let execution_timeout_in_ms = 100; + let start_time = SystemTime::now(); + ExecutionProfileBuilder::new() + .execution_timeout(execution_timeout_in_ms) + .build() + .run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + term1.intersection(&[term2]).unwrap_err() + ); + + let run_duration = SystemTime::now() + .duration_since(start_time) + .expect("Time went backwards") + .as_millis(); + + println!("{run_duration}"); + assert!(run_duration <= execution_timeout_in_ms + 100); + }); - println!("{run_duration}"); - assert!(run_duration <= execution_profile.execution_timeout + 50); Ok(()) - } + }*/ } diff --git a/src/fast_automaton/analyze/cardinality.rs b/src/fast_automaton/analyze/cardinality.rs index 04ea226..1bf13e4 100644 --- a/src/fast_automaton/analyze/cardinality.rs +++ b/src/fast_automaton/analyze/cardinality.rs @@ -3,20 +3,51 @@ use std::hash::BuildHasherDefault; use super::*; impl FastAutomaton { - pub fn get_cardinality(&self) -> Option> { + /// Returns the cardinality of the automaton (i.e., the number of possible matched strings). + /// + /// Works on non-deterministic automata too: acyclic NFAs are determinized + /// internally (the only fallible step, subject to the + /// [`crate::execution_profile::ExecutionProfile`] budget, and rejected + /// with [`EngineError::DeterministicAutomatonRequired`] when the profile + /// disables implicit determinization). + /// + /// As in [`length`](Self::length), only cycles **on accepting + /// paths** make the count infinite: cycles among dead or unreachable + /// states don't add a single matched string. + #[tracing::instrument(level = "debug", skip_all, fields(states = self.number_of_states(), deterministic = self.is_deterministic()))] + pub fn cardinality(&self) -> Result, EngineError> { if self.is_empty() { - return Some(Cardinality::Integer(0)); - } else if self.cyclic || self.is_total() { - return Some(Cardinality::Infinite); - } else if !self.deterministic { - return None; + return Ok(Cardinality::Integer(0)); + } else if self.is_total() { + return Ok(Cardinality::Infinite); } - let topologically_sorted_states = self.topological_sorted_states(); - if topologically_sorted_states.is_none() { - return Some(Cardinality::Infinite); + // Only states on an accepting path (reachable from the start AND + // able to reach an accept) contribute strings; everything else is + // excluded from both the cycle check and the count. + let live = self.live_states(); + let relevant: IntSet = self + .forward_reachable_states() + .intersection(&live) + .copied() + .collect(); + + // A cycle among relevant states means infinitely many strings. + // `topological_sorted_states` returns `None` exactly when that + // subgraph is cyclic and needs no determinism, so this also covers + // cyclic non-deterministic inputs. + let topologically_sorted_states = match self.topological_sorted_states(&relevant) { + None => return Ok(Cardinality::Infinite), + Some(states) => states, + }; + + // The finite count below assumes deterministic (single-path) + // transitions. Determinizing an automaton with a finite language + // yields one whose relevant subgraph is acyclic too, so the + // recursion takes the deterministic path on the second call. + if !self.is_deterministic() { + return self.determinize_implicit()?.cardinality(); } - let topologically_sorted_states = topologically_sorted_states.unwrap(); let len = self.transitions.len(); let mut distances: IntMap = @@ -27,20 +58,19 @@ impl FastAutomaton { let current_distance = *distances.entry(state).or_insert(0); if let Some(to_states) = self.transitions.get(state) { for (to_state, condition) in to_states { - if let Some(distance) = current_distance.checked_mul( - condition - .get_cardinality(&self.spanning_set) - .expect("It should be possible to get the cardinality of a condition."), - ) { - if let Some(new_distance) = - distances.get(to_state).unwrap_or(&0).checked_add(distance) - { - distances.insert(*to_state, new_distance); + if !relevant.contains(to_state) { + continue; + } + let condition_cardinality = condition.cardinality(&self.spanning_set)?; + if let Some(distance) = current_distance.checked_mul(condition_cardinality) { + let slot = distances.entry(*to_state).or_insert(0); + if let Some(new_distance) = slot.checked_add(distance) { + *slot = new_distance; continue; } } - return Some(Cardinality::BigInteger); + return Ok(Cardinality::BigInteger); } } } @@ -52,22 +82,33 @@ impl FastAutomaton { temp_cardinality = add; continue; } - return Some(Cardinality::BigInteger); + return Ok(Cardinality::BigInteger); } } - Some(Cardinality::Integer(temp_cardinality)) + Ok(Cardinality::Integer(temp_cardinality)) } - fn topological_sorted_states(&self) -> Option> { - let len = self.get_number_of_states(); + /// Kahn's algorithm restricted to the `relevant` subgraph (transitions + /// with empty conditions can't be taken and are ignored). Returns `None` + /// when that subgraph contains a cycle. + fn topological_sorted_states(&self, relevant: &IntSet) -> Option> { + let len = relevant.len(); let mut in_degree: IntMap = IntMap::with_capacity_and_hasher(len, BuildHasherDefault::default()); let mut queue = VecDeque::with_capacity(len); let mut order = Vec::with_capacity(len); - for from_state in &self.transitions_vec() { - in_degree.entry(*from_state).or_insert(0); - for to_state in self.transitions_from_state_iter(from_state) { + let successors = |from_state: State| { + self.transitions_from(from_state) + .filter(|(condition, to_state)| { + !condition.is_empty() && relevant.contains(to_state) + }) + .map(|(_, to_state)| *to_state) + }; + + for &from_state in relevant { + in_degree.entry(from_state).or_insert(0); + for to_state in successors(from_state) { *in_degree.entry(to_state).or_insert(0) += 1; } } @@ -80,7 +121,7 @@ impl FastAutomaton { while let Some(from_state) = queue.pop_front() { order.push(from_state); - for to_state in self.transitions_from_state_iter(&from_state) { + for to_state in successors(from_state) { *in_degree.entry(to_state).or_default() -= 1; if in_degree[&to_state] == 0 { @@ -96,3 +137,51 @@ impl FastAutomaton { } } } + +#[cfg(test)] +mod tests { + use crate::cardinality::Cardinality; + use crate::fast_automaton::FastAutomaton; + use crate::fast_automaton::condition::Condition; + + // Only cycles on accepting paths make a language infinite: a cycle among + // dead states (that cannot reach an accept) must not turn a finite + // language's cardinality into Infinite. + #[test] + fn get_cardinality_ignores_dead_cycles() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.accept(0); + a.add_transition(0, s1, &cond); + a.add_transition(s1, s2, &cond); + a.add_transition(s2, s1, &cond); + // s1, s2 can't reach an accept → language is {""} only. + + assert_eq!(a.cardinality().unwrap(), Cardinality::Integer(1)); + } + + // `cardinality` determinizes internally, so it returns a finite count for + // an acyclic NFA (the only nondeterministic input that reaches the finite + // count; cyclic ones return Infinite earlier) rather than requiring a DFA. + #[test] + fn get_cardinality_determinizes_acyclic_nfas() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + // Two overlapping transitions from the start: nondeterministic, but + // both lead to accepting states after exactly one character. + a.add_transition(0, s1, &cond); + a.add_transition(0, s2, &cond); + a.accept(s1); + a.accept(s2); + assert!(!a.is_deterministic()); + + let cardinality = a.cardinality().unwrap(); + let expected = a.determinize().unwrap().cardinality().unwrap(); + assert_eq!(cardinality, expected); + assert!(matches!(cardinality, Cardinality::Integer(n) if n > 0)); + } +} diff --git a/src/fast_automaton/analyze/equivalence.rs b/src/fast_automaton/analyze/equivalence.rs index d81294c..387007a 100644 --- a/src/fast_automaton/analyze/equivalence.rs +++ b/src/fast_automaton/analyze/equivalence.rs @@ -3,21 +3,32 @@ use crate::error::EngineError; use super::*; impl FastAutomaton { - pub fn is_equivalent_of(&self, other: &FastAutomaton) -> Result { - if self.is_empty() != other.is_empty() && self.is_total() != other.is_total() { + /// Returns `true` if both automata accept the same language. + /// + /// Non-deterministic operands are determinized internally, unless the + /// execution profile disables implicit determinization, in which case + /// [`EngineError::DeterministicAutomatonRequired`] is returned. + #[tracing::instrument(level = "debug", skip_all, fields(self_states = self.number_of_states(), self_deterministic = self.is_deterministic(), other_states = other.number_of_states(), other_deterministic = other.is_deterministic()))] + pub fn equivalent(&self, other: &FastAutomaton) -> Result { + // `is_empty` is exact, so a mismatch proves the languages differ. + // (`is_total` must NOT be part of this fast path: it is conservative + // on non-deterministic automata — it can return `false` for an + // automaton that actually accepts every string — so an `is_total` + // mismatch alone proves nothing.) + if self.is_empty() != other.is_empty() { return Ok(false); } else if self == other { return Ok(true); } - let mut other_complement = other.determinize()?; + let mut other_complement = other.determinize_implicit()?.into_owned(); other_complement.complement()?; if self.has_intersection(&other_complement)? { return Ok(false); } - let mut self_complement = self.determinize()?; + let mut self_complement = self.determinize_implicit()?.into_owned(); self_complement.complement()?; Ok(!self_complement.has_intersection(other)?) @@ -43,26 +54,26 @@ mod tests { false, ); - let regex_1 = RegularExpression::new("cd").unwrap(); - let regex_2 = RegularExpression::new("cd").unwrap(); + let regex_1 = RegularExpression::parse("cd", false).unwrap(); + let regex_2 = RegularExpression::parse("cd", false).unwrap(); assert_equivalent(®ex_1, ®ex_2, true); - let regex_1 = RegularExpression::new("test.*other").unwrap(); - let regex_2 = RegularExpression::new("test.*othew").unwrap(); + let regex_1 = RegularExpression::parse("test.*other", false).unwrap(); + let regex_2 = RegularExpression::parse("test.*othew", false).unwrap(); assert_equivalent(®ex_1, ®ex_2, false); - let regex_1 = RegularExpression::new("test.{0,50}other").unwrap(); - let regex_2 = RegularExpression::new("test.{0,49}other").unwrap(); + let regex_1 = RegularExpression::parse("test.{0,50}other", false).unwrap(); + let regex_2 = RegularExpression::parse("test.{0,49}other", false).unwrap(); assert_equivalent(®ex_1, ®ex_2, false); - let regex_1 = RegularExpression::new("[0]").unwrap(); - let regex_2 = RegularExpression::new("[01]").unwrap(); + let regex_1 = RegularExpression::parse("[0]", false).unwrap(); + let regex_2 = RegularExpression::parse("[01]", false).unwrap(); assert_equivalent(®ex_1, ®ex_2, false); - let regex_1 = RegularExpression::new("(b+a+)*").unwrap(); - let regex_2 = RegularExpression::new("(b[a-b]*a)?").unwrap(); + let regex_1 = RegularExpression::parse("(b+a+)*", false).unwrap(); + let regex_2 = RegularExpression::parse("(b[a-b]*a)?", false).unwrap(); assert_equivalent(®ex_1, ®ex_2, true); Ok(()) @@ -71,14 +82,11 @@ mod tests { fn assert_equivalent(regex_1: &RegularExpression, regex_2: &RegularExpression, expected: bool) { println!("{regex_1} and {regex_2}"); let automaton_1 = regex_1.to_automaton().unwrap(); - assert_eq!(true, automaton_1.is_equivalent_of(&automaton_1).unwrap()); + assert!(automaton_1.equivalent(&automaton_1).unwrap()); let automaton_2 = regex_2.to_automaton().unwrap(); - assert_eq!(true, automaton_2.is_equivalent_of(&automaton_2).unwrap()); + assert!(automaton_2.equivalent(&automaton_2).unwrap()); - assert_eq!( - expected, - automaton_1.is_equivalent_of(&automaton_2).unwrap() - ); + assert_eq!(expected, automaton_1.equivalent(&automaton_2).unwrap()); } } diff --git a/src/fast_automaton/analyze/length.rs b/src/fast_automaton/analyze/length.rs index 70eccbd..f858855 100644 --- a/src/fast_automaton/analyze/length.rs +++ b/src/fast_automaton/analyze/length.rs @@ -1,67 +1,183 @@ use super::*; impl FastAutomaton { - pub fn get_length(&self) -> (Option, Option) { - if self.is_empty() { + /// Returns the minimum and maximum length of matched strings. + /// + /// Cycles are only treated as "language-extending" if they sit on an + /// accepting path. Cycles among dead states (states that can't reach any + /// accept) don't extend the language and therefore don't make the max + /// infinite. + /// + /// Runs in O(V + E): the minimum is a BFS distance; the maximum is a + /// longest path over the subgraph of states lying on accepting paths, + /// which is unbounded exactly when that subgraph has a cycle (any such + /// cycle can be pumped). + #[must_use] + pub fn length(&self) -> (Option, Option) { + // States that can reach an accept state. If the start state can't, + // the language is empty. + let live = self.live_states(); + if !live.contains(&self.start_state) { return (None, None); - } else if self.is_total(){ - return (Some(0), None); } + // BFS from the start over live states only; every state on an + // accepting path is live, so this loses no accepting path. BFS visits + // in non-decreasing depth, hence the first accept hit is the minimum. + // The visited set (reachable ∩ live) is exactly the subgraph relevant + // for the maximum. let mut min = None; - let mut is_infinite = false; - - let mut worklist = VecDeque::with_capacity(self.get_number_of_states()); - worklist.push_back((self.start_state, 0, IntSet::default())); - - while let Some(element) = worklist.pop_front() { - let state = element.0; - let length = element.1; - let mut seen = element.2; - if min.is_some() && length > min.unwrap() { - continue; - } - if self.accept_states.contains(&state) && (min.is_none() || length < min.unwrap()) { + let mut visited = IntSet::default(); + let mut worklist = VecDeque::with_capacity(self.number_of_states()); + visited.insert(self.start_state); + worklist.push_back((self.start_state, 0u32)); + while let Some((state, length)) = worklist.pop_front() { + if min.is_none() && self.accept_states.contains(&state) { min = Some(length); } - seen.insert(state); - - for to_state in self.transitions_from_state_iter(&state) { - if to_state == state || seen.contains(&to_state) { - is_infinite = true; + for (condition, to_state) in self.transitions_from(state) { + if condition.is_empty() || !live.contains(to_state) { continue; } - worklist.push_back((to_state, length + 1, seen.clone())); + if visited.insert(*to_state) { + worklist.push_back((*to_state, length + 1)); + } } } - if is_infinite || min.is_none() { - return (min, None); + // Longest path via Kahn's algorithm on the visited subgraph. In the + // acyclic case the topological order covers all visited states and + // every state's longest distance is final when it is dequeued. + let mut in_degree: IntMap = IntMap::default(); + for &from in &visited { + in_degree.entry(from).or_insert(0); + for (condition, to_state) in self.transitions_from(from) { + if condition.is_empty() || !visited.contains(to_state) { + continue; + } + *in_degree.entry(*to_state).or_insert(0) += 1; + } } - let mut max = None; - - worklist.clear(); - worklist.push_back((self.start_state, 0, IntSet::default())); + let mut queue: VecDeque = in_degree + .iter() + .filter(|&(_, °ree)| degree == 0) + .map(|(&state, _)| state) + .collect(); - while let Some(element) = worklist.pop_back() { - let state = element.0; - let length = element.1; - let mut seen = element.2; - if self.accept_states.contains(&state) && (max.is_none() || length > max.unwrap()) { - max = Some(length); + let mut longest: IntMap = IntMap::default(); + longest.insert(self.start_state, 0); + let mut max = None; + let mut processed = 0usize; + while let Some(from) = queue.pop_front() { + processed += 1; + let length = *longest.get(&from).unwrap_or(&0); + if self.accept_states.contains(&from) { + max = Some(max.map_or(length, |m: u32| m.max(length))); } - seen.insert(state); - - for to_state in self.transitions_from_state_iter(&state) { - if to_state == state || seen.contains(&to_state) { - max = None; - break; + for (condition, to_state) in self.transitions_from(from) { + if condition.is_empty() || !visited.contains(to_state) { + continue; + } + longest + .entry(*to_state) + .and_modify(|l| *l = (*l).max(length + 1)) + .or_insert(length + 1); + let degree = in_degree + .get_mut(to_state) + .expect("every visited target was counted above"); + *degree -= 1; + if *degree == 0 { + queue.push_back(*to_state); } - worklist.push_back((to_state, length + 1, seen.clone())); } } + if processed != visited.len() { + // A cycle on an accepting path: matched strings can be pumped + // arbitrarily, the maximum is unbounded. + return (min, None); + } + (min, max) } -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use crate::fast_automaton::FastAutomaton; + use crate::fast_automaton::condition::Condition; + + // A dead cycle (among states that cannot reach an accept) does not extend + // the language, so `length` must keep the max finite: only cycles in the + // live (co-reachable-from-accept) subgraph make the max unbounded. + #[test] + fn length_handles_dead_cycle() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.accept(0); + a.add_transition(0, s1, &cond); + a.add_transition(s1, s2, &cond); + a.add_transition(s2, s1, &cond); + // s1, s2 not accepting → language is {""} only. + + let (min, max) = a.length(); + assert_eq!(min, Some(0), "min length of {{\"\"}} is 0"); + assert_eq!( + max, + Some(0), + "max length of {{\"\"}} is 0; got {max:?} (cycle is dead, shouldn't extend the language)" + ); + } + + #[test] + fn length_finite_and_infinite() { + // Chain 0 -> 1 -> 2, accepts {0, 2}: min 0, max 2. + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.add_transition(0, s1, &cond); + a.add_transition(s1, s2, &cond); + a.accept(0); + a.accept(s2); + assert_eq!(a.length(), (Some(0), Some(2))); + + // Live cycle 0 <-> 1, accept {1}: min 1, max unbounded. + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.add_transition(0, s1, &cond); + a.add_transition(s1, 0, &cond); + a.accept(s1); + assert_eq!(a.length(), (Some(1), None)); + } + + // `length` must be linear in the graph size, not in the number of paths: + // a chain of `k` diamonds has 2^k paths, which path enumeration could not + // handle but the longest-path algorithm does instantly. + #[test] + fn length_linear_on_branching_dag() { + const DIAMONDS: usize = 24; + + let mut a = FastAutomaton::new_empty(); + let cond = Condition::total(a.spanning_set()); + let mut current = 0; + for _ in 0..DIAMONDS { + let upper = a.new_state(); + let lower = a.new_state(); + let next = a.new_state(); + a.add_transition(current, upper, &cond); + a.add_transition(current, lower, &cond); + a.add_transition(upper, next, &cond); + a.add_transition(lower, next, &cond); + current = next; + } + a.accept(current); + + let expected = 2 * DIAMONDS as u32; + assert_eq!(a.length(), (Some(expected), Some(expected))); + } +} diff --git a/src/fast_automaton/analyze/mod.rs b/src/fast_automaton/analyze/mod.rs index 56f0884..69d8fe5 100644 --- a/src/fast_automaton/analyze/mod.rs +++ b/src/fast_automaton/analyze/mod.rs @@ -1,5 +1,3 @@ -use std::hash::BuildHasherDefault; - use crate::{cardinality::Cardinality, error::EngineError}; use super::*; @@ -10,50 +8,175 @@ mod length; mod subset; impl FastAutomaton { - #[inline] + /// Checks if the automaton matches the empty language. + /// + /// Sound and complete: works on NFAs and non-minimal automata without + /// requiring determinization or minimization. O(V + E) worst case, with + /// O(1) fast paths for the common cases. pub fn is_empty(&self) -> bool { - self.accept_states.is_empty() + if self.accept_states.is_empty() { + return true; + } + if self.accept_states.contains(&self.start_state) { + return false; + } + if self.minimal { + // A minimal automaton with at least one accept state has a + // non-empty language (minimization prunes dead accepts). + return false; + } + + // Forward BFS from `start_state`; stop on first accept hit. + let mut visited = IntSet::default(); + let mut worklist = VecDeque::new(); + visited.insert(self.start_state); + worklist.push_back(self.start_state); + + while let Some(s) = worklist.pop_front() { + for (cond, to) in self.transitions_from(s) { + if cond.is_empty() { + continue; + } + if self.accept_states.contains(to) { + return false; + } + if visited.insert(*to) { + worklist.push_back(*to); + } + } + } + true } - #[inline] + /// Checks if the automaton matches all possible strings. + /// + /// Sound and complete for **deterministic** automata: a DFA's language + /// equals Σ\* iff every reachable state is accepting AND its outgoing + /// conditions union to Σ. For NFAs this is sound but conservative: + /// alternative paths may cover a character that no single reachable + /// state covers, so callers that need an exact answer on an NFA should + /// determinize first. + /// + /// O(V + E) plus one condition-union per outgoing transition. pub fn is_total(&self) -> bool { - if self.accept_states.contains(&self.start_state) { - if let Some(condition) = self.transitions[self.start_state].get(&self.start_state) { - return condition.is_total(); + let mut visited = IntSet::default(); + let mut worklist = VecDeque::new(); + visited.insert(self.start_state); + worklist.push_back(self.start_state); + + while let Some(s) = worklist.pop_front() { + if !self.accept_states.contains(&s) { + return false; + } + let mut covered = Condition::empty(&self.spanning_set); + for (cond, to) in self.transitions_from(s) { + if cond.is_empty() { + continue; + } + covered.union_with(cond); + if visited.insert(*to) { + worklist.push_back(*to); + } + } + if !covered.is_total() { + return false; } } - false + true } - pub fn get_reacheable_states(&self) -> IntSet { - let mut states_map: IntMap> = - IntMap::with_capacity_and_hasher(self.transitions.len(), BuildHasherDefault::default()); - for from_state in self.transitions_iter() { - for (to_state, transition) in self.transitions_from_state_enumerate_iter(&from_state) { - if transition.is_empty() { + /// Checks if the automaton only matches the empty string `""`. + /// + /// Sound and complete on any automaton (DFA or NFA): the language equals + /// `{""}` iff start is accepting AND no state reachable from start by at + /// least one non-empty transition is, or can reach, an accept state. + /// O(V + E). + pub fn is_empty_string(&self) -> bool { + if !self.accept_states.contains(&self.start_state) { + return false; + } + + let mut visited = IntSet::default(); + let mut worklist = VecDeque::new(); + + // Seed with states reachable in exactly one non-empty step from start. + for (cond, to) in self.transitions_from(self.start_state) { + if cond.is_empty() { + continue; + } + if visited.insert(*to) { + worklist.push_back(*to); + } + } + + while let Some(s) = worklist.pop_front() { + if self.accept_states.contains(&s) { + return false; + } + for (cond, to) in self.transitions_from(s) { + if cond.is_empty() { continue; } - match states_map.entry(*to_state) { - Entry::Occupied(mut o) => o.get_mut().insert(from_state), - Entry::Vacant(v) => { - let mut new_states = IntSet::default(); - new_states.insert(from_state); - v.insert(new_states); - true - } - }; + if visited.insert(*to) { + worklist.push_back(*to); + } + } + } + true + } + + /// Returns the states reachable **from the start state** by following + /// non-empty transitions (the start state is always included). + /// + /// This is forward reachability. Contrast with [`Self::live_states`], + /// which returns the states that can **reach an accept state** + /// (co-reachability). + pub(crate) fn forward_reachable_states(&self) -> IntSet { + let mut visited = IntSet::default(); + let mut worklist = VecDeque::new(); + visited.insert(self.start_state); + worklist.push_back(self.start_state); + while let Some(s) = worklist.pop_front() { + for (condition, to_state) in self.transitions_from(s) { + if condition.is_empty() { + continue; + } + if visited.insert(*to_state) { + worklist.push_back(*to_state); + } } } + visited + } + /// Returns the "live" (co-reachable) states: those that can **reach an + /// accept state** by following non-empty transitions. Computed by a reverse + /// traversal from the accept states. + /// + /// This is co-reachability; note it is *not* the set of states reachable + /// from the start state. + pub fn live_states(&self) -> IntSet { + // Reverse BFS over the maintained `transitions_in` adjacency. + // `transitions_in` doesn't filter empty-condition edges + // (constructible via the public `add_transition`), so each edge's + // condition is checked on traversal — the lookup on `transitions` + // also makes tombstoned predecessors fall out naturally. let mut worklist = VecDeque::from_iter(self.accept_states.iter().cloned()); let mut live = self.accept_states.clone(); while let Some(live_state) = worklist.pop_front() { - if let Some(states) = states_map.get(&live_state) { - for state in states { - if !live.contains(state) { - live.insert(*state); - worklist.push_back(*state); - } + let Some(predecessors) = self.transitions_in.get(&live_state) else { + continue; + }; + for &from_state in predecessors { + if live.contains(&from_state) { + continue; + } + let takeable = self + .condition(from_state, live_state) + .is_some_and(|condition| !condition.is_empty()); + if takeable { + live.insert(from_state); + worklist.push_back(from_state); } } } @@ -61,9 +184,74 @@ impl FastAutomaton { live } - pub fn get_ranges(&self) -> Result, EngineError> { - self.spanning_set.get_spanning_ranges().map(|range| { - Condition::from_range(range, &self.spanning_set) - }).collect() + /// Returns one [`Condition`] per base of the spanning set, including the + /// "rest" range when it is non-empty. + /// + /// The bases must partition the whole alphabet Σ: subset construction + /// ([`determinize`](Self::determinize)) and Hopcroft partitioning + /// ([`minimize`](Self::minimize)) iterate them and would otherwise silently + /// drop transitions whose condition lies in the "rest" range. (For a + /// spanning set with an empty rest this is exactly the spanning ranges, so + /// well-formed automata are unaffected.) + pub fn spanning_bases(&self) -> Result, EngineError> { + // Base `i` is by construction exactly bit `i` of a condition. + Ok((0..self.spanning_set.spanning_ranges_with_rest_len()) + .map(|i| Condition::single_base(i, &self.spanning_set)) + .collect()) + } +} + +#[cfg(test)] +mod tests { + + use crate::fast_automaton::FastAutomaton; + + #[test] + fn test_empty() -> Result<(), String> { + assert!(!FastAutomaton::new_total().is_empty()); + assert!(!FastAutomaton::new_empty_string().is_empty()); + assert!(FastAutomaton::new_empty().is_empty()); + + Ok(()) + } + + #[test] + fn test_empty_string() -> Result<(), String> { + assert!(!FastAutomaton::new_total().is_empty_string()); + assert!(FastAutomaton::new_empty_string().is_empty_string()); + assert!(!FastAutomaton::new_empty().is_empty_string()); + + Ok(()) + } + + #[test] + fn test_total() -> Result<(), String> { + assert!(FastAutomaton::new_total().is_total()); + assert!(!FastAutomaton::new_empty_string().is_total()); + assert!(!FastAutomaton::new_empty().is_total()); + + Ok(()) + } + + // An empty-condition transition (constructible via the public + // `add_transition`) can't be taken, so a state whose only path to an + // accept state goes through one is dead. `live_states` walks + // `transitions_in`, which records such edges — it must check the + // condition instead of trusting the adjacency. + #[test] + fn live_states_ignores_empty_condition_edges() { + use crate::fast_automaton::condition::Condition; + + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + a.add_transition(0, s1, &Condition::total(a.spanning_set())); + a.add_transition(s2, s1, &Condition::empty(a.spanning_set())); + a.accept(s1); + + let live = a.live_states(); + assert!(live.contains(&0)); + assert!(live.contains(&s1)); + assert!(!live.contains(&s2)); } } diff --git a/src/fast_automaton/analyze/subset.rs b/src/fast_automaton/analyze/subset.rs index 5705fc2..da9e4d0 100644 --- a/src/fast_automaton/analyze/subset.rs +++ b/src/fast_automaton/analyze/subset.rs @@ -3,14 +3,25 @@ use crate::error::EngineError; use super::*; impl FastAutomaton { - pub fn is_subset_of(&self, other: &FastAutomaton) -> Result { + /// Returns `true` if all strings accepted by `self` are also accepted by `other`. + /// + /// A non-deterministic `other` is determinized internally, unless the + /// execution profile disables implicit determinization, in which case + /// [`EngineError::DeterministicAutomatonRequired`] is returned. + #[tracing::instrument(level = "debug", skip_all, fields(self_states = self.number_of_states(), self_deterministic = self.is_deterministic(), other_states = other.number_of_states(), other_deterministic = other.is_deterministic()))] + pub fn subset(&self, other: &FastAutomaton) -> Result { if self.is_empty() || other.is_total() || self == other { return Ok(true); - } else if other.is_empty() || self.is_total() { + } else if other.is_empty() { return Ok(false); + } else if self.is_total() { + // self ⊆ other iff Σ* ⊆ other iff other = Σ*. We already failed + // the cheap `other.is_total()` check above; that check is sound + // but conservative on NFAs, so retry on the determinized form. + return Ok(other.determinize_implicit()?.is_total()); } - let mut other = other.determinize()?; + let mut other = other.determinize_implicit()?.into_owned(); other.complement()?; Ok(!self.has_intersection(&other)?) @@ -38,33 +49,33 @@ mod tests { true, ); - let regex1 = RegularExpression::new("test.*other").unwrap(); - let regex2 = RegularExpression::new("test.*othew").unwrap(); + let regex1 = RegularExpression::parse("test.*other", false).unwrap(); + let regex2 = RegularExpression::parse("test.*othew", false).unwrap(); assert_subset(®ex1, ®ex2, false, false); - let regex1 = RegularExpression::new("test.{0,50}other").unwrap(); - let regex2 = RegularExpression::new("test.{0,49}other").unwrap(); + let regex1 = RegularExpression::parse("test.{0,50}other", false).unwrap(); + let regex2 = RegularExpression::parse("test.{0,49}other", false).unwrap(); assert_subset(®ex1, ®ex2, false, true); - let regex1 = RegularExpression::new("(abc|def)").unwrap(); - let regex2 = RegularExpression::new("(abc|def|xyz)").unwrap(); + let regex1 = RegularExpression::parse("(abc|def)", false).unwrap(); + let regex2 = RegularExpression::parse("(abc|def|xyz)", false).unwrap(); assert_subset(®ex1, ®ex2, true, false); - let regex1 = RegularExpression::new("[0]").unwrap(); - let regex2 = RegularExpression::new("[01]").unwrap(); + let regex1 = RegularExpression::parse("[0]", false).unwrap(); + let regex2 = RegularExpression::parse("[01]", false).unwrap(); assert_subset(®ex1, ®ex2, true, false); - let regex1 = RegularExpression::new("a.*b.*c.*").unwrap(); - let regex2 = RegularExpression::new("a.*b.*").unwrap(); + let regex1 = RegularExpression::parse("a.*b.*c.*", false).unwrap(); + let regex2 = RegularExpression::parse("a.*b.*", false).unwrap(); assert_subset(®ex1, ®ex2, true, false); - let regex1 = RegularExpression::new("1..").unwrap(); - let regex2 = RegularExpression::new("...").unwrap(); + let regex1 = RegularExpression::parse("1..", false).unwrap(); + let regex2 = RegularExpression::parse("...", false).unwrap(); assert_subset(®ex1, ®ex2, true, false); @@ -79,18 +90,12 @@ mod tests { ) { println!("{regex_1} and {regex_2}"); let automaton_1 = regex_1.to_automaton().unwrap(); - assert_eq!(true, automaton_1.is_subset_of(&automaton_1).unwrap()); + assert!(automaton_1.subset(&automaton_1).unwrap()); let automaton_2 = regex_2.to_automaton().unwrap(); - assert_eq!(true, automaton_2.is_subset_of(&automaton_2).unwrap()); + assert!(automaton_2.subset(&automaton_2).unwrap()); - assert_eq!( - expected_1_2, - automaton_1.is_subset_of(&automaton_2).unwrap() - ); - assert_eq!( - expected_2_1, - automaton_2.is_subset_of(&automaton_1).unwrap() - ); + assert_eq!(expected_1_2, automaton_1.subset(&automaton_2).unwrap()); + assert_eq!(expected_2_1, automaton_2.subset(&automaton_1).unwrap()); } } diff --git a/src/fast_automaton/builder.rs b/src/fast_automaton/builder.rs index b6cf50b..7330548 100644 --- a/src/fast_automaton/builder.rs +++ b/src/fast_automaton/builder.rs @@ -5,6 +5,7 @@ use crate::error::EngineError; use super::*; impl FastAutomaton { + /// Creates an automaton that matches the empty language. #[inline] pub fn new_empty() -> Self { Self { @@ -15,102 +16,120 @@ impl FastAutomaton { removed_states: IntSet::default(), spanning_set: SpanningSet::new_empty(), deterministic: true, - cyclic: false, + minimal: true, } } + /// Creates an automaton that only matches the empty string `""`. #[inline] pub fn new_empty_string() -> Self { let mut automaton = Self::new_empty(); automaton.accept(automaton.start_state); + automaton.minimal = true; automaton } + /// Creates an automaton that matches all possible strings. #[inline] pub fn new_total() -> Self { let mut automaton: FastAutomaton = Self::new_empty(); automaton.spanning_set = SpanningSet::new_total(); automaton.accept(automaton.start_state); - automaton.add_transition_to(0, 0, &Condition::total(&automaton.spanning_set)); + automaton.add_transition(0, 0, &Condition::total(&automaton.spanning_set)); + automaton.minimal = true; automaton } - #[inline] - pub fn make_empty(&mut self) { - self.apply_model(&Self::new_empty()) - } - - #[inline] - pub fn make_total(&mut self) { - self.apply_model(&Self::new_total()) - } - - pub fn make_from_range(range: &Range) -> Result { + /// Creates an automaton that matches one of the characters in the given [`CharRange`]. + pub fn new_from_range(range: &CharRange) -> Self { let mut automaton = Self::new_empty(); if range.is_empty() { - return Ok(automaton); + return automaton; } let new_state = automaton.new_state(); - let spanning_set = SpanningSet::compute_spanning_set(&[range.clone()]); - let condition = Condition::from_range(range, &spanning_set)?; + let spanning_set = SpanningSet::compute_spanning_set(std::slice::from_ref(range)); + let condition = + Condition::from_range(range, &spanning_set).expect("The spanning set should be valid"); automaton.spanning_set = spanning_set; - automaton.add_transition_to(0, new_state, &condition); + automaton.add_transition(0, new_state, &condition); automaton.accept(new_state); - Ok(automaton) - } - - pub fn apply_new_spanning_set( - &mut self, - new_spanning_set: &SpanningSet, - ) -> Result<(), EngineError> { - if new_spanning_set == &self.spanning_set { - return Ok(()); - } - let condition_converter = ConditionConverter::new(&self.spanning_set, new_spanning_set)?; - for from_state in &self.transitions_vec() { - for to_state in self.transitions_from_state(from_state) { - match self.transitions[*from_state].entry(to_state) { - Entry::Occupied(mut o) => { - o.insert(condition_converter.convert(o.get())?); - } - Entry::Vacant(_) => {} - }; - } - } - self.spanning_set = new_spanning_set.clone(); - Ok(()) - } - - #[inline] - pub fn apply_model(&mut self, model: &FastAutomaton) { - self.transitions = model.transitions.clone(); - self.start_state = model.start_state; - self.accept_states = model.accept_states.clone(); - self.removed_states = model.removed_states.clone(); - self.spanning_set = model.spanning_set.clone(); - self.deterministic = model.deterministic; - self.cyclic = model.cyclic; + automaton.minimal = true; + automaton } + /// Creates a new state and returns its identifier. #[inline] pub fn new_state(&mut self) -> State { - if let Some(new_state) = self.removed_states.clone().iter().next() { - self.removed_states.remove(new_state); - *new_state + self.minimal = false; + if let Some(new_state) = self.removed_states.iter().next().copied() { + self.removed_states.remove(&new_state); + new_state } else { self.transitions.push(Transitions::default()); self.transitions.len() - 1 } } + /// Marks the provided state as an accepting (final) state. #[inline] pub fn accept(&mut self, state: State) { self.assert_state_exists(state); + self.minimal = false; self.accept_states.insert(state); } - pub fn add_transition_to(&mut self, from_state: State, to_state: State, new_cond: &Condition) { + /// Marks the provided state as a non-accepting state. + #[inline] + pub fn unaccept(&mut self, state: State) { + self.assert_state_exists(state); + self.minimal = false; + self.accept_states.remove(&state); + } + + /// Creates a new transition with the given condition; the condition must follow the automaton’s current spanning set. + /// + /// If you don't want to deal with conditions and spanning sets, use + /// [`add_transition_from_range`](Self::add_transition_from_range), which + /// handles the bookkeeping for you. + /// + /// This method accepts a [`Condition`] rather than a raw character set. To build a [`Condition`], call: + /// ```rust + /// # use regexsolver::CharRange; + /// # use regexsolver::fast_automaton::{condition::Condition, spanning_set::SpanningSet}; + /// # let range = CharRange::total(); + /// # let spanning_set = SpanningSet::new_total(); + /// Condition::from_range(&range, &spanning_set); + /// ``` + /// where `spanning_set` is the automaton's current [`SpanningSet`]. The [`CharRange`] you pass must be fully covered by that spanning set. If it isn't, you have two options: + /// + /// 1. Merge an existing spanning set with another: + /// ```rust + /// # use regexsolver::fast_automaton::spanning_set::SpanningSet; + /// # let old_set = SpanningSet::new_total(); + /// # let other_set = SpanningSet::new_total(); + /// let new_set = SpanningSet::merge(&old_set, &other_set); + /// ``` + /// + /// 2. Recompute from a list of ranges: + /// ```rust + /// # use regexsolver::CharRange; + /// # use regexsolver::fast_automaton::spanning_set::SpanningSet; + /// # let range_set1 = CharRange::total(); + /// # let range_set2 = CharRange::total(); + /// let new_set = SpanningSet::compute_spanning_set(&[range_set1, range_set2]); + /// ``` + /// + /// After constructing `new_set`, apply it to the automaton: + /// ```rust + /// # use regexsolver::fast_automaton::{FastAutomaton, spanning_set::SpanningSet}; + /// # let mut fast_automaton = FastAutomaton::new_total(); + /// # let new_set = SpanningSet::new_total(); + /// fast_automaton.apply_new_spanning_set(&new_set); + /// ``` + /// + /// This design allows us to perform unions, intersections, and complements of transition conditions in O(1) time, but it does add some complexity to automaton construction. For more details, you can check [this article](https://alexvbrdn.me/post/optimizing-transition-conditions-automaton-representation). + pub fn add_transition(&mut self, from_state: State, to_state: State, new_cond: &Condition) { self.assert_state_exists(from_state); if from_state != to_state { self.assert_state_exists(to_state); @@ -119,9 +138,10 @@ impl FastAutomaton { return; } + self.minimal = false; if self.deterministic { let mut deterministic = true; - for (state, condition) in self.transitions_from_state_enumerate_iter(&from_state) { + for (condition, state) in self.transitions_from(from_state) { if state == &to_state { continue; } @@ -139,7 +159,7 @@ impl FastAutomaton { .insert(from_state); match self.transitions[from_state].entry(to_state) { Entry::Occupied(mut o) => { - o.insert(o.get().union(new_cond)); + o.get_mut().union_with(new_cond); } Entry::Vacant(v) => { v.insert(new_cond.clone()); @@ -147,22 +167,130 @@ impl FastAutomaton { }; } - pub fn add_epsilon(&mut self, from_state: State, to_state: State) { + /// Adds a transition labeled with the given character range, taking care + /// of the spanning-set bookkeeping. + /// + /// This is the convenient counterpart to + /// [`add_transition`](Self::add_transition): the range is converted to a + /// [`Condition`] automatically, and when it is not exactly expressible + /// in the automaton's current spanning set, the spanning set is extended + /// and every existing condition is re-projected first. + /// + /// An empty range matches no character, so no transition is added. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::CharRange; + /// use regexsolver::fast_automaton::FastAutomaton; + /// use regex_charclass::char::Char; + /// + /// let mut automaton = FastAutomaton::new_empty(); + /// let s1 = automaton.new_state(); + /// automaton.accept(s1); + /// + /// let a_to_c = CharRange::new_from_range(Char::new('a')..=Char::new('c')); + /// automaton.add_transition_from_range(0, s1, &a_to_c).unwrap(); + /// + /// assert!(automaton.is_match("b")); + /// assert!(!automaton.is_match("d")); + /// ``` + pub fn add_transition_from_range( + &mut self, + from_state: State, + to_state: State, + range: &CharRange, + ) -> Result<(), EngineError> { + if range.is_empty() { + return Ok(()); + } + + // Fast path: the range is exactly expressible in the current + // spanning set. `Condition::from_range` alone cannot tell us that + // (it silently drops partially-covered bases), so round-trip the + // condition to check exactness. + if let Ok(condition) = Condition::from_range(range, &self.spanning_set) + && condition.to_range(&self.spanning_set)? == *range + { + self.add_transition(from_state, to_state, &condition); + return Ok(()); + } + + // The range is not (fully) covered: extend the spanning set, + // re-project the existing conditions, then add. + let new_spanning_set = + self.spanning_set + .merge(&SpanningSet::compute_spanning_set(std::slice::from_ref( + range, + ))); + self.apply_new_spanning_set(&new_spanning_set)?; + + let condition = Condition::from_range(range, &self.spanning_set)?; + self.add_transition(from_state, to_state, &condition); + Ok(()) + } + + /// Adds a transition, but refuses if it would turn a DFA into an NFA. + /// + /// On `Err(DeterminismLost)` the automaton is left untouched; on `Ok`, + /// the transition has been added and `is_deterministic()` still holds + /// (provided it held before the call). This is the opt-in strict + /// counterpart to [`add_transition`](Self::add_transition). + pub fn try_add_transition( + &mut self, + from_state: State, + to_state: State, + new_cond: &Condition, + ) -> Result<(), super::DeterminismLost> { + self.assert_state_exists(from_state); + if from_state != to_state { + self.assert_state_exists(to_state); + } + if new_cond.is_empty() { + return Ok(()); + } + if self.deterministic { + for (condition, state) in self.transitions_from(from_state) { + if *state == to_state { + continue; + } + if condition.has_intersection(new_cond) { + return Err(super::DeterminismLost); + } + } + } + self.add_transition(from_state, to_state, new_cond); + Ok(()) + } + + /// Adds an epsilon transition by eagerly folding `to_state`'s **current** + /// transitions (and acceptance) into `from_state`. + /// + /// This is a snapshot: transitions added to `to_state` *afterwards* are + /// not propagated retroactively. When building automata incrementally, + /// add epsilon transitions last. + pub fn add_epsilon_transition(&mut self, from_state: State, to_state: State) { if from_state == to_state { return; } self.assert_state_exists(from_state); self.assert_state_exists(to_state); + + self.minimal = false; + if self.accept_states.contains(&to_state) { self.accept_states.insert(from_state); } - let transitions_to: Vec<_> = self.transitions_from_state_into_iter(&to_state).collect(); + let transitions_to: Vec<_> = self + .transitions_from(to_state) + .map(|(cond, to_state)| (cond.clone(), *to_state)) + .collect(); - for (state, cond) in transitions_to { + for (cond, state) in transitions_to { if self.deterministic { let mut deterministic = true; - for (s, c) in self.transitions_from_state_enumerate_iter(&from_state) { + for (c, s) in self.transitions_from(from_state) { if state == *s { continue; } @@ -179,7 +307,7 @@ impl FastAutomaton { .insert(from_state); match self.transitions[from_state].entry(state) { Entry::Occupied(mut o) => { - o.insert(o.get().union(&cond)); + o.get_mut().union_with(&cond); } Entry::Vacant(v) => { v.insert(cond); @@ -188,24 +316,39 @@ impl FastAutomaton { } } + /// Removes the transition between the two provided states if it exists. + pub fn remove_transition(&mut self, from_state: State, to_state: State) { + self.assert_state_exists(from_state); + if from_state != to_state { + self.assert_state_exists(to_state); + } + + self.minimal = false; + + self.transitions_in + .entry(to_state) + .or_default() + .remove(&from_state); + self.transitions[from_state].remove(&to_state); + } + + /// Removes the state and its connected transitions; panics if it's a start state. pub fn remove_state(&mut self, state: State) { self.assert_state_exists(state); if self.start_state == state { - panic!( - "Can not remove the state {}, it is still used as start state.", - state - ); + panic!("Can not remove the state {state}, it is still used as start state."); } + self.minimal = false; self.accept_states.remove(&state); self.transitions_in.remove(&state); if self.transitions.len() - 1 == state { self.transitions.remove(state); let mut s = state; - while self.removed_states.contains(&s) { + while s > 0 && self.removed_states.contains(&(s - 1)) { + s -= 1; self.transitions.remove(s); self.removed_states.remove(&s); - s -= 1; } } else { self.transitions[state].clear(); @@ -215,44 +358,47 @@ impl FastAutomaton { for transitions in self.transitions.iter_mut() { transitions.remove(&state); } - for (_, transitions) in self.transitions_in.iter_mut() { + for transitions in self.transitions_in.values_mut() { transitions.remove(&state); } } + /// Removes the given states and their connected transitions; panics if any + /// state does not exist or is the start state. pub fn remove_states(&mut self, states: &IntSet) { + for &state in states { + self.assert_state_exists(state); + if self.start_state == state { + panic!("Can not remove the state {state}, it is still used as start state."); + } + } + + if states.is_empty() { + return; + } + self.accept_states.retain(|e| !states.contains(e)); - let mut states_to_remove = Vec::with_capacity(states.len()); + self.minimal = false; for &state in states { - if self.start_state == state { - panic!( - "Can not remove the state {}, it is still used as start state.", - state - ); - } if self.transitions.len() - 1 == state { self.transitions.remove(state); let mut s = state; - while self.removed_states.contains(&s) { + while s > 0 && self.removed_states.contains(&(s - 1)) { + s -= 1; self.transitions.remove(s); self.removed_states.remove(&s); - s -= 1; } } else { self.transitions[state].clear(); self.removed_states.insert(state); } - states_to_remove.push(state); - } - if states_to_remove.is_empty() { - return; } for transitions in self.transitions.iter_mut() { - for state in &states_to_remove { + for state in states { if transitions.is_empty() { break; } @@ -260,13 +406,254 @@ impl FastAutomaton { transitions.remove(state); } } + + for state in states { + self.transitions_in.remove(state); + } + for predecessors in self.transitions_in.values_mut() { + for state in states { + predecessors.remove(state); + } + } + } + + /// Recompute a minimal spanning set for the automaton and apply it. + pub fn recompute_minimal_spanning_set(&mut self) -> Result<(), EngineError> { + let mut ranges = Vec::with_capacity(self.number_of_states()); + + for state in self.states() { + for (condition, _) in self.transitions_from(state) { + ranges.push(condition.to_range(&self.spanning_set)?); + } + } + + let new_spanning_set = SpanningSet::compute_spanning_set(&ranges); + + self.apply_new_spanning_set(&new_spanning_set) + } + + /// Applies the provided spanning set and projects all existing conditions onto it. + pub fn apply_new_spanning_set( + &mut self, + new_spanning_set: &SpanningSet, + ) -> Result<(), EngineError> { + if new_spanning_set == &self.spanning_set { + return Ok(()); + } + let condition_converter = ConditionConverter::new(&self.spanning_set, new_spanning_set)?; + // Removed states keep a cleared transition map (see `remove_state`), + // so every stored condition can be converted in place directly. + for transitions in self.transitions.iter_mut() { + for condition in transitions.values_mut() { + *condition = condition_converter.convert(condition)?; + } + } + self.spanning_set = new_spanning_set.clone(); + Ok(()) + } + + #[inline] + pub(crate) fn make_empty(&mut self) { + self.apply_model(&Self::new_empty()) + } + + #[inline] + pub(crate) fn make_total(&mut self) { + self.apply_model(&Self::new_total()) + } + + #[inline] + pub(crate) fn make_empty_string(&mut self) { + self.apply_model(&Self::new_empty_string()) + } + + #[inline] + pub(crate) fn apply_model(&mut self, model: &FastAutomaton) { + self.transitions = model.transitions.clone(); + self.transitions_in = model.transitions_in.clone(); + self.start_state = model.start_state; + self.accept_states = model.accept_states.clone(); + self.removed_states = model.removed_states.clone(); + self.spanning_set = model.spanning_set.clone(); + self.deterministic = model.deterministic; + self.minimal = model.minimal; } } #[cfg(test)] mod tests { + use crate::IntSet; + use crate::fast_automaton::FastAutomaton; + use crate::fast_automaton::condition::Condition; use crate::regex::RegularExpression; + fn rng(a: char, b: char) -> crate::CharRange { + use regex_charclass::char::Char; + crate::CharRange::new_from_range(Char::new(a)..=Char::new(b)) + } + + #[test] + fn small_mutators_and_queries() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + a.add_transition_from_range(0, s1, &rng('a', 'a')).unwrap(); + a.accept(s1); + + assert!(a.is_accepted(s1)); + assert!(a.has_transition(0, s1)); + assert!(a.condition(0, s1).is_some()); + assert_eq!(a.in_degree(s1), 1); + assert_eq!(a.out_degree(0), 1); + assert!(a.is_match("a")); + + // try_add_transition: refuses determinism-breaking additions and + // leaves the automaton untouched on Err. + let condition_a = Condition::from_range(&rng('a', 'a'), a.spanning_set()).unwrap(); + assert!(a.is_deterministic()); + assert!(a.try_add_transition(0, s2, &condition_a).is_err()); + assert!(a.is_deterministic()); + assert!(!a.has_transition(0, s2)); + // ...but accepts disjoint conditions. + let condition_not_a = condition_a.complement(); + a.try_add_transition(0, s2, &condition_not_a).unwrap(); + assert!(a.is_deterministic()); + assert!(a.has_transition(0, s2)); + + // unaccept flips membership and the language. + a.unaccept(s1); + assert!(!a.is_accepted(s1)); + assert!(!a.is_match("a")); + a.accept(s1); + assert!(a.is_match("a")); + + // remove_transition removes the edge and updates queries. + a.remove_transition(0, s1); + assert!(!a.has_transition(0, s1)); + assert!(a.condition(0, s1).is_none()); + assert_eq!(a.in_degree(s1), 0); + assert!(!a.is_match("a")); + } + + #[test] + fn add_transition_from_range_extends_the_spanning_set() { + let mut automaton = FastAutomaton::new_empty(); + let s1 = automaton.new_state(); + let s2 = automaton.new_state(); + automaton.accept(s2); + + // Both ranges extend the (initially empty) spanning set. + automaton + .add_transition_from_range(0, s1, &rng('a', 'c')) + .unwrap(); + automaton + .add_transition_from_range(s1, s2, &rng('x', 'z')) + .unwrap(); + + assert!(automaton.is_match("ax")); + assert!(automaton.is_match("cz")); + assert!(!automaton.is_match("aa")); + assert!(!automaton.is_match("x")); + + // An exactly-covered range takes the fast path: same spanning set. + let before = automaton.spanning_set().clone(); + automaton + .add_transition_from_range(0, s1, &rng('x', 'z')) + .unwrap(); + assert_eq!(&before, automaton.spanning_set()); + assert!(automaton.is_match("zx")); + + // An empty range adds nothing. + automaton + .add_transition_from_range(0, s2, &crate::CharRange::empty()) + .unwrap(); + assert!(!automaton.is_match("a")); + } + + // `Condition::from_range` only sets bits for fully-covered bases, so + // adding a range that partially covers the "rest" (here [a-e] over an + // existing [a-c] base) must refine the spanning set to stay exact rather + // than truncate [a-e] to [a-c]. + #[test] + fn add_transition_from_range_is_exact_on_partial_coverage() { + let mut automaton = FastAutomaton::new_empty(); + let s1 = automaton.new_state(); + automaton.accept(s1); + + automaton + .add_transition_from_range(0, s1, &rng('a', 'c')) + .unwrap(); + // Contains the whole [a-c] base but only part of the rest. + automaton + .add_transition_from_range(0, s1, &rng('a', 'e')) + .unwrap(); + + for accepted in ["a", "b", "c", "d", "e"] { + assert!(automaton.is_match(accepted), "{accepted:?} must match"); + } + assert!(!automaton.is_match("f")); + } + + // Removing the trailing state must also physically drop tombstones that + // become trailing, so `transitions` does not stay at its peak length + // forever. + #[test] + fn remove_state_compacts_trailing_tombstones() { + let mut a = FastAutomaton::new_empty(); // state 0 + let s1 = a.new_state(); + let s2 = a.new_state(); + let s3 = a.new_state(); + + a.remove_state(s2); // tombstoned (not trailing) + assert_eq!(4, a.transitions.len()); + a.remove_state(s3); // trailing: pops s3 AND compacts the s2 tombstone + assert_eq!(2, a.transitions.len()); + assert!(a.removed_states.is_empty()); + assert!(a.has_state(s1)); + + // Same through `remove_states`, in one call. + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let s3 = a.new_state(); + let mut to_remove = IntSet::default(); + to_remove.insert(s2); + to_remove.insert(s3); + a.remove_states(&to_remove); + assert_eq!(2, a.transitions.len()); + assert!(a.removed_states.is_empty()); + assert!(a.has_state(s1)); + } + + // `remove_states` must perform the same `transitions_in` cleanup as the + // single-state `remove_state` (drop entries keyed by removed states and + // purge them from surviving predecessor sets); otherwise `in_degree` of a + // removed state stays stale for callers like repeat/concat/union. + #[test] + fn remove_states_cleans_transitions_in() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let cond = Condition::total(a.spanning_set()); + a.add_transition(0, s1, &cond); + a.add_transition(0, s2, &cond); + a.accept(s1); + a.accept(s2); + + assert_eq!(a.in_degree(s1), 1); + assert_eq!(a.in_degree(s2), 1); + + let mut to_remove = IntSet::default(); + to_remove.insert(s1); + a.remove_states(&to_remove); + + // After removing s1, its in_degree should report 0 (or, equivalently, + // queries on a removed state should be a clean no-op). Currently it + // still reports the pre-removal count. + assert_eq!(a.in_degree(s1), 0, "in_degree of removed state should be 0"); + assert_eq!(a.in_degree(s2), 1); + } + #[test] fn test_regex_build_deterministic_automaton() -> Result<(), String> { assert_regex_build_deterministic_automaton("...", true); @@ -278,10 +665,10 @@ mod tests { } fn assert_regex_build_deterministic_automaton(regex: &str, deterministic: bool) { - let automaton = RegularExpression::new(regex) + let automaton = RegularExpression::parse(regex, false) .unwrap() .to_automaton() .unwrap(); - assert_eq!(deterministic, automaton.is_determinitic()); + assert_eq!(deterministic, automaton.is_deterministic()); } } diff --git a/src/fast_automaton/condition/converter.rs b/src/fast_automaton/condition/converter.rs index 89bb123..624ba4d 100644 --- a/src/fast_automaton/condition/converter.rs +++ b/src/fast_automaton/condition/converter.rs @@ -1,7 +1,4 @@ -use ahash::HashMapExt; -use nohash_hasher::IntMap; - -use crate::{error::EngineError, fast_automaton::spanning_set::SpanningSet}; +use crate::{CharRange, error::EngineError, fast_automaton::spanning_set::SpanningSet}; use super::Condition; @@ -15,33 +12,35 @@ pub struct ConditionConverter<'a, 'b> { impl<'a, 'b> ConditionConverter<'a, 'b> { /// Build a converter to project [`Condition`] from `from_spanning_set` to `to_spanning_set`. /// - /// Currently this method does not check that the provided [`SpanningSet`] are actually convertible. + /// Two directions are legitimate: refinement (a merged spanning set + /// before a binary operation) and coarsening (a recomputed minimal + /// spanning set, where bases no transition uses fold into the rest). The + /// pair is therefore not validated here; instead [`convert`](Self::convert) + /// asserts in debug builds that each projection preserves the + /// condition's character range. pub fn new( from_spanning_set: &'a SpanningSet, to_spanning_set: &'b SpanningSet, ) -> Result { - let mut to_base_map = - IntMap::with_capacity(to_spanning_set.spanning_ranges_with_rest_len()); - for (i, base) in to_spanning_set - .get_spanning_ranges_with_rest() - .into_iter() - .enumerate() - { - to_base_map.insert(i, base); - } + // Each target base maps to at most one source base; consumed entries + // are marked `None` so later source bases skip them. + let mut to_bases: Vec> = to_spanning_set + .spanning_ranges_with_rest() + .map(Some) + .collect(); let mut equivalence_map: Vec> = - Vec::with_capacity(from_spanning_set.get_number_of_spanning_ranges() + 1); - for from_base in from_spanning_set.get_spanning_ranges_with_rest().iter() { + Vec::with_capacity(from_spanning_set.spanning_ranges_with_rest_len()); + for from_base in from_spanning_set.spanning_ranges_with_rest() { let mut index = Vec::with_capacity(1); - for (i, to_base) in &to_base_map { - if from_base == to_base || from_base.has_intersection(to_base) { - index.push(*i); + for (i, slot) in to_bases.iter_mut().enumerate() { + if let Some(to_base) = slot + && (from_base == *to_base || from_base.has_intersection(to_base)) + { + index.push(i); + *slot = None; } } - index.iter().for_each(|i| { - to_base_map.remove(i); - }); equivalence_map.push(index); } @@ -54,61 +53,73 @@ impl<'a, 'b> ConditionConverter<'a, 'b> { /// Project the given [`Condition`] from `from_spanning_set` to `to_spanning_set`. /// - /// If `from_spanning_set` is not convertible to `to_spanning_set` or if the given [`Condition`] is not based on `from_spanning_set`, - /// the resulting [`Condition`] will not have any relevance. + /// Returns [`EngineError::IncompatibleSpanningSet`] if the given + /// [`Condition`] was not built over `from_spanning_set`. pub fn convert(&self, condition: &Condition) -> Result { + if condition.0.len() != self.from_spanning_set.spanning_ranges_with_rest_len() { + return Err(EngineError::IncompatibleSpanningSet); + } let mut new_condition = Condition::empty(self.to_spanning_set); for (from_index, to_indexes) in self.equivalence_map.iter().enumerate() { - if let Some(has) = condition.0.get(from_index) { - if has && !to_indexes.is_empty() { - to_indexes.iter().for_each(|&to_index| { - new_condition.0.set(to_index, true); - }); - } - } else { - return Err(EngineError::ConditionIndexOutOfBound); + if condition.0.get(from_index) && !to_indexes.is_empty() { + to_indexes.iter().for_each(|&to_index| { + new_condition.0.set(to_index, true); + }); } } + // The one invariant every legitimate use (refining and coarsening + // alike) must uphold: the projection denotes the same character set. + // A violation means a condition referenced a base the target spanning + // set cannot express, causing silent language corruption in release. + debug_assert_eq!( + condition + .to_range(self.from_spanning_set) + .expect("the length was checked above"), + new_condition + .to_range(self.to_spanning_set) + .expect("the condition was built over the target spanning set"), + "the projection changed the condition's character range" + ); + Ok(new_condition) } /// Returns `from_spanning_set`. - pub fn get_from_spanning_set(&self) -> &'a SpanningSet { + pub fn from_spanning_set(&self) -> &'a SpanningSet { self.from_spanning_set } /// Returns `to_spanning_set`. - pub fn get_to_spanning_set(&self) -> &'b SpanningSet { + pub fn to_spanning_set(&self) -> &'b SpanningSet { self.to_spanning_set } } #[cfg(test)] mod tests { + use crate::CharRange; use regex_charclass::{char::Char, irange::range::AnyRange}; - use crate::Range; - use super::*; - fn get_from_spanning_set() -> SpanningSet { + fn from_spanning_set() -> SpanningSet { let ranges = vec![ - Range::new_from_range(Char::new('\0')..=Char::new('\u{2}')), - Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), - Range::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\0')..=Char::new('\u{2}')), + CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), + CharRange::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), ]; SpanningSet::compute_spanning_set(&ranges) } - fn get_to_spanning_set() -> SpanningSet { + fn to_spanning_set() -> SpanningSet { let ranges = vec![ - Range::new_from_range(Char::new('\0')..=Char::new('\u{1}')), - Range::new_from_range(Char::new('\u{2}')..=Char::new('\u{2}')), - Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), - Range::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), - Range::new_from_range(Char::new('\u{20}')..=Char::new('\u{22}')), + CharRange::new_from_range(Char::new('\0')..=Char::new('\u{1}')), + CharRange::new_from_range(Char::new('\u{2}')..=Char::new('\u{2}')), + CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), + CharRange::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\u{20}')..=Char::new('\u{22}')), ]; SpanningSet::compute_spanning_set(&ranges) @@ -116,8 +127,8 @@ mod tests { #[test] fn test_convert() -> Result<(), String> { - let from_spanning_set = get_from_spanning_set(); - let to_spanning_set = get_to_spanning_set(); + let from_spanning_set = from_spanning_set(); + let to_spanning_set = to_spanning_set(); let converter = ConditionConverter::new(&from_spanning_set, &to_spanning_set).unwrap(); @@ -127,7 +138,7 @@ mod tests { let total = Condition::total(&from_spanning_set); assert!(converter.convert(&total).unwrap().is_total()); - let range = Range::new_from_range(Char::new('\0')..=Char::new('\u{2}')); + let range = CharRange::new_from_range(Char::new('\0')..=Char::new('\u{2}')); let condition = Condition::from_range(&range, &from_spanning_set).unwrap(); assert_eq!( range, @@ -138,7 +149,7 @@ mod tests { .unwrap() ); - let range = Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')); + let range = CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')); let condition = Condition::from_range(&range, &from_spanning_set).unwrap(); assert_eq!( range, @@ -149,7 +160,7 @@ mod tests { .unwrap() ); - let range = Range::new_from_ranges(&[ + let range = CharRange::new_from_ranges(&[ AnyRange::from(Char::new('\u{4}')..=Char::new('\u{6}')), AnyRange::from(Char::new('\u{9}')..=Char::new('\u{9}')), ]); diff --git a/src/fast_automaton/condition/fast_bit_vec/mod.rs b/src/fast_automaton/condition/fast_bit_vec/mod.rs index bbf4376..481b157 100644 --- a/src/fast_automaton/condition/fast_bit_vec/mod.rs +++ b/src/fast_automaton/condition/fast_bit_vec/mod.rs @@ -7,8 +7,8 @@ pub struct FastBitVec { impl std::fmt::Display for FastBitVec { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { for i in 0..self.n { - let bit = if self.get(i).unwrap() { 1 } else { 0 }; - write!(f, "{}", bit)?; + let bit = if self.get(i) { 1 } else { 0 }; + write!(f, "{bit}")?; } Ok(()) } @@ -17,7 +17,11 @@ impl std::fmt::Display for FastBitVec { impl FastBitVec { #[inline] pub fn from_elem(n: usize, bit: bool) -> Self { - let nblocks = if n % 64 == 0 { n / 64 } else { n / 64 + 1 }; + let nblocks = if n.is_multiple_of(64) { + n / 64 + } else { + n / 64 + 1 + }; let bits = vec![if bit { !0_u64 } else { 0_u64 }; nblocks]; let mut bit_vec = FastBitVec { bits, n }; bit_vec.fix_last_block(); @@ -48,17 +52,16 @@ impl FastBitVec { } #[inline] - pub fn get(&self, i: usize) -> Option { - if i >= self.n { - return None; - } + pub fn get(&self, i: usize) -> bool { + assert!(i < self.n, "The provided bit index is out of bound."); let w = i / 64; let b = i % 64; - self.bits.get(w).map(|&block| (block & (1 << b)) != 0) + (self.bits[w] & (1 << b)) != 0 } #[inline] pub fn set(&mut self, i: usize, x: bool) { + assert!(i < self.n, "The provided bit index is out of bound."); let w = i / 64; let b = i % 64; let flag = 1 << b; @@ -78,8 +81,22 @@ impl FastBitVec { self.fix_last_block(); } + /// The binary operations combine blocks pairwise with `zip`, which would + /// silently truncate to the shorter operand if two bitvectors built over + /// different spanning sets were ever combined, producing a wrong + /// language instead of a loud failure. Catch that in debug builds (and + /// therefore in every test run). + #[inline] + fn assert_same_len(&self, other: &Self) { + debug_assert_eq!( + self.n, other.n, + "conditions built over different spanning sets cannot be combined" + ); + } + #[inline] pub fn union(&mut self, other: &Self) { + self.assert_same_len(other); for (a, b) in self.bits.iter_mut().zip(&other.bits) { let w = *a | b; *a = w; @@ -88,14 +105,24 @@ impl FastBitVec { #[inline] pub fn intersection(&mut self, other: &Self) { + self.assert_same_len(other); for (a, b) in self.bits.iter_mut().zip(&other.bits) { let w = *a & b; *a = w; } } + #[inline] + pub fn difference(&mut self, other: &Self) { + self.assert_same_len(other); + for (a, b) in self.bits.iter_mut().zip(&other.bits) { + *a &= !b; + } + } + #[inline] pub fn has_intersection(&self, other: &Self) -> bool { + self.assert_same_len(other); for (a, b) in self.bits.iter().zip(&other.bits) { if *a & b != 0 { return true; @@ -123,11 +150,32 @@ impl FastBitVec { (!0) >> ((64 - bits % 64) % 64) } - pub fn get_bits(&self) -> Vec { - let mut hot_bits = Vec::with_capacity(self.n); + pub fn bits(&self) -> Vec { + let mut bits = Vec::with_capacity(self.n); for i in 0..self.n { - hot_bits.push(self.get(i).unwrap()); + bits.push(self.get(i)); } - hot_bits + bits + } + + /// Iterates the indices of the set bits in ascending order, word-wise + /// (no allocation). + #[inline] + pub fn iter_set_bits(&self) -> impl Iterator + '_ { + self.bits + .iter() + .enumerate() + .flat_map(|(word_index, &word)| { + let mut remaining = word; + std::iter::from_fn(move || { + if remaining == 0 { + None + } else { + let bit = remaining.trailing_zeros() as usize; + remaining &= remaining - 1; + Some(word_index * 64 + bit) + } + }) + }) } } diff --git a/src/fast_automaton/condition/mod.rs b/src/fast_automaton/condition/mod.rs index da9c2b8..3269d8a 100644 --- a/src/fast_automaton/condition/mod.rs +++ b/src/fast_automaton/condition/mod.rs @@ -1,16 +1,18 @@ use std::hash::Hash; -use crate::Range; use fast_bit_vec::FastBitVec; -use regex_charclass::{char::Char, CharacterClass}; +use regex_charclass::{CharacterClass, char::Char}; -use crate::error::EngineError; +use crate::{CharRange, error::EngineError}; use super::spanning_set::SpanningSet; +/// The [`ConditionConverter`](converter::ConditionConverter): remaps a +/// [`Condition`] from one spanning set to another (used when merging automata +/// with different alphabets). pub mod converter; mod fast_bit_vec; -/// Contains the condition of a transition in a [`crate::FastAutomaton`] +/// Represents the condition of a transition in a [`crate::FastAutomaton`]. #[derive(Clone, PartialEq, Eq, Debug)] pub struct Condition(FastBitVec); @@ -27,6 +29,8 @@ impl Hash for Condition { } impl Condition { + /// Returns the condition that matches no character, sized for + /// `spanning_set` (every bit cleared). #[inline] pub fn empty(spanning_set: &SpanningSet) -> Self { Self(FastBitVec::from_elem( @@ -35,6 +39,8 @@ impl Condition { )) } + /// Returns the condition that matches every character, sized for + /// `spanning_set` (every bit set). #[inline] pub fn total(spanning_set: &SpanningSet) -> Self { Self(FastBitVec::from_elem( @@ -43,7 +49,14 @@ impl Condition { )) } - pub fn from_range(range: &Range, spanning_set: &SpanningSet) -> Result { + /// Converts a [`CharRange`] to a `Condition` sized for `spanning_set`. + /// + /// Returns [`EngineError::ConditionInvalidRange`] if the range is not + /// expressible in the current spanning set (no base is fully contained in + /// `range`). In that case, extend the spanning set first with + /// [`SpanningSet::merge`] or [`SpanningSet::compute_spanning_set`], apply + /// it with [`crate::fast_automaton::FastAutomaton::apply_new_spanning_set`], then retry. + pub fn from_range(range: &CharRange, spanning_set: &SpanningSet) -> Result { if range.is_empty() { return Ok(Self::empty(spanning_set)); } else if range.is_total() { @@ -52,11 +65,7 @@ impl Condition { let mut cond = Self::empty(spanning_set); - for (i, base) in spanning_set - .get_spanning_ranges_with_rest() - .iter() - .enumerate() - { + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { if range.contains_all(base) { cond.0.set(i, true); } @@ -69,40 +78,68 @@ impl Condition { Ok(cond) } - pub fn to_range(&self, spanning_set: &SpanningSet) -> Result { - let mut range = Range::empty(); - - for (i, base) in spanning_set - .get_spanning_ranges_with_rest() - .iter() - .enumerate() - { - if let Some(has) = self.0.get(i) { - if has { - range = range.union(base); - } - } else { - return Err(EngineError::ConditionIndexOutOfBound); + /// Returns the condition whose only set bit is base `i` of `spanning_set` + /// (bit `i` corresponds to `spanning_ranges_with_rest()`'s element `i`, + /// the rest range first when present). + #[inline] + pub(crate) fn single_base(i: usize, spanning_set: &SpanningSet) -> Self { + let mut cond = Self::empty(spanning_set); + cond.0.set(i, true); + cond + } + + /// Converts this `Condition` back to the [`CharRange`] it represents, + /// evaluated against `spanning_set`. + /// + /// Returns [`EngineError::IncompatibleSpanningSet`] if this condition's + /// bit width does not match `spanning_set` (they were built from different + /// spanning sets). + pub fn to_range(&self, spanning_set: &SpanningSet) -> Result { + // A condition only carries meaning relative to the spanning set it + // was built from; a width mismatch means they differ, so reject it + // rather than index out of bounds or silently drop bits. + if self.0.len() != spanning_set.spanning_ranges_with_rest_len() { + return Err(EngineError::IncompatibleSpanningSet); + } + + let mut range = CharRange::empty(); + + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { + if self.0.get(i) { + range = range.union(base); } } Ok(range) } + /// Returns the condition matching characters in `self` or `other` (bitwise + /// OR). Both must share the same spanning set. #[inline] - pub fn union(&self, cond: &Condition) -> Self { + pub fn union(&self, other: &Condition) -> Self { let mut new_cond = self.clone(); - new_cond.0.union(&cond.0); + new_cond.0.union(&other.0); new_cond } + /// Unions `other` into `self` in place (bitwise OR). Both must share the + /// same spanning set. + #[inline] + pub fn union_with(&mut self, other: &Condition) { + self.0.union(&other.0); + } + + /// Returns the condition matching characters in both `self` and `other` + /// (bitwise AND). Both must share the same spanning set. #[inline] - pub fn intersection(&self, cond: &Condition) -> Self { + pub fn intersection(&self, other: &Condition) -> Self { let mut new_cond = self.clone(); - new_cond.0.intersection(&cond.0); + new_cond.0.intersection(&other.0); new_cond } + /// Returns the condition matching exactly the characters `self` does not, + /// relative to its spanning set. #[inline] pub fn complement(&self) -> Self { let mut new_cond = self.clone(); @@ -110,49 +147,107 @@ impl Condition { new_cond } + /// Returns the condition matching characters in `self` but not in `other` + /// (bitwise AND-NOT). Both must share the same spanning set. #[inline] - pub fn difference(&self, cond: &Condition) -> Self { + pub fn difference(&self, other: &Condition) -> Self { let mut new_cond = self.clone(); - let subtrahend = cond.complement(); - new_cond.0.intersection(&subtrahend.0); + new_cond.0.difference(&other.0); new_cond } + /// Returns `true` if `self` and `other` share at least one character (their + /// intersection is non-empty). Both must share the same spanning set. #[inline] - pub fn has_intersection(&self, cond: &Condition) -> bool { - self.0.has_intersection(&cond.0) + pub fn has_intersection(&self, other: &Condition) -> bool { + self.0.has_intersection(&other.0) } + /// Returns `true` if the condition matches `character` (a Unicode scalar + /// value), evaluated against `spanning_set`. Values that are not valid + /// scalar values never match. + /// + /// Returns [`EngineError::IncompatibleSpanningSet`] if this condition's + /// bit width does not match `spanning_set`. #[inline] pub fn has_character( &self, character: &u32, spanning_set: &SpanningSet, ) -> Result { - if let Some(character) = Char::from_u32(*character) { - Ok(self.to_range(spanning_set)?.contains(character)) - } else { - Ok(false) + let Some(character) = Char::from_u32(*character) else { + return Ok(false); + }; + if self.0.len() != spanning_set.spanning_ranges_with_rest_len() { + return Err(EngineError::IncompatibleSpanningSet); } + + // Bit `i` corresponds to `spanning_ranges_with_rest()[i]` (the rest + // range first, when present). Testing set bits directly avoids + // materializing the union of their ranges (`to_range` clones and + // unions every base) on the `is_match` hot path. + let mut i = 0; + if !spanning_set.rest().is_empty() { + if self.0.get(i) && spanning_set.rest().contains(character) { + return Ok(true); + } + i += 1; + } + for base in spanning_set.spanning_ranges() { + if self.0.get(i) && base.contains(character) { + return Ok(true); + } + i += 1; + } + Ok(false) } + /// Returns `true` if the condition matches no character. #[inline] pub fn is_empty(&self) -> bool { self.0.empty() } + /// Returns `true` if the condition matches every character. #[inline] pub fn is_total(&self) -> bool { self.0.total() } + /// Returns the number of characters the condition matches, evaluated + /// against `spanning_set`. + /// + /// Returns [`EngineError::IncompatibleSpanningSet`] if this condition's + /// bit width does not match `spanning_set`. #[inline] - pub fn get_cardinality(&self, spanning_set: &SpanningSet) -> Result { - Ok(self.to_range(spanning_set)?.get_cardinality()) + pub fn cardinality(&self, spanning_set: &SpanningSet) -> Result { + if self.0.len() != spanning_set.spanning_ranges_with_rest_len() { + return Err(EngineError::IncompatibleSpanningSet); + } + + // The bases are disjoint, so the cardinality of their union is the + // sum of their cardinalities. + let mut cardinality = 0u32; + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { + if self.0.get(i) { + cardinality += base.get_cardinality(); + } + } + Ok(cardinality) } - pub fn get_bits(&self) -> Vec { - self.0.get_bits() + /// Returns the condition as a vector of bits, one per range of the spanning + /// set it was built against (the rest range first, when present). + #[inline] + pub fn binary_representation(&self) -> Vec { + self.0.bits() + } + + /// Iterates the indices of the set bits (i.e., the bases the condition + /// covers) in ascending order, without allocating. + #[inline] + pub(crate) fn iter_set_bits(&self) -> impl Iterator + '_ { + self.0.iter_set_bits() } } @@ -163,52 +258,93 @@ mod tests { use super::*; - fn get_spanning_set() -> SpanningSet { + fn spanning_set() -> SpanningSet { let ranges = vec![ - Range::new_from_range(Char::new('\u{0}')..=Char::new('\u{2}')), - Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), - Range::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{2}')), + CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), + CharRange::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), ]; SpanningSet::compute_spanning_set(&ranges) } - fn get_test_cases_range() -> Vec { + fn get_test_cases_range() -> Vec { vec![ - Range::empty(), - Range::total(), - Range::new_from_range(Char::new('\u{0}')..=Char::new('\u{2}')), - Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), - Range::new_from_ranges(&[ + CharRange::empty(), + CharRange::total(), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{2}')), + CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), + CharRange::new_from_ranges(&[ AnyRange::from(Char::new('\u{0}')..=Char::new('\u{2}')), AnyRange::from(Char::new('\u{4}')..=Char::new('\u{6}')), ]), - Range::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), ] } + // Evaluating a condition against a spanning set it was not built from + // must report the incompatibility rather than panic (too short) or + // silently drop bits (too long). + #[test] + fn to_range_rejects_incompatible_spanning_set() { + let small = SpanningSet::compute_spanning_set(&[CharRange::new_from_range( + Char::new('a')..=Char::new('a'), + )]); + let large = spanning_set(); + + let condition = Condition::total(&small); + assert_eq!( + condition.to_range(&large), + Err(EngineError::IncompatibleSpanningSet) + ); + + let condition = Condition::total(&large); + assert_eq!( + condition.to_range(&small), + Err(EngineError::IncompatibleSpanningSet) + ); + } + + // `ConditionConverter::convert` must report an error, not panic, on a + // condition that was not built over its source spanning set. + #[test] + fn convert_rejects_incompatible_condition() { + let small = SpanningSet::compute_spanning_set(&[CharRange::new_from_range( + Char::new('a')..=Char::new('a'), + )]); + let merged = small.merge(&spanning_set()); + let converter = ConditionConverter::new(&small, &merged).unwrap(); + + let foreign = Condition::total(&merged); + assert_eq!( + converter.convert(&foreign), + Err(EngineError::IncompatibleSpanningSet) + ); + } + #[test] fn test_empty_total() -> Result<(), String> { - let spanning_set = get_spanning_set(); + let spanning_set = spanning_set(); let empty = Condition::empty(&spanning_set); - //println!("{empty}"); assert!(empty.is_empty()); - assert_eq!(vec![false, false, false, false], empty.get_bits()); + assert_eq!( + vec![false, false, false, false], + empty.binary_representation() + ); let total = Condition::total(&spanning_set); - //println!("{total}"); assert!(total.is_total()); - assert_eq!(vec![true, true, true, true], total.get_bits()); + assert_eq!(vec![true, true, true, true], total.binary_representation()); - assert_eq!(Range::empty(), empty.to_range(&spanning_set).unwrap()); - assert_eq!(Range::total(), total.to_range(&spanning_set).unwrap()); + assert_eq!(CharRange::empty(), empty.to_range(&spanning_set).unwrap()); + assert_eq!(CharRange::total(), total.to_range(&spanning_set).unwrap()); assert_eq!( empty, - Condition::from_range(&Range::empty(), &spanning_set).unwrap() + Condition::from_range(&CharRange::empty(), &spanning_set).unwrap() ); assert_eq!( total, - Condition::from_range(&Range::total(), &spanning_set).unwrap() + Condition::from_range(&CharRange::total(), &spanning_set).unwrap() ); assert_eq!(empty, total.complement()); @@ -218,20 +354,20 @@ mod tests { let empty = Condition::empty(&spanning_set); let total = Condition::total(&spanning_set); - assert_eq!(Range::empty(), empty.to_range(&spanning_set).unwrap()); - assert_eq!(Range::total(), total.to_range(&spanning_set).unwrap()); + assert_eq!(CharRange::empty(), empty.to_range(&spanning_set).unwrap()); + assert_eq!(CharRange::total(), total.to_range(&spanning_set).unwrap()); assert_eq!( empty, - Condition::from_range(&Range::empty(), &spanning_set).unwrap() + Condition::from_range(&CharRange::empty(), &spanning_set).unwrap() ); - assert_eq!(vec![false], empty.get_bits()); + assert_eq!(vec![false], empty.binary_representation()); assert_eq!( total, - Condition::from_range(&Range::total(), &spanning_set).unwrap() + Condition::from_range(&CharRange::total(), &spanning_set).unwrap() ); - assert_eq!(vec![true], total.get_bits()); + assert_eq!(vec![true], total.binary_representation()); assert_eq!(empty, total.complement()); assert_eq!(total, empty.complement()); @@ -241,7 +377,7 @@ mod tests { #[test] fn test_from_to_range() -> Result<(), String> { - let spanning_set = get_spanning_set(); + let spanning_set = spanning_set(); for range in get_test_cases_range() { assert_range_convertion_to_range(&range, &spanning_set); @@ -251,7 +387,7 @@ mod tests { Ok(()) } - fn assert_range_convertion_to_range(range: &Range, spanning_set: &SpanningSet) { + fn assert_range_convertion_to_range(range: &CharRange, spanning_set: &SpanningSet) { let condition = Condition::from_range(range, spanning_set).unwrap(); let range_from_condition = condition.to_range(spanning_set).unwrap(); assert_eq!(range, &range_from_condition); @@ -263,14 +399,14 @@ mod tests { #[test] fn test_project_to() -> Result<(), String> { - let current_spanning_set = get_spanning_set(); + let current_spanning_set = spanning_set(); let ranges = vec![ - Range::new_from_range(Char::new('\u{0}')..=Char::new('\u{1}')), - Range::new_from_range(Char::new('\u{2}')..=Char::new('\u{2}')), - Range::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), - Range::new_from_range(Char::new('\u{5}')..=Char::new('\u{6}')), - Range::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{1}')), + CharRange::new_from_range(Char::new('\u{2}')..=Char::new('\u{2}')), + CharRange::new_from_range(Char::new('\u{4}')..=Char::new('\u{6}')), + CharRange::new_from_range(Char::new('\u{5}')..=Char::new('\u{6}')), + CharRange::new_from_range(Char::new('\u{9}')..=Char::new('\u{9}')), ]; let new_spanning_set = SpanningSet::compute_spanning_set(&ranges); let condition_converter = @@ -295,7 +431,7 @@ mod tests { } fn assert_project_to( - range: &Range, + range: &CharRange, currently_used_spanning_set: &SpanningSet, newly_used_spanning_set: &SpanningSet, condition_converter: &ConditionConverter, @@ -320,7 +456,7 @@ mod tests { #[test] fn test_union_intersection_complement() -> Result<(), String> { - let used_characters = get_spanning_set(); + let used_characters = spanning_set(); for range_1 in get_test_cases_range() { for range_2 in get_test_cases_range() { @@ -347,20 +483,20 @@ mod tests { } fn assert_union_intersection_complement( - range_1: &Range, - range_2: &Range, + range_1: &CharRange, + range_2: &CharRange, used_characters: &SpanningSet, ) { let condition_1 = Condition::from_range(range_1, used_characters).unwrap(); let condition_2 = Condition::from_range(range_2, used_characters).unwrap(); assert_eq!( - Condition::empty(&used_characters), + Condition::empty(used_characters), condition_1.intersection(&condition_1.complement()) ); assert_eq!( - Condition::empty(&used_characters), + Condition::empty(used_characters), condition_2.intersection(&condition_2.complement()) ); @@ -377,14 +513,14 @@ mod tests { #[test] fn test_1() -> Result<(), String> { let ranges = vec![ - Range::new_from_range(Char::new('\u{0}')..=Char::new('\u{9}')), - Range::new_from_range(Char::new('\u{B}')..=Char::new('\u{63}')), - Range::new_from_range(Char::new('\u{65}')..=Char::new('\u{10FFFF}')), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{9}')), + CharRange::new_from_range(Char::new('\u{B}')..=Char::new('\u{63}')), + CharRange::new_from_range(Char::new('\u{65}')..=Char::new('\u{10FFFF}')), ]; let spanning_set = SpanningSet::compute_spanning_set(&ranges); println!("{:?}", spanning_set); - let range1 = Range::new_from_ranges(&[ + let range1 = CharRange::new_from_ranges(&[ AnyRange::from(Char::new('\u{0}')..=Char::new('\u{9}')), AnyRange::from(Char::new('\u{B}')..=Char::new('\u{63}')), AnyRange::from(Char::new('\u{65}')..=Char::new('\u{10FFFF}')), @@ -392,7 +528,7 @@ mod tests { let condition1 = Condition::from_range(&range1, &spanning_set).unwrap(); assert_eq!(range1, condition1.to_range(&spanning_set).unwrap()); - let range2 = Range::new_from_range(Char::new('\u{B}')..=Char::new('\u{63}')); + let range2 = CharRange::new_from_range(Char::new('\u{B}')..=Char::new('\u{63}')); let condition2 = Condition::from_range(&range2, &spanning_set).unwrap(); assert_eq!(range2, condition2.to_range(&spanning_set).unwrap()); diff --git a/src/fast_automaton/convert/to_regex/builder/mod.rs b/src/fast_automaton/convert/to_regex/builder/mod.rs deleted file mode 100644 index b6c8dd5..0000000 --- a/src/fast_automaton/convert/to_regex/builder/mod.rs +++ /dev/null @@ -1,170 +0,0 @@ -use super::*; - -mod scc; - -impl StateEliminationAutomaton { - pub fn new(automaton: &FastAutomaton) -> Result, EngineError> { - if automaton.is_empty() { - return Ok(None); - } - - let mut state_elimination_automaton = StateEliminationAutomaton { - start_state: 0, // start_state is not set yet - accept_state: 0, // accept_state is not set yet - transitions: Vec::with_capacity(automaton.get_number_of_states()), - transitions_in: IntMap::with_capacity(automaton.get_number_of_states()), - removed_states: IntSet::new(), - cyclic: false, - }; - - let mut states_map = IntMap::with_capacity(automaton.get_number_of_states()); - - for from_state in automaton.transitions_iter() { - let new_from_state = *states_map - .entry(from_state) - .or_insert_with(|| state_elimination_automaton.new_state()); - for (to_state, condition) in - automaton.transitions_from_state_enumerate_into_iter(&from_state) - { - let new_to_state = *states_map - .entry(to_state) - .or_insert_with(|| state_elimination_automaton.new_state()); - - state_elimination_automaton.add_transition_to( - new_from_state, - new_to_state, - GraphTransition::Weight(condition.to_range(automaton.get_spanning_set())?), - ); - } - } - - state_elimination_automaton.start_state = - *states_map.get(&automaton.get_start_state()).unwrap(); // We finally set start_state - - if automaton.get_accept_states().len() == 1 { - // If there is only one accept state with just set it - state_elimination_automaton.accept_state = *states_map - .get(automaton.get_accept_states().iter().next().unwrap()) - .unwrap(); - } else { - // If not we create a new state that will be the new accept state - state_elimination_automaton.accept_state = state_elimination_automaton.new_state(); - for accept_state in automaton.get_accept_states() { - let accept_state = *states_map.get(accept_state).unwrap(); - // We add an empty string transition to the new accept state - state_elimination_automaton.add_transition_to( - accept_state, - state_elimination_automaton.accept_state, - GraphTransition::Epsilon, - ); - } - } - state_elimination_automaton.identify_and_apply_components()?; - //state_elimination_automaton.to_dot(); - Ok(Some(state_elimination_automaton)) - } - - pub fn new_state(&mut self) -> usize { - if let Some(new_state) = self.removed_states.clone().iter().next() { - self.removed_states.remove(new_state); - self.transitions_in.insert(*new_state, IntSet::new()); - *new_state - } else { - self.transitions.push(IntMap::default()); - self.transitions_in - .insert(self.transitions.len() - 1, IntSet::new()); - self.transitions.len() - 1 - } - } - - #[inline] - pub fn has_state(&self, state: State) -> bool { - !(state >= self.transitions.len() || self.removed_states.contains(&state)) - } - - #[inline] - fn assert_state_exists(&self, state: State) { - if !self.has_state(state) { - panic!("The state {} does not exist", state); - } - } - - pub fn add_transition_to( - &mut self, - from_state: State, - to_state: State, - transition: GraphTransition, - ) { - self.assert_state_exists(from_state); - if from_state != to_state { - self.assert_state_exists(to_state); - } - - self.transitions_in - .entry(to_state) - .or_default() - .insert(from_state); - match self.transitions[from_state].entry(to_state) { - Entry::Occupied(mut o) => { - if let (GraphTransition::Weight(current_regex), GraphTransition::Weight(regex)) = - (o.get(), transition) - { - o.insert(GraphTransition::Weight(current_regex.union(®ex))); - } else { - panic!("Cannot add transition"); - } - } - Entry::Vacant(v) => { - v.insert(transition); - } - }; - } - - pub fn remove_state(&mut self, state: State) { - self.assert_state_exists(state); - if self.start_state == state || self.accept_state == state { - panic!( - "Can not remove the state {}, it is still used as start state or accept state.", - state - ); - } - self.transitions_in.remove(&state); - if self.transitions.len() - 1 == state { - self.transitions.remove(state); - - let mut s = state; - while self.removed_states.contains(&s) { - self.transitions.remove(s); - self.removed_states.remove(&s); - s -= 1; - } - } else { - self.transitions[state].clear(); - self.removed_states.insert(state); - } - - for transitions in self.transitions.iter_mut() { - transitions.remove(&state); - } - for (_, transitions) in self.transitions_in.iter_mut() { - transitions.remove(&state); - } - } - - pub fn remove_transition(&mut self, from_state: State, to_state: State) { - self.assert_state_exists(from_state); - if from_state != to_state { - self.assert_state_exists(to_state); - } - - if let Some(from_states) = self.transitions_in.get_mut(&to_state) { - from_states.remove(&from_state); - } - - self.transitions[from_state].remove(&to_state); - } - - pub fn get_transition(&self, from_state: State, to_state: State) -> Option<&GraphTransition> { - self.transitions.get(from_state)?.get(&to_state) - } -} diff --git a/src/fast_automaton/convert/to_regex/builder/scc.rs b/src/fast_automaton/convert/to_regex/builder/scc.rs deleted file mode 100644 index 815188a..0000000 --- a/src/fast_automaton/convert/to_regex/builder/scc.rs +++ /dev/null @@ -1,207 +0,0 @@ -use super::*; - -impl StateEliminationAutomaton { - pub fn identify_and_apply_components(&mut self) -> Result<(), EngineError> { - let mut index = 0; - let mut stack = Vec::new(); - let mut indices = vec![-1; self.transitions.len()]; - let mut lowlink = vec![-1; self.transitions.len()]; - let mut on_stack = vec![false; self.transitions.len()]; - let mut scc = Vec::new(); - - for state in self.states_iter() { - if self.removed_states.contains(&state) { - continue; - } - if indices[state] == -1 { - self.strongconnect( - state, - &mut index, - &mut stack, - &mut indices, - &mut lowlink, - &mut on_stack, - &mut scc, - ); - } - } - - let scc = scc - .into_iter() - .filter(|states| { - let first_state = states.iter().next().unwrap(); - let self_loop = if let Some(transitions_in) = self.transitions_in.get(first_state) { - transitions_in.contains(first_state) - } else { - false - }; - states.len() != 1 || self_loop - }) - .collect::>(); - - for component in scc { - self.build_component(&component)?; - } - - self.cyclic = false; - - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - fn strongconnect( - &self, - v: usize, - index: &mut usize, - stack: &mut Vec, - indices: &mut Vec, - lowlink: &mut Vec, - on_stack: &mut Vec, - scc: &mut Vec>, - ) { - indices[v] = *index as i32; - lowlink[v] = *index as i32; - *index += 1; - stack.push(v); - on_stack[v] = true; - - if let Some(neighbors) = self.transitions.get(v) { - for &w in neighbors.keys() { - if indices[w] == -1 { - self.strongconnect(w, index, stack, indices, lowlink, on_stack, scc); - lowlink[v] = lowlink[v].min(lowlink[w]); - } else if on_stack[w] { - lowlink[v] = lowlink[v].min(indices[w]); - } - } - } - - if lowlink[v] == indices[v] { - let mut component = Vec::new(); - while let Some(w) = stack.pop() { - on_stack[w] = false; - component.push(w); - if w == v { - break; - } - } - scc.push(component); - } - } - - fn build_component(&mut self, states: &[usize]) -> Result<(), EngineError> { - let state_set = states.iter().copied().collect::>(); - let mut start_states = IntMap::new(); - let mut accept_states = IntMap::new(); - - let mut state_elimination_automaton = StateEliminationAutomaton { - start_state: 0, // start_state is not set yet - accept_state: 0, // accept_state is not set yet - transitions: Vec::with_capacity(states.len()), - transitions_in: IntMap::with_capacity(states.len()), - removed_states: IntSet::new(), - cyclic: true, - }; - - let mut states_map = IntMap::with_capacity(states.len()); - for from_state in states { - if *from_state == self.accept_state { - self.accept_state = self.new_state(); - self.add_transition_to(*from_state, self.accept_state, GraphTransition::Epsilon); - } - if *from_state == self.start_state { - self.start_state = self.new_state(); - self.add_transition_to(self.start_state, *from_state, GraphTransition::Epsilon); - } - let from_state_new = *states_map - .entry(*from_state) - .or_insert_with(|| state_elimination_automaton.new_state()); - for (to_state, transition) in self.transitions_from_state_enumerate_iter(from_state) { - if !state_set.contains(to_state) { - accept_states - .entry(*to_state) - .or_insert_with(Vec::new) - .push((from_state_new, transition.clone())); - continue; - } - - let to_state_new = *states_map - .entry(*to_state) - .or_insert_with(|| state_elimination_automaton.new_state()); - - state_elimination_automaton.add_transition_to( - from_state_new, - to_state_new, - transition.clone(), - ); - } - - for (parent_state, transition) in self.in_transitions_vec(*from_state) { - if !state_set.contains(&parent_state) { - start_states - .entry(from_state_new) - .or_insert_with(Vec::new) - .push((parent_state, transition.clone())); - } - } - } - - for state in states { - self.remove_state(*state); - } - - for (start_state, parent_states) in &start_states { - for (parent_state, transition) in parent_states { - let new_parent_state = if !transition.is_empty_string() { - let new_parent_state = self.new_state(); - - self.add_transition_to(*parent_state, new_parent_state, transition.clone()); - new_parent_state - } else { - *parent_state - }; - for (target_state, accept_states_transition) in &accept_states { - let mut new_automaton = state_elimination_automaton.clone(); - - let target_state = if accept_states_transition.len() > 1 { - new_automaton.accept_state = new_automaton.new_state(); - for (accept_state, transition) in accept_states_transition { - new_automaton.add_transition_to( - *accept_state, - new_automaton.accept_state, - transition.clone(), - ); - } - *target_state - } else { - let (accept_state, transition) = - accept_states_transition.iter().next().unwrap(); - - new_automaton.accept_state = *accept_state; - if !transition.is_empty_string() { - let new_target_state = self.new_state(); - self.add_transition_to( - new_target_state, - *target_state, - transition.clone(), - ); - new_target_state - } else { - *target_state - } - }; - - new_automaton.start_state = *start_state; - - self.add_transition_to( - new_parent_state, - target_state, - GraphTransition::Graph(new_automaton), - ); - } - } - } - - Ok(()) - } -} diff --git a/src/fast_automaton/convert/to_regex/mod.rs b/src/fast_automaton/convert/to_regex/mod.rs index d9a1dd0..a44e8d1 100644 --- a/src/fast_automaton/convert/to_regex/mod.rs +++ b/src/fast_automaton/convert/to_regex/mod.rs @@ -1,297 +1,31 @@ -use std::{ - collections::{hash_map::Entry, VecDeque}, - fmt::Display, -}; +use super::*; -use ahash::{HashMapExt, HashSetExt}; -use log::warn; -use nohash_hasher::IntMap; - -use crate::{error::EngineError, execution_profile::ThreadLocalParams, regex::RegularExpression}; - -use super::{FastAutomaton, IntSet, Range, State}; - -mod builder; -mod transform; - -#[derive(Clone, Debug)] -enum GraphTransition { - Graph(StateEliminationAutomaton), - Weight(T), - Epsilon, -} - -impl GraphTransition { - pub fn is_empty_string(&self) -> bool { - matches!(self, GraphTransition::Epsilon) - } - - pub fn get_weight(&self) -> Option<&T> { - if let GraphTransition::Weight(weight) = self { - Some(weight) - } else { - None - } - } -} - -#[derive(Clone, Debug)] -struct StateEliminationAutomaton { - start_state: usize, - accept_state: usize, - transitions: Vec>>, - transitions_in: IntMap>, - removed_states: IntSet, - cyclic: bool, -} - -impl Display for StateEliminationAutomaton { - fn fmt(&self, sb: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.to_graph_dot(sb, None) - } -} - -impl StateEliminationAutomaton { - //#[cfg(test)] - #[allow(dead_code)] - #[inline] - pub fn to_dot(&self) { - println!("{}", self); - } - - #[inline] - fn to_graph_dot( - &self, - sb: &mut std::fmt::Formatter<'_>, - prefix: Option<&str>, - ) -> std::fmt::Result { - let is_subgraph; - let indent; - let prefix = if let Some(prefix) = prefix { - writeln!(sb, "\tsubgraph cluster_{} {{", prefix)?; - writeln!(sb, "\t\tlabel = \"{} - cyclic={}\";", prefix, self.cyclic)?; - indent = "\t"; - is_subgraph = true; - prefix - } else { - writeln!(sb, "digraph Automaton {{")?; - writeln!(sb, "\trankdir = LR;")?; - writeln!(sb, "\tlabel = \"cyclic={}\";", self.cyclic)?; - indent = ""; - is_subgraph = false; - "" - }; - - for from_state in self.states_iter() { - let from_state_with_prefix = if is_subgraph { - format!("S{prefix}_{from_state}") - } else { - format!("S{from_state}") - }; - - write!(sb, "{indent}\t{}", from_state_with_prefix)?; - if !is_subgraph && self.accept_state == from_state { - writeln!(sb, "\t[shape=doublecircle,label=\"{}\"];", from_state)?; - } else { - writeln!(sb, "{indent}\t[shape=circle,label=\"{}\"];", from_state)?; - } - - if !is_subgraph && self.start_state == from_state { - writeln!(sb, "\tinitial [shape=plaintext,label=\"\"];")?; - writeln!(sb, "\tinitial -> {}", from_state_with_prefix)?; - } - for (to_state, weight) in self.transitions_from_state_enumerate_iter(&from_state) { - let to_state_with_prefix = if is_subgraph { - format!("S{prefix}_{to_state}") - } else { - format!("S{to_state}") - }; - - match weight { - GraphTransition::Graph(state_elimination_automaton) => { - let subgraph_prefix = if is_subgraph { - format!("{prefix}_{from_state}_{to_state}") - } else { - format!("{from_state}_{to_state}") - }; - state_elimination_automaton.to_graph_dot(sb, Some(&subgraph_prefix))?; - writeln!(sb)?; - let subgraph_start_state = format!( - "S{}_{}", - subgraph_prefix, state_elimination_automaton.start_state - ); - writeln!( - sb, - "{indent}\t{} -> {} [label=\"ε\"]", - from_state_with_prefix, subgraph_start_state - )?; - - let subgraph_accept_state = format!( - "S{}_{}", - subgraph_prefix, state_elimination_automaton.accept_state - ); - writeln!( - sb, - "{indent}\t{} -> {} [label=\"ε\"]", - subgraph_accept_state, to_state_with_prefix - ) - } - GraphTransition::Weight(range) => { - writeln!( - sb, - "{indent}\t{} -> {} [label=\"{}\"]", - from_state_with_prefix, - to_state_with_prefix, - RegularExpression::Character(range.clone()) - .to_string() - .replace('\\', "\\\\") - .replace('"', "\\\"") - ) - } - GraphTransition::Epsilon => writeln!( - sb, - "{indent}\t{} -> {} [label=\"ε\"]", - from_state_with_prefix, to_state_with_prefix - ), - }?; - } - } - write!(sb, "{indent}}}") - } - - #[inline] - pub fn states_iter(&self) -> impl Iterator + '_ { - (0..self.transitions.len()).filter(|s| !self.removed_states.contains(s)) - } - - #[inline] - pub fn transitions_from_state_enumerate_iter( - &self, - from_state: &State, - ) -> impl Iterator)> { - self.transitions[*from_state] - .iter() - .filter(|s| !self.removed_states.contains(s.0)) - } - - #[inline] - pub fn transitions_from_state_vec(&self, from_state: &State) -> Vec { - self.transitions[*from_state] - .keys() - .filter(|s| !self.removed_states.contains(s)) - .copied() - .collect() - } - - pub fn in_transitions_vec(&self, to_state: State) -> Vec<(State, GraphTransition)> { - let mut in_transitions = vec![]; - for from_state in self.transitions_in.get(&to_state).unwrap_or(&IntSet::new()) { - for (state, transition) in self.transitions_from_state_enumerate_iter(from_state) { - if to_state == *state { - in_transitions.push((*from_state, transition.clone())); - } - } - } - in_transitions - } - - pub fn states_topo_vec(&self) -> Vec { - if self.cyclic { - panic!("The graph has a cycle"); - } - - let mut in_degree: IntMap = self - .transitions_in - .iter() - .map(|(state, parents)| (*state, parents.len())) - .collect(); - - let mut worklist: VecDeque = VecDeque::new(); - for (&state, °ree) in &in_degree { - if degree == 0 { - worklist.push_back(state); - } - } - - let mut sorted_order = Vec::with_capacity(self.get_number_of_states()); - while let Some(state) = worklist.pop_front() { - sorted_order.push(state); - - if let Some(neighbors) = self.transitions.get(state) { - let neighbors = neighbors.keys(); - for &neighbor in neighbors { - if let Some(degree) = in_degree.get_mut(&neighbor) { - *degree -= 1; - if *degree == 0 { - worklist.push_back(neighbor); - } - } - } - } - } - - if sorted_order.len() == self.get_number_of_states() { - sorted_order - } else { - panic!("The graph has a cycle"); - } - } - - #[inline] - pub fn get_number_of_states(&self) -> usize { - self.transitions.len() - self.removed_states.len() - } -} +mod state_elimination; impl FastAutomaton { - /// Try to convert the current FastAutomaton to a RegularExpression. - /// If it cannot find an equivalent regex it returns None. - /// This method is still a work in progress. - pub fn to_regex(&self) -> Option { - if self.is_empty() { - return Some(RegularExpression::new_empty()); - } - let execution_profile = ThreadLocalParams::get_execution_profile(); - if let Ok(graph) = StateEliminationAutomaton::new(self) { - if let Ok(regex) = graph?.convert_to_regex(&execution_profile) { - let regex = regex?; - match regex.to_automaton() { - Ok(automaton) => match self.is_equivalent_of(&automaton) { - Ok(result) => { - if !result { - warn!("The automaton is not equivalent to the generated regex; automaton={}, regex={}", self, regex); - None - } else { - Some(regex) - } - } - Err(err) => { - warn!("Engine error while checking for equivalence ({}); automaton={}, regex={}", err, self, regex); - None - } - }, - Err(err) => { - if let crate::error::EngineError::RegexSyntaxError(err) = err { - warn!("The generated regex cannot be converted to automaton to be checked for equivalence ({}); automaton={}, regex={}", err, self, regex); - } - None - } - } - } else { - None - } - } else { - None - } + /// Converts the automaton to a [`RegularExpression`]. + #[tracing::instrument(level = "debug", skip_all, fields(states = self.number_of_states()))] + pub fn to_regex(&self) -> Result { + state_elimination::convert_to_regex(self) } } #[cfg(test)] mod tests { + use ::regex::Regex; + use super::*; #[test] fn test_convert() -> Result<(), String> { + assert_convert(".*u(ab|de)"); + assert_convert(".*sf.*uif(ab|de)"); + + assert_convert("(a+|,)*"); + assert_convert("((ab)*,(cd)*)*"); + assert_convert("(a*,a*,a*)*"); + assert_convert("(a*,a*)*"); + assert_convert("(ac|ads|a)*"); assert_convert(".*sf"); assert_convert(".*sf.*uif(ab|de)"); @@ -325,45 +59,40 @@ mod tests { } fn assert_convert(regex: &str) { - let input_regex = RegularExpression::new(regex).unwrap(); + let input_regex = RegularExpression::parse(regex, false).unwrap(); println!("IN : {}", input_regex); let input_automaton = input_regex.to_automaton().unwrap(); - //input_automaton.to_dot(); - let output_regex = input_automaton.to_regex().unwrap(); println!("OUT (non deterministic): {}", output_regex); let output_automaton = output_regex.to_automaton().unwrap(); - assert!(input_automaton.is_equivalent_of(&output_automaton).unwrap()); + assert!(input_automaton.equivalent(&output_automaton).unwrap()); let input_automaton = input_automaton.determinize().unwrap(); - //input_automaton.to_dot(); - let output_regex = input_automaton.to_regex().unwrap(); println!("OUT (deterministic) : {}", output_regex); let output_automaton = output_regex.to_automaton().unwrap(); - assert!(input_automaton.is_equivalent_of(&output_automaton).unwrap()); + assert!(input_automaton.equivalent(&output_automaton).unwrap()); } #[test] fn test_convert_after_operation_1() -> Result<(), String> { - let automaton1 = RegularExpression::new("(ab|cd)") + let automaton1 = RegularExpression::parse("(ab|cd)", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("ab") + let automaton2 = RegularExpression::parse("ab", false) .unwrap() .to_automaton() - .unwrap() - .determinize() .unwrap(); + let automaton2 = automaton2.determinize().unwrap(); - let result = automaton1.subtraction(&automaton2).unwrap(); + let result = automaton1.difference(&automaton2).unwrap(); - result.to_dot(); + result.print_dot(); let output_regex = result.to_regex().unwrap(); assert_eq!("cd", output_regex.to_string()); @@ -373,18 +102,18 @@ mod tests { #[test] fn test_convert_after_operation_2() -> Result<(), String> { - let automaton1 = RegularExpression::new("a*") + let automaton1 = RegularExpression::parse("a*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("b*") + let automaton2 = RegularExpression::parse("b*", false) .unwrap() .to_automaton() .unwrap(); let result = automaton1.intersection(&automaton2).unwrap(); - result.to_dot(); + result.print_dot(); let output_regex = result.to_regex().unwrap(); assert_eq!("", output_regex.to_string()); @@ -394,34 +123,33 @@ mod tests { #[test] fn test_convert_after_operation_3() -> Result<(), String> { - let automaton1 = RegularExpression::new("x*") + let automaton1 = RegularExpression::parse("x*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("(xxx)*") + let automaton2 = RegularExpression::parse("(xxx)*", false) .unwrap() .to_automaton() - .unwrap() - .determinize() .unwrap(); + let automaton2 = automaton2.determinize().unwrap(); - let result = automaton1.subtraction(&automaton2).unwrap(); - result.to_dot(); + let result = automaton1.difference(&automaton2).unwrap(); + result.print_dot(); let result = result.to_regex().unwrap(); - assert_eq!("(x{3})*x{1,2}", result.to_string()); + assert_eq!("x(x{3})*x?", result.to_string()); Ok(()) } #[test] fn test_convert_after_operation_4() -> Result<(), String> { - let automaton1 = RegularExpression::new(".*abc.*") + let automaton1 = RegularExpression::parse(".*abc.*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new(".*def.*") + let automaton2 = RegularExpression::parse(".*def.*", false) .unwrap() .to_automaton() .unwrap(); @@ -435,31 +163,57 @@ mod tests { Ok(()) } - /*#[test] + #[test] fn test_convert_after_operation_5() -> Result<(), String> { - if std::env::var_os("RUST_LOG").is_none() { - std::env::set_var("RUST_LOG", "regexsolver=debug"); - } - env_logger::init(); - - let automaton1 = RegularExpression::new(".*abc.*") + let automaton = RegularExpression::parse(".*abc.*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new(".*def.*") + let mut automaton = automaton.determinize().unwrap().into_owned(); + + automaton.complement().unwrap(); + + let result = format!("^{}$", automaton.to_regex().unwrap()); + + println!("{result}"); + + let result = Regex::new(&result).unwrap(); + + assert!(!result.is_match("abc")); + assert!(!result.is_match("2374abc012")); + + assert!(result.is_match("bc")); + assert!(result.is_match("237a4bc012")); + + Ok(()) + } + + #[test] + fn test_automaton() -> Result<(), String> { + let automaton = RegularExpression::parse("a*ba*", false) .unwrap() .to_automaton() + .unwrap(); + automaton.print_dot(); + + let automaton1 = RegularExpression::parse("(a*ba*)*", false) .unwrap() - .determinize() + .to_automaton() .unwrap(); + automaton1.print_dot(); - let result = automaton1.subtraction(&automaton2).unwrap(); - result.to_dot(); + automaton1.determinize().unwrap().print_dot(); - let result = result.to_regex().unwrap(); + // (a*b[ab]*)? + // a*b+a+b+ - assert_eq!("(x{3})*x{1,2}", result.to_string()); + let automaton2 = RegularExpression::parse("(a*b[ab]*)?", false) + .unwrap() + .to_automaton() + .unwrap(); + + assert!(automaton1.equivalent(&automaton2).unwrap()); Ok(()) - }*/ + } } diff --git a/src/fast_automaton/convert/to_regex/state_elimination/builder.rs b/src/fast_automaton/convert/to_regex/state_elimination/builder.rs new file mode 100644 index 0000000..568e9ae --- /dev/null +++ b/src/fast_automaton/convert/to_regex/state_elimination/builder.rs @@ -0,0 +1,171 @@ +use ahash::HashMapExt; + +use super::*; + +impl Gnfa { + pub(super) fn from_automaton(automaton: &FastAutomaton) -> Result { + let mut state_elimination_automaton = Gnfa { + start_state: 0, // start_state is not set yet + accept_state: 0, // accept_state is not set yet + transitions: Vec::with_capacity(automaton.number_of_states()), + transitions_in: IntMap::with_capacity(automaton.number_of_states()), + removed_states: IntSet::with_capacity(automaton.number_of_states()), + empty: false, + }; + + if automaton.is_empty() { + state_elimination_automaton.empty = true; + return Ok(state_elimination_automaton); + } + + let mut states_map = IntMap::with_capacity(automaton.number_of_states()); + + for from_state in automaton.states() { + let new_from_state = *states_map + .entry(from_state) + .or_insert_with(|| state_elimination_automaton.new_state()); + for (condition, to_state) in automaton.transitions_from(from_state) { + let new_to_state = *states_map + .entry(*to_state) + .or_insert_with(|| state_elimination_automaton.new_state()); + + let range = condition.to_range(automaton.spanning_set())?; + state_elimination_automaton.add_transition( + new_from_state, + new_to_state, + RegularExpression::Character(range), + ); + } + } + + if automaton.in_degree(automaton.start_state()) == 0 { + // If the start state does not have any incoming state we just set it + state_elimination_automaton.start_state = + *states_map.get(&automaton.start_state()).unwrap(); + } else { + // If not we create a new state that will be the new start state + state_elimination_automaton.start_state = state_elimination_automaton.new_state(); + + let previous_start_state = *states_map.get(&automaton.start_state()).unwrap(); + // We add an empty string transition to the new start state + state_elimination_automaton.add_transition( + state_elimination_automaton.start_state, + previous_start_state, + RegularExpression::new_empty_string(), + ); + } + + let accept_state = *automaton.accept_states().iter().next().unwrap(); + if automaton.accept_states().len() == 1 && automaton.out_degree(accept_state) == 0 { + // If there is only one accept state we just set it + state_elimination_automaton.accept_state = *states_map + .get(automaton.accept_states().iter().next().unwrap()) + .unwrap(); + } else { + // If not we create a new state that will be the new accept state + state_elimination_automaton.accept_state = state_elimination_automaton.new_state(); + for accept_state in automaton.accept_states() { + let accept_state = *states_map.get(accept_state).unwrap(); + // We add an empty string transition to the new accept state + state_elimination_automaton.add_transition( + accept_state, + state_elimination_automaton.accept_state, + RegularExpression::new_empty_string(), + ); + } + } + + Ok(state_elimination_automaton) + } + + fn new_state(&mut self) -> usize { + if let Some(&new_state) = self.removed_states.iter().next() { + self.removed_states.remove(&new_state); + self.transitions_in.insert(new_state, IntSet::new()); + new_state + } else { + self.transitions.push(IntMap::default()); + self.transitions_in + .insert(self.transitions.len() - 1, IntSet::new()); + self.transitions.len() - 1 + } + } + + #[inline] + pub(super) fn has_state(&self, state: State) -> bool { + !(state >= self.transitions.len() || self.removed_states.contains(&state)) + } + + #[inline] + fn assert_state_exists(&self, state: State) { + if !self.has_state(state) { + panic!("The state {state} does not exist"); + } + } + + pub(crate) fn add_transition( + &mut self, + from_state: State, + to_state: State, + transition: RegularExpression, + ) { + self.assert_state_exists(from_state); + if from_state != to_state { + self.assert_state_exists(to_state); + } + + self.transitions_in + .entry(to_state) + .or_default() + .insert(from_state); + match self.transitions[from_state].entry(to_state) { + Entry::Occupied(mut o) => { + let merged = transition.union(o.get()); + *o.get_mut() = merged; + } + Entry::Vacant(v) => { + v.insert(transition); + } + }; + } + + pub(super) fn remove_state(&mut self, state: State) { + self.assert_state_exists(state); + if self.start_state == state || self.accept_state == state { + panic!( + "Can not remove the state {state}, it is still used as start state or accept state." + ); + } + // Snapshot the exact predecessor and successor sets before the maps + // below are cleared; only their entries need updating. + let predecessors = self.transitions_in.remove(&state).unwrap_or_default(); + let successors: Vec = self.transitions[state].keys().copied().collect(); + + if self.transitions.len() - 1 == state { + self.transitions.remove(state); + + // Compact tombstones that are now trailing; see + // `FastAutomaton::remove_state`. + let mut s = state; + while s > 0 && self.removed_states.contains(&(s - 1)) { + s -= 1; + self.transitions.remove(s); + self.removed_states.remove(&s); + } + } else { + self.transitions[state].clear(); + self.removed_states.insert(state); + } + + for &from_state in &predecessors { + if let Some(transitions) = self.transitions.get_mut(from_state) { + transitions.remove(&state); + } + } + for to_state in successors { + if let Some(transitions_in) = self.transitions_in.get_mut(&to_state) { + transitions_in.remove(&state); + } + } + } +} diff --git a/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs b/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs new file mode 100644 index 0000000..92d49cf --- /dev/null +++ b/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs @@ -0,0 +1,180 @@ +use super::*; + +impl Gnfa { + pub(super) fn convert(&mut self) -> Result { + if self.empty { + return Ok(RegularExpression::new_empty()); + } + + let execution_profile = crate::execution_profile::ExecutionProfile::get(); + + // Cached elimination scores, indexed by state. Ids are stable during + // elimination (no state is created; tombstone compaction only pops + // trailing, already-removed entries), and `None` marks + // non-candidates: start, accept, and eliminated states. + // + // A state's score depends only on its own in/out edges, and + // eliminating `k` only touches edges incident to k's predecessors + // and successors — so exactly those need re-scoring each round. + let mut scores: Vec> = vec![None; self.transitions.len()]; + for state in self.all_states_iter() { + if state != self.start_state && state != self.accept_state { + scores[state] = Some(self.score_state(state)); + } + } + + loop { + execution_profile.assert_not_timed_out()?; + + // Lowest score wins; `<=` keeps the last minimal state on ties + // (the order `convert_reference` pins). + let mut best: Option<(u128, usize)> = None; + for (state, &score) in scores.iter().enumerate() { + if let Some(score) = score + && best.is_none_or(|(best_score, _)| score <= best_score) + { + best = Some((score, state)); + } + } + let Some((_, k)) = best else { + break; + }; + + scores[k] = None; + let (predecessors, successors) = self.eliminate_state(k); + for state in predecessors.into_iter().chain(successors) { + if scores[state].is_some() { + scores[state] = Some(self.score_state(state)); + } + } + } + + // Moved out, not cloned: the `Gnfa` is discarded right after. + Ok(self + .transitions + .get_mut(self.start_state) + .and_then(|transitions| transitions.remove(&self.accept_state)) + .unwrap_or_else(RegularExpression::new_empty_string)) + } + + /// The elimination score. It reads only `state`'s own degrees and label + /// complexities, which is what makes the cached, neighbors-only + /// re-scoring in [`convert`](Self::convert) sound. + fn score_state(&self, state: usize) -> u128 { + let mut in_deg: u128 = 0; + let mut label_cost: u128 = 0; + for (_, regex) in self.transitions_to_iter(state) { + in_deg += 1; + label_cost += regex.evaluate_complexity() as u128; + } + let mut out_deg: u128 = 0; + for (regex, _) in self.transitions_from_iter(state) { + out_deg += 1; + label_cost += regex.evaluate_complexity() as u128; + } + + if in_deg == 0 || out_deg == 0 { + return (state as u128) & 0xFF; + } + + let mut score: u128 = in_deg * out_deg; + + if self.has_self_loop(state) { + score = score + (score >> 1); + } + + if let Some(re) = self.get_transition(state, state) { + label_cost += (re.evaluate_complexity() as u128) * 2; + } + + score = score.saturating_add(label_cost); + + let tie = (state as u128) & 0xFFFF; + score.saturating_add(tie) + } + + /// Bridges every predecessor to every successor and removes `k`, + /// returning those predecessors and successors (`k` excluded): the only + /// states whose elimination scores the operation changed. + fn eliminate_state(&mut self, k: usize) -> (Vec, Vec) { + if self.removed_states.contains(&k) { + return (vec![], vec![]); + } + + let in_states = self + .transitions_in + .get(&k) + .unwrap() + .iter() + .cloned() + .filter(|&s| s != k) + .collect::>(); + let out_states = self.transitions[k] + .keys() + .cloned() + .filter(|&s| s != k) + .collect::>(); + + // The k→k self-loop star is the same for every (p, q) pair, and + // bridging never touches the (k, k) edge, so build it once. + let star = self + .get_transition(k, k) + .map(|self_loop| self_loop.repeat(0, None)); + + for &p in &in_states { + for &q in &out_states { + self.bridge(p, k, q, star.as_ref()); + } + } + + self.remove_state(k); + + (in_states, out_states) + } + + fn bridge(&mut self, p: usize, k: usize, q: usize, star: Option<&RegularExpression>) { + let rpk = self.get_transition(p, k); + let rkq = self.get_transition(k, q); + + if let (Some(rpk), Some(rkq)) = (rpk, rkq) { + let mut regex = rpk.clone(); + if let Some(star) = star { + regex = regex.concat(star, true); + } + regex = regex.concat(rkq, true); + self.add_transition(p, q, regex); + } + } +} + +#[cfg(test)] +impl Gnfa { + /// [`convert`](Self::convert) with the score cache disabled: every + /// candidate is re-scored from scratch each round, with the serial fold + /// (last minimal state wins on ties) the cached version replaced. The + /// oracle proving the cache never yields a stale score — i.e. the + /// produced pattern is identical to the pre-cache implementation's. + pub(super) fn convert_reference(&mut self) -> Result { + if self.empty { + return Ok(RegularExpression::new_empty()); + } + + loop { + let best = self + .all_states_iter() + .filter(|&s| s != self.start_state && s != self.accept_state) + .map(|state| (self.score_state(state), state)) + .reduce(|a, b| if a.0 < b.0 { a } else { b }); + let Some((_, state)) = best else { + break; + }; + self.eliminate_state(state); + } + + Ok(self + .transitions + .get_mut(self.start_state) + .and_then(|transitions| transitions.remove(&self.accept_state)) + .unwrap_or_else(RegularExpression::new_empty_string)) + } +} diff --git a/src/fast_automaton/convert/to_regex/state_elimination/mod.rs b/src/fast_automaton/convert/to_regex/state_elimination/mod.rs new file mode 100644 index 0000000..dcf43f9 --- /dev/null +++ b/src/fast_automaton/convert/to_regex/state_elimination/mod.rs @@ -0,0 +1,232 @@ +use super::*; + +mod builder; +mod eliminate; + +struct Gnfa { + start_state: usize, + accept_state: usize, + transitions: Vec>, + transitions_in: IntMap>, + removed_states: IntSet, + empty: bool, +} + +impl Display for Gnfa { + fn fmt(&self, sb: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(sb, "digraph GNFA {{")?; + writeln!(sb, "\trankdir = LR;")?; + for from_state in self.all_states_iter() { + write!(sb, "\t{from_state}")?; + if self.accept_state == from_state { + writeln!(sb, "\t[shape=doublecircle,label=\"{from_state}\"];")?; + } else { + writeln!(sb, "\t[shape=circle,label=\"{from_state}\"];")?; + } + + if self.start_state == from_state { + writeln!(sb, "\tinitial [shape=plaintext,label=\"\"];")?; + writeln!(sb, "\tinitial -> {from_state}")?; + } + for (regex, to_state) in self.transitions_from_iter(from_state) { + writeln!(sb, "\t{from_state} -> {to_state} [label=\"{regex}\"]")?; + } + } + write!(sb, "}}") + } +} + +impl Gnfa { + fn get_transition(&self, from_state: State, to_state: State) -> Option<&RegularExpression> { + self.transitions.get(from_state)?.get(&to_state) + } + + #[inline] + fn all_states_iter(&self) -> impl Iterator + '_ { + (0..self.transitions.len()).filter(|s| !self.removed_states.contains(s)) + } + + /// Incoming transitions by reference: `transitions_in` gives the exact + /// predecessor set, so each edge label is one direct map lookup. + fn transitions_to_iter( + &self, + state: State, + ) -> impl Iterator { + self.transitions_in + .get(&state) + .into_iter() + .flatten() + .filter_map(move |&from_state| { + self.get_transition(from_state, state) + .map(|regex| (from_state, regex)) + }) + } + + /// Outgoing transitions by reference. + #[inline] + fn transitions_from_iter( + &self, + state: State, + ) -> impl Iterator { + self.transitions[state] + .iter() + .filter(|(s, _)| !self.removed_states.contains(*s)) + .map(|(s, c)| (c, *s)) + } + + #[inline] + fn has_self_loop(&self, state: State) -> bool { + self.get_transition(state, state).is_some() + } +} + +pub(super) fn convert_to_regex( + automaton: &FastAutomaton, +) -> Result { + let mut gnfa = Gnfa::from_automaton(automaton)?; + gnfa.convert() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CharRange; + use proptest::prelude::*; + use regex_charclass::char::Char; + + /// Every produced pattern must be byte-identical to what full + /// re-scoring produces: the score cache must never go stale. + fn assert_same_pattern_as_reference(automaton: &FastAutomaton) { + let incremental = Gnfa::from_automaton(automaton).unwrap().convert().unwrap(); + let reference = Gnfa::from_automaton(automaton) + .unwrap() + .convert_reference() + .unwrap(); + assert_eq!( + incremental.to_string(), + reference.to_string(), + "incremental scoring changed the elimination order" + ); + } + + #[test] + fn incremental_scoring_matches_full_rescoring() { + let patterns = [ + "abc", + ".*de", + "(a*ba*)*", + ".*u(ab|de)", + "a(bcfe|bcdg|mkv)*(abc){2,3}", + "(aad|ads|a)*abc.*def(x|q)", + "(a|b)*a(a|b){3}", + "[a-z]{1,6}", + "(ab|xy){2}", + "x*|(xxx)*|y", + "a{0,3}b{2}(c|d)?", + ]; + for pattern in patterns { + let automaton = RegularExpression::new(pattern) + .unwrap() + .to_automaton() + .unwrap(); + assert_same_pattern_as_reference(&automaton); + assert_same_pattern_as_reference(&automaton.determinize().unwrap()); + } + } + + /// A small palette of character ranges for random automata. + fn palette(index: usize) -> CharRange { + let bounds = [ + ('a', 'a'), + ('b', 'b'), + ('c', 'c'), + ('a', 'c'), + ('b', 'd'), + ('x', 'z'), + ('\u{0}', '\u{10FFFF}'), + ]; + let (low, high) = bounds[index % bounds.len()]; + CharRange::new_from_range(Char::new(low)..=Char::new(high)) + } + + fn arb_automaton() -> impl Strategy { + ( + 2usize..7, + proptest::collection::vec((0usize..6, 0usize..6, 0usize..7), 1..15), + 0u8..=255, + ) + .prop_map(|(number_of_states, edges, accept_mask)| { + let mut automaton = FastAutomaton::new_empty(); + for _ in 1..number_of_states { + automaton.new_state(); + } + for (from, to, range) in edges { + automaton + .add_transition_from_range( + from % number_of_states, + to % number_of_states, + &palette(range), + ) + .unwrap(); + } + for state in 0..number_of_states { + if accept_mask & (1 << state) != 0 { + automaton.accept(state); + } + } + automaton + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + #[test] + fn incremental_scoring_matches_full_rescoring_on_random_automata( + automaton in arb_automaton() + ) { + let incremental = Gnfa::from_automaton(&automaton).unwrap().convert().unwrap(); + let reference = Gnfa::from_automaton(&automaton) + .unwrap() + .convert_reference() + .unwrap(); + prop_assert_eq!(incremental.to_string(), reference.to_string()); + } + } + + #[test] + fn test_state_elimination() -> Result<(), String> { + test_correct("abc"); + test_correct(".*de"); + test_correct(".*def"); + test_correct("(a*ba*)*"); + test_correct(".*u(ab|d)"); + test_correct(".*u(ab|de)"); + Ok(()) + } + + fn test_correct(pattern: &str) { + println!("Pattern: {pattern}"); + + let automaton = RegularExpression::new(pattern) + .unwrap() + .to_automaton() + .unwrap(); + + let regex = Gnfa::from_automaton(&automaton).unwrap().convert().unwrap(); + println!("-> {regex}"); + + let new_automaton = regex.to_automaton().unwrap(); + + assert!(automaton.equivalent(&new_automaton).unwrap()); + + let automaton = automaton.determinize().unwrap().into_owned(); + + let regex = Gnfa::from_automaton(&automaton).unwrap().convert().unwrap(); + println!("-> {regex}"); + + let new_automaton = regex.to_automaton().unwrap(); + + assert!(automaton.equivalent(&new_automaton).unwrap()); + } +} diff --git a/src/fast_automaton/convert/to_regex/transform.rs b/src/fast_automaton/convert/to_regex/transform.rs deleted file mode 100644 index aaeca76..0000000 --- a/src/fast_automaton/convert/to_regex/transform.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::hash::BuildHasherDefault; - -use crate::execution_profile::ExecutionProfile; - -use super::*; - -impl StateEliminationAutomaton { - pub fn convert_to_regex( - &self, - execution_profile: &ExecutionProfile, - ) -> Result, EngineError> { - if self.cyclic { - return self.convert_graph_to_regex(execution_profile); - } - execution_profile.assert_not_timed_out()?; - - let mut regex_map: IntMap = IntMap::with_capacity_and_hasher( - self.get_number_of_states(), - BuildHasherDefault::default(), - ); - regex_map.insert(self.start_state, RegularExpression::new_empty_string()); - for from_state in self.states_topo_vec() { - let current_regex = if let Some(current_regex) = regex_map.get(&from_state) { - current_regex.clone() - } else { - RegularExpression::new_empty_string() - }; - if let Some(transitions) = self.transitions.get(from_state) { - for (to_state, transition) in transitions { - let transition_regex = match transition { - GraphTransition::Graph(graph) => { - if let Some(regex) = graph.convert_graph_to_regex(execution_profile)? { - regex - } else { - return Ok(None); - } - } - GraphTransition::Weight(range) => { - RegularExpression::Character(range.clone()) - } - GraphTransition::Epsilon => RegularExpression::new_empty_string(), - }; - let new_regex = current_regex.concat(&transition_regex, true); - match regex_map.entry(*to_state) { - Entry::Occupied(mut o) => { - o.insert(new_regex.union(o.get()).simplify()); - } - Entry::Vacant(v) => { - v.insert(new_regex); - } - }; - } - } - } - - Ok(regex_map.get(&self.accept_state).cloned()) - } - - fn convert_graph_to_regex( - &self, - execution_profile: &ExecutionProfile, - ) -> Result, EngineError> { - execution_profile.assert_not_timed_out()?; - if let Some(regex) = self.convert_shape_dot_star(execution_profile)? { - return Ok(Some(regex)); - } else if let Some(regex) = self.convert_shape_self_loop(execution_profile)? { - return Ok(Some(regex)); - } - Ok(None) - } - - /// We try to idenfify the regex following the shape: - /// A*B - fn convert_shape_dot_star( - &self, - execution_profile: &ExecutionProfile, - ) -> Result, EngineError> { - if self.get_number_of_states() < 2 { - return Ok(None); - } - //self.to_dot(); - let mut dot_value = - if let Some(dot_value) = self.get_transition(self.start_state, self.start_state) { - if let Some(dot_value) = dot_value.get_weight() { - dot_value.clone() - } else { - return Ok(None); - } - } else { - return Ok(None); - }; - - for state in self.states_iter() { - if state == self.start_state { - continue; - } - let weight = if let Some(weight) = self.get_transition(state, self.start_state) { - if let Some(weight) = weight.get_weight() { - weight - } else { - return Ok(None); - } - } else if state == self.accept_state { - continue; - } else { - return Ok(None); - }; - - if !dot_value.contains_all(weight) { - return Ok(None); - } - } - - let mut graph = self.clone(); - - for (from_state, transition) in graph.in_transitions_vec(graph.start_state) { - let weight = if let Some(weight) = transition.get_weight() { - weight - } else { - return Ok(None); - }; - dot_value = dot_value.union(weight); - graph.remove_transition(from_state, graph.start_state); - } - - let mut worklist = VecDeque::new(); - let mut seen = IntSet::with_capacity(graph.get_number_of_states()); - - worklist.push_back(graph.start_state); - seen.insert(self.start_state); - - while let Some(from_state) = worklist.pop_front() { - for to_state in graph.transitions_from_state_vec(&from_state) { - let transition = - if let Some(transition) = graph.get_transition(from_state, to_state) { - transition - } else { - return Ok(None); - }; - let weight = if let Some(weight) = transition.get_weight() { - weight - } else { - continue; - }; - dot_value = dot_value.union(weight); - if seen.contains(&to_state) { - if graph.accept_state != to_state || to_state == from_state { - graph.remove_transition(from_state, to_state); - } - } else { - seen.insert(to_state); - worklist.push_back(to_state); - } - } - } - - graph.add_transition_to( - self.start_state, - self.start_state, - GraphTransition::Weight(dot_value), - ); - - graph.identify_and_apply_components()?; - graph.convert_to_regex(execution_profile) - } - - /// We try to identify the regex following the shape: - /// A*B - fn convert_shape_self_loop( - &self, - execution_profile: &ExecutionProfile, - ) -> Result, EngineError> { - let mut graph = self.clone(); - - graph.accept_state = graph.new_state(); - - for (from_state, transition) in graph.in_transitions_vec(self.start_state) { - graph.remove_transition(from_state, self.start_state); - - graph.add_transition_to(from_state, graph.accept_state, transition); - } - - graph.identify_and_apply_components()?; - - let a_part = if let Some(a_part) = graph.convert_to_regex(execution_profile)? { - a_part - } else { - return Ok(None); - }; - - let mut graph = self.clone(); - - for (from_state, _) in graph.in_transitions_vec(self.start_state) { - graph.remove_transition(from_state, self.start_state); - } - - graph.identify_and_apply_components()?; - let b_part = if let Some(b_part) = graph.convert_to_regex(execution_profile)? { - b_part - } else { - return Ok(None); - }; - - let regex = a_part.repeat(0, None).concat(&b_part, true); - - Ok(Some(regex)) - } -} diff --git a/src/fast_automaton/generate.rs b/src/fast_automaton/generate.rs index 638ba11..cc8bec0 100644 --- a/src/fast_automaton/generate.rs +++ b/src/fast_automaton/generate.rs @@ -1,139 +1,1981 @@ -use std::cmp; - -use crate::{execution_profile::ThreadLocalParams, EngineError}; -use ahash::AHashSet; +use crate::{EngineError, execution_profile::ExecutionProfile}; +use ahash::RandomState; +use indexmap::IndexSet; use super::*; +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::ops::Range; -impl FastAutomaton { - pub fn generate_strings(&self, number: usize) -> Result, EngineError> { - if self.is_empty() { - return Ok(AHashSet::new()); +/// Each transition condition's index into the range pool the generation +/// resolved, the charset already taken out; `None` for the conditions the +/// charset leaves nothing of. +type RangeIds<'a> = AHashMap<&'a Condition, Option>; + +/// How [`FastAutomaton::generate_strings`] schedules the *paths* of a +/// language: one at a time, or interleaved so that every shape the pattern +/// allows is covered early. Orthogonal to [`CharacterOrder`], which chooses +/// the strings within each path. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum PathOrder { + /// One path at a time, shortest first, expanded in full before the next + /// path is visited: `.*abc.*` yields `abc`, `abc\u{0}`, `abc\u{1}`, ... + /// The cheapest way to page through a whole language with `offset`. + #[default] + Sweep, + /// A few strings per path before moving to the next one, so `.*abc.*` + /// yields `abc` and a string for each of the other shapes (`abc\u{0}`, + /// `\u{0}abc`, ...) — the shapes the pattern allows, instead of a million + /// variations of one of them, which is what makes it usable to derive + /// test cases. + /// + /// Shape comes first: the strings cover every path the automaton holds + /// before any path is asked for a second one, so a `limit` smaller than + /// the number of shapes is spent entirely on distinct shapes, and only a + /// larger one starts varying the characters within them. + /// + /// Deterministic, and pages with `offset` like [`Sweep`](Self::Sweep). + /// Each pass takes twice as many strings per path as the previous one, so + /// a finite language is still enumerated in full given a large enough + /// `limit`; those repeated passes make it slower than `Sweep`. + Interleave, + /// [`Interleave`](Self::Interleave), with same-length paths visited in an + /// order drawn by the seed ([`GenerationOptions::with_seed`], 0 by + /// default) instead of a fixed one: which *shapes* a small `limit` + /// reaches looks random too. Shorter paths still come first — on an + /// infinite language the search has to stay shortest-first to ever emit + /// anything — so the seed only draws among paths of equal length. + /// + /// The draw within a length is a randomized *cascade*, not a uniform + /// shuffle: the seed randomizes the pop order of the underlying + /// shortest-first search, and a path only becomes available once its + /// whole prefix chain has popped. A shape branching off an + /// already-visited path is ready immediately, while one that shares + /// nothing has to win a tie draw per prefix — on `.*abc.*`, `abc·x` + /// (one expansion past `abc` itself) leads more often than `x·abc`. The + /// bias fades as the pass proceeds, and coverage is untouched: every + /// shape still comes before any shape's second string. A uniform draw + /// would need every same-length path materialized before emitting any, + /// which an unbounded, incrementally-discovered path set rules out. + /// + /// Independent of [`CharacterOrder`]: shuffled paths over + /// [`Ascending`](CharacterOrder::Ascending) characters yield each drawn + /// shape's smallest witness; pair with + /// [`CharacterOrder::Shuffled`] for fully random-looking test cases. + /// Deterministic for a given seed, and pages with `offset` like the + /// other orders; offsets are only consistent between calls sharing the + /// seed. + Shuffled, +} + +/// Which strings of a path [`FastAutomaton::generate_strings`] reaches for +/// first: its character combinations in ascending order, or a seeded shuffle +/// of them. Orthogonal to [`PathOrder`], which schedules the paths +/// themselves. +/// +/// Neither changes *what* is generated: on a finite language every +/// combination of the two axes enumerates exactly the same strings, given the +/// `limit`. To generate from specific characters, restrict generation with +/// [`GenerationOptions::with_charset`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum CharacterOrder { + /// Each position expanded from the low end of its character range first: + /// `[a-z]{8}` yields `aaaaaaaa`, `aaaaaaab`, ... A stable, spec-defined + /// order — the smallest witnesses of a path come first. + #[default] + Ascending, + /// A seeded permutation of each path's combinations — `[a-z]{8}` yields + /// something like `sjtwsive` rather than `aaaaaaaa`: the strings drawn + /// from each shape look like real inputs. To draw the *shapes* by the + /// seed too, pair with [`PathOrder::Shuffled`]. + /// + /// Random in look only: the seed ([`GenerationOptions::with_seed`], 0 by + /// default) picks one fixed permutation, so generation is reproducible, + /// pages with `offset` like [`Ascending`](Self::Ascending), and — the + /// permutation being a bijection — never repeats a string across offsets + /// any more than it does. Offsets are only consistent between calls + /// sharing the seed. Unlike [`Ascending`](Self::Ascending)'s, the exact + /// sequence is implementation-defined: it may change between releases. + Shuffled, +} + +/// The most strings to reserve room for up front. `limit` is caller-controlled +/// and huge values (up to `usize::MAX`) are legitimate ways to ask for +/// everything, so it cannot size the allocation on its own; past this hint the +/// set grows as it fills. +const STRINGS_CAPACITY_LIMIT: usize = 1 << 12; + +/// How much a [`PathCache`] may hold — a finite language can still have far +/// more paths than fit in memory. Past these, recording gives up and the +/// later interleave passes search the automaton again: time spent instead of +/// memory. +const CACHE_IDS_LIMIT: usize = 1 << 20; +const CACHE_PATHS_LIMIT: usize = 1 << 17; + +/// Salts the seed into [`Generation::shape_key`], so the shape draw and the +/// [`Permuter`]'s character draw are independent functions of the same seed. +const SHAPE_KEY_SALT: u64 = 0x517C_C1B7_2722_0A95; + +#[derive(Clone, Eq, PartialEq)] +struct QueueItem { + score: usize, + /// Seeded tie-break between items of equal score, 0 unless paths are + /// [`PathOrder::Shuffled`]: what draws the shapes a small `limit` reaches + /// (see [`Generation::tie`]). + tie: u64, + depth: usize, + state: usize, + /// The path's transitions as indices into [`Generation::range_pool`] + ranges: Vec, +} + +impl Ord for QueueItem { + fn cmp(&self, other: &Self) -> Ordering { + other + .score + .cmp(&self.score) + .then_with(|| self.tie.cmp(&other.tie)) + .then_with(|| self.depth.cmp(&other.depth)) + .then_with(|| self.state.cmp(&other.state)) + .then_with(|| self.ranges.cmp(&other.ranges)) + } +} + +impl PartialOrd for QueueItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// What [`FastAutomaton::generate_strings`] is allowed to generate: how paths +/// are scheduled ([`PathOrder`]), how the strings within them are ordered +/// ([`CharacterOrder`], with the seed behind the `Shuffled` modes of both +/// axes), the characters it may use, and the string lengths it is confined +/// to. +/// +/// Either axis converts into it — so one can be passed on its own wherever +/// options are expected, the other keeping its default — and so does a +/// `(PathOrder, CharacterOrder)` pair. +/// +/// # Examples +/// +/// ``` +/// use regexsolver::{CharRange, Term, fast_automaton::{CharacterOrder, GenerationOptions, PathOrder}}; +/// use regexsolver::regex_charclass::char::Char; +/// +/// let term = Term::from_pattern(".{2}").unwrap(); +/// +/// // An axis on its own. +/// let strings = term.generate_strings(3, 0, PathOrder::Interleave).unwrap(); +/// +/// // Both axes, restricted to lowercase letters. +/// let lowercase = CharRange::new_from_range(Char::new('a')..=Char::new('z')); +/// let options = GenerationOptions::from((PathOrder::Interleave, CharacterOrder::Shuffled)) +/// .with_charset(lowercase) +/// .with_seed(42); +/// +/// let strings = term.generate_strings(3, 0, options).unwrap(); +/// assert!(strings.iter().all(|s| s.chars().all(|c| c.is_ascii_lowercase()))); +/// ``` +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct GenerationOptions { + paths: PathOrder, + characters: CharacterOrder, + charset: Option, + seed: u64, + min_length: usize, + max_length: Option, +} + +impl GenerationOptions { + /// Default options: [`PathOrder::Sweep`] over + /// [`CharacterOrder::Ascending`] combinations, using every character and + /// string length the automaton allows. + pub fn new() -> Self { + Self::default() + } + + /// Returns a copy of these options scheduling paths in `paths` order. + pub fn with_paths(mut self, paths: PathOrder) -> Self { + self.paths = paths; + self + } + + /// Returns a copy of these options ordering each path's strings in + /// `characters` order. + pub fn with_characters(mut self, characters: CharacterOrder) -> Self { + self.characters = characters; + self + } + + /// Returns a copy of these options restricted to `charset`. + /// + /// Only strings made entirely of those characters are generated: a path + /// through a transition that `charset` rules out is dropped whole, never + /// shortened. Restricting to characters the automaton never matches + /// generates nothing. + /// + /// A [`CharRange`] is built from bounds, or out of a character class + /// pattern through [`RegularExpression`](crate::regex::RegularExpression): + /// + /// ``` + /// use regexsolver::{CharRange, regex::RegularExpression}; + /// use regexsolver::regex_charclass::char::Char; + /// + /// let printable = CharRange::new_from_range(Char::new(' ')..=Char::new('~')); + /// + /// let no_controls = match RegularExpression::new("\\P{C}").unwrap() { + /// RegularExpression::Character(charset) => charset, + /// other => panic!("not a character class: {other}"), + /// }; + /// ``` + pub fn with_charset(mut self, charset: CharRange) -> Self { + self.charset = Some(charset); + self + } + + /// Returns a copy of these options drawing [`PathOrder::Shuffled`]'s + /// path draws and [`CharacterOrder::Shuffled`]'s permutation from `seed`; + /// generation using neither ignores it. + /// + /// The default seed is 0 — a fixed seed, not a random one, so two calls + /// with the same options generate the same strings and `offset` pages + /// through them consistently. Change the seed to draw a different + /// sequence of strings from the same pattern. + pub fn with_seed(mut self, seed: u64) -> Self { + self.seed = seed; + self + } + + /// Returns a copy of these options generating only strings at least + /// `min_length` characters long: the shorter strings the automaton + /// matches are left out of the enumeration, `offset` never counting + /// them. 0 — every string — by default. + pub fn with_min_length(mut self, min_length: usize) -> Self { + self.min_length = min_length; + self + } + + /// Returns a copy of these options generating only strings at most + /// `max_length` characters long: the longer strings the automaton + /// matches are left out of the enumeration, `offset` never counting + /// them. Unbounded by default — and without a bound, a deep `offset` + /// into a looping language (`.*`) pages into arbitrarily long strings, + /// so bound it when the offset is not under your control. + /// + /// A bound below `min_length` leaves nothing to generate. + pub fn with_max_length(mut self, max_length: usize) -> Self { + self.max_length = Some(max_length); + self + } + + /// The order paths are scheduled in. + pub fn paths(&self) -> PathOrder { + self.paths + } + + /// The order each path's strings come out in. + pub fn characters(&self) -> CharacterOrder { + self.characters + } + + /// The characters generation is restricted to, `None` when it is not. + pub fn charset(&self) -> Option<&CharRange> { + self.charset.as_ref() + } + + /// The seed behind [`PathOrder::Shuffled`] and + /// [`CharacterOrder::Shuffled`]. + pub fn seed(&self) -> u64 { + self.seed + } + + /// The shortest string generation may emit. + pub fn min_length(&self) -> usize { + self.min_length + } + + /// The longest string generation may emit, `None` when unbounded. + pub fn max_length(&self) -> Option { + self.max_length + } +} + +impl From for GenerationOptions { + fn from(paths: PathOrder) -> Self { + GenerationOptions { + paths, + ..Default::default() + } + } +} + +impl From for GenerationOptions { + fn from(characters: CharacterOrder) -> Self { + GenerationOptions { + characters, + ..Default::default() + } + } +} + +impl From<(PathOrder, CharacterOrder)> for GenerationOptions { + fn from((paths, characters): (PathOrder, CharacterOrder)) -> Self { + GenerationOptions { + paths, + characters, + ..Default::default() } + } +} + +impl FastAutomaton { + /// Generates up to `limit` distinct strings matched by the automaton under + /// the given [`GenerationOptions`], skipping the first `offset` strings. + /// + /// `options` is a [`PathOrder`] or [`CharacterOrder`] on its own (or a + /// pair of them), or a full [`GenerationOptions`] to also set the seed + /// and restrict the characters and string lengths used. + /// + /// Strings are only guaranteed to be distinct **within a single call**: + /// the offset fast-skips by counting paths, and in a non-deterministic + /// automaton the same string can be reached through several paths, so + /// calls with different offsets may repeat strings (or skip some). + /// [`determinize`](Self::determinize) (and ideally + /// [`minimize`](Self::minimize)) first to make pages disjoint. Offsets are + /// also only consistent between calls made with the same options. + /// + /// [`GenerationOptions::with_min_length`] and + /// [`with_max_length`](GenerationOptions::with_max_length) confine the + /// enumeration to a band of string lengths — without a max, a deep + /// `offset` into a looping language (`.*`) pages into arbitrarily long + /// strings. Generation runs under the active [`ExecutionProfile`]: its + /// timeout aborts with [`EngineError::OperationTimeOutError`]. + #[tracing::instrument(level = "debug", skip(self, options), fields(states = self.number_of_states(), deterministic=self.is_deterministic(), limit=limit, offset=offset, paths=tracing::field::Empty, characters=tracing::field::Empty, charset=tracing::field::Empty, seed=tracing::field::Empty, min_length=tracing::field::Empty, max_length=tracing::field::Empty))] + pub fn generate_strings( + &self, + limit: usize, + offset: usize, + options: impl Into, + ) -> Result, EngineError> { + let options = options.into(); - let mut strings = AHashSet::with_capacity(cmp::min(number, 1000)); + // Serializing the charset is not free: only when the span is recorded. + let span = tracing::Span::current(); + if !span.is_disabled() { + span.record("paths", tracing::field::debug(options.paths)); + span.record("characters", tracing::field::debug(options.characters)); + span.record( + "charset", + tracing::field::debug(options.charset.as_ref().map(|charset| charset.to_regex())), + ); + span.record("seed", options.seed); + span.record("min_length", options.min_length as u64); + span.record("max_length", tracing::field::debug(options.max_length)); + } - let execution_profile = ThreadLocalParams::get_execution_profile(); + self.generate(limit, offset, &options) + } - let mut ranges_cache: AHashMap<&Condition, Range> = - AHashMap::with_capacity(self.get_number_of_states()); + /// [`generate_strings`](Self::generate_strings) over borrowed options, for + /// the callers holding them across several batches. + pub(crate) fn generate( + &self, + limit: usize, + offset: usize, + options: &GenerationOptions, + ) -> Result, EngineError> { + if self.is_empty() || limit == 0 { + return Ok(vec![]); + } - let mut worklist: VecDeque<(Vec, usize)> = - VecDeque::with_capacity(cmp::min(number, 1000)); - let mut visited = AHashSet::with_capacity(cmp::min(number, 1000)); + let mut generation = Generation::new(self, limit, offset, options)?; - worklist.push_back((vec![], self.start_state)); - while let Some((ranges, state)) = worklist.pop_front() { - if self.accept_states.contains(&state) { - if ranges.is_empty() { - strings.insert(String::new()); - } else { - let mut end = false; - let mut ranges_iter: Vec<_> = ranges.iter().map(|range| range.iter()).collect(); - while strings.len() < number { - execution_profile.assert_not_timed_out()?; - let mut string = vec![]; - for i in 0..ranges.len() { - if let Some(character) = ranges_iter[i].next() { - string.push(character); - } else { - ranges_iter[i] = ranges[i].iter(); - if i + 1 < ranges.len() { - string.push(ranges_iter[i].next().unwrap()); - } else { - end = true; - break; - } - } - } - if end { - break; - } - strings.insert(string.into_iter().map(|c| c.to_char()).collect()); + match options.paths { + PathOrder::Sweep => { + // The ascending sweep walks each path's combinations with + // cursors; the shuffled one has to index them through the + // permutation, which a window over everything is. + let window = + (options.characters == CharacterOrder::Shuffled).then_some(0..usize::MAX); + generation.walk(self, window.as_ref(), None)?; + } + PathOrder::Interleave | PathOrder::Shuffled => { + // A pass takes at most `window` combinations per path, so no + // single path can spend the whole `limit` on itself. The + // windows double and pick up where the previous one stopped: + // a pass that exhausts the automaton without filling `limit` + // is followed by one digging deeper into the same paths, until + // a pass finds nothing left to cover. Exhausting the automaton + // also proves the paths finite and leaves them in `cache`, so + // the passes after it replay them instead of searching again. + let mut window = 0..1; + let mut cache = PathCache::new(); + loop { + let covered = if cache.complete { + generation.replay(&cache, &window)? + } else { + generation.walk(self, Some(&window), Some(&mut cache))? + }; + if covered == 0 || generation.emitter.is_full() { + break; } + window = window.end..window.end.saturating_mul(2).saturating_add(1); } + } + } + + Ok(generation.emitter.strings.into_iter().collect()) + } +} + +/// The state of a single [`FastAutomaton::generate_strings`] call, shared by +/// every pass an interleaving generation makes over the automaton. +struct Generation<'a> { + /// Number of transitions from each state to the nearest accept state; + /// `usize::MAX` for the states that cannot reach one. + distances: Vec, + /// Length of the longest string generation may emit: what the automaton + /// matches, capped at the options' + /// [`max_length`](GenerationOptions::max_length). + max_len: usize, + /// Length of the shortest string generation may emit + /// ([`GenerationOptions::min_length`]); the search still walks the + /// shorter accepting paths — they lead to long enough ones — it just + /// does not emit them. + min_len: usize, + /// The characters each transition stands for, resolved once: the paths + /// refer to them by index (see [`QueueItem::ranges`]). + range_pool: Vec, + /// Each transition condition's index into + /// [`range_pool`](Self::range_pool), the charset already taken out. A + /// condition the charset leaves nothing of holds `None`, which is what + /// makes its transition impassable. + range_ids: RangeIds<'a>, + /// Seeds [`QueueItem::tie`] under [`PathOrder::Shuffled`]; `None` leaves + /// every tie at 0 and the pop order to the deterministic fallback. + shape_key: Option, + emitter: Emitter, +} + +/// Turns the paths the search pops into strings: the half of a [`Generation`] +/// the emission mutates, kept apart from the search data so that a borrow of +/// the range pool can live alongside it. +struct Emitter { + limit: usize, + offset: usize, + /// Scrambles each path's combination indices for + /// [`CharacterOrder::Shuffled`]; `None` walks them in ascending order. + permuter: Option, + strings: IndexSet, + execution_profile: ExecutionProfile, +} + +/// The accepting paths an interleave pass popped, in pop order — flat, path `i` +/// being `ids[starts[i]..starts[i + 1]]`. A pass that runs out of paths has +/// recorded all of them, and the passes after it replay the cache instead of +/// searching the automaton again. +struct PathCache { + ids: Vec, + starts: Vec, + /// The cache holds every path of the automaton and can stand in for it. + complete: bool, + /// Recording outgrew [`CACHE_IDS_LIMIT`]/[`CACHE_PATHS_LIMIT`] and gave + /// up; the cache stays empty and every pass searches. + overflowed: bool, +} + +impl PathCache { + fn new() -> Self { + PathCache { + ids: vec![], + starts: vec![0], + complete: false, + overflowed: false, + } + } + + fn record(&mut self, path: &[u32]) { + if self.overflowed { + return; + } + if self.ids.len().saturating_add(path.len()) > CACHE_IDS_LIMIT + || self.starts.len() > CACHE_PATHS_LIMIT + { + self.overflowed = true; + self.ids = vec![]; + self.starts = vec![0]; + return; + } + + self.ids.extend_from_slice(path); + self.starts.push(self.ids.len()); + } + + fn paths(&self) -> impl Iterator { + self.starts + .windows(2) + .map(|window| &self.ids[window[0]..window[1]]) + } +} + +/// What every transition condition leaves once the charset is taken out. +/// Resolved up front rather than as the walk reaches them, so that the search +/// already knows which transitions are impassable: a pool of the non-empty +/// ranges, and each condition's index into it — `None` for the conditions the +/// charset leaves nothing of. +fn resolve_ranges<'a>( + automaton: &'a FastAutomaton, + charset: Option<&CharRange>, +) -> Result<(Vec, RangeIds<'a>), EngineError> { + let mut range_pool: Vec = Vec::new(); + let mut range_ids: RangeIds = AHashMap::with_capacity(automaton.transitions.len()); + + for state in automaton.states() { + for (cond, _) in automaton.transitions_from(state) { + let std::collections::hash_map::Entry::Vacant(vacant) = range_ids.entry(cond) else { + continue; + }; + let range = cond.to_range(&automaton.spanning_set)?; + let range = match charset { + Some(charset) => range.intersection(charset), + None => range, + }; + let id = if range.is_empty() { + None + } else { + range_pool.push(range); + Some((range_pool.len() - 1) as u32) + }; + vacant.insert(id); + } + } + + Ok((range_pool, range_ids)) +} - if strings.len() == number { +/// REVERSE BFS: the exact distance from every state to an accept state, which +/// drives the A* search and prunes the states that never accept; `usize::MAX` +/// for the states that cannot reach one. A state the charset leaves no way out +/// of (no id in `range_ids`) is one of those dead ends. +fn distances_to_accept(automaton: &FastAutomaton, range_ids: &RangeIds) -> Vec { + let num_states = automaton.transitions.len(); + let mut incoming = vec![vec![]; num_states]; + let mut dist_q = VecDeque::new(); + let mut distances = vec![usize::MAX; num_states]; + + for state in automaton.states() { + if automaton.is_accepted(state) { + distances[state] = 0; + dist_q.push_back(state); + } + for (cond, &to_state) in automaton.transitions_from(state) { + if range_ids[cond].is_some() { + incoming[to_state].push(state); + } + } + } + + while let Some(state) = dist_q.pop_front() { + let d = distances[state]; + for &prev in &incoming[state] { + if distances[prev] == usize::MAX { + distances[prev] = d + 1; + dist_q.push_back(prev); + } + } + } + + distances +} + +/// The ranges of a path's transitions, looked up from the pool. +fn resolve<'p>(pool: &'p [CharRange], path: &[u32]) -> Vec<&'p CharRange> { + path.iter().map(|&id| &pool[id as usize]).collect() +} + +impl<'a> Generation<'a> { + fn new( + automaton: &'a FastAutomaton, + limit: usize, + offset: usize, + options: &GenerationOptions, + ) -> Result { + let (range_pool, range_ids) = resolve_ranges(automaton, options.charset())?; + let distances = distances_to_accept(automaton, &range_ids); + let (_, max) = automaton.length(); + let execution_profile = ExecutionProfile::get(); + + // The options' length bounds: without a max, a deep offset into a + // looping language would page into arbitrarily long strings. + let max_len = + (max.unwrap_or(u32::MAX) as usize).min(options.max_length().unwrap_or(usize::MAX)); + + Ok(Generation { + distances, + max_len, + min_len: options.min_length(), + range_pool, + range_ids, + shape_key: (options.paths == PathOrder::Shuffled) + .then(|| mix(options.seed ^ SHAPE_KEY_SALT)), + emitter: Emitter { + limit, + offset, + permuter: (options.characters == CharacterOrder::Shuffled) + .then(|| Permuter::new(options.seed)), + strings: IndexSet::with_capacity_and_hasher( + limit.min(STRINGS_CAPACITY_LIMIT), + RandomState::default(), + ), + execution_profile, + }, + }) + } + + /// A* SEARCH: walks the automaton once, shortest path first, emitting the + /// strings of every accepting path it pops until `limit` strings are + /// collected or the automaton runs out of paths. + /// + /// `window` restricts each path to the combinations whose index falls + /// inside it; `None` takes them all. `cache`, when given, records the + /// accepting paths in pop order, and running out of paths marks it + /// complete: [`replay`](Self::replay) then stands in for the next passes. + /// Returns how many combinations the pass covered, the ones `offset` + /// skipped included. + fn walk( + &mut self, + automaton: &'a FastAutomaton, + window: Option<&Range>, + mut cache: Option<&mut PathCache>, + ) -> Result { + let start_state = automaton.start_state(); + + // If the start state can't reach an accept state, exit immediately + if self.distances[start_state] == usize::MAX { + return Ok(0); + } + + let mut covered = 0usize; + + let mut q = BinaryHeap::new(); + q.push(QueueItem { + score: self.distances[start_state], + tie: 0, + depth: 0, + state: start_state, + ranges: vec![], + }); + + while let Some(QueueItem { + score: _, + tie, + depth: current_depth, + state, + ranges, + }) = q.pop() + { + self.emitter.execution_profile.assert_not_timed_out()?; + + // A path shorter than `min_len` is walked — its extensions are + // long enough — but never emitted, recorded, or counted. + if automaton.is_accepted(state) && current_depth >= self.min_len { + if let Some(cache) = cache.as_deref_mut() { + cache.record(&ranges); + } + + let resolved = resolve(&self.range_pool, &ranges); + covered = covered.saturating_add(match window { + Some(window) => self.emitter.emit_window(&resolved, &ranges, window)?, + None => self.emitter.emit_all(&resolved)?, + }); + + if self.emitter.is_full() { break; } } - for (to_state, cond) in self.transitions_from_state_enumerate_iter(&state) { - execution_profile.assert_not_timed_out()?; - let range = match ranges_cache.entry(cond) { - Entry::Occupied(o) => o.get().clone(), - Entry::Vacant(v) => { - let range = cond.to_range(&self.spanning_set)?; - v.insert(range.clone()); - range - } - }; - if range.is_empty() { - continue; - } + + if current_depth >= self.max_len { + continue; + } + + self.expand(automaton, &mut q, current_depth + 1, state, tie, ranges); + } + + // An empty queue means every path was popped, so a recording cache + // now holds them all. + if q.is_empty() + && let Some(cache) = cache + && !cache.overflowed + { + cache.complete = true; + } + + Ok(covered) + } + + /// Queues every passable one-transition extension of a popped path, the + /// last one taking over the path's own vector instead of cloning it. + fn expand( + &self, + automaton: &'a FastAutomaton, + q: &mut BinaryHeap, + next_depth: usize, + state: State, + tie: u64, + mut ranges: Vec, + ) { + let mut valid_transitions = Vec::new(); + + for (cond, &to_state) in automaton.transitions_from(state) { + // DEAD-END PRUNING: Instantly kill paths that cannot accept + if self.distances[to_state] == usize::MAX { + continue; + } + + // ...and the transitions the charset closed off. + let Some(range_id) = self.range_ids[cond] else { + continue; + }; + + valid_transitions.push((to_state, range_id)); + } + + // Vector Reuse Optimization + if let Some((last_state, last_id)) = valid_transitions.pop() { + for (to_state, range_id) in valid_transitions { let mut new_ranges = ranges.clone(); - new_ranges.push(range); - let element = (new_ranges, *to_state); + new_ranges.push(range_id); + q.push(QueueItem { + score: next_depth + self.distances[to_state], // A* Score Formula + tie: self.tie(tie, range_id, to_state), + depth: next_depth, + state: to_state, + ranges: new_ranges, + }); + } + + let tie = self.tie(tie, last_id, last_state); + ranges.push(last_id); + q.push(QueueItem { + score: next_depth + self.distances[last_state], // A* Score Formula + tie, + depth: next_depth, + state: last_state, + ranges, + }); + } + } + + /// The tie of a path extended by `range_id` into `to_state`: the parent's + /// tie folded with a seeded hash of the transition, so equal-score paths + /// pop in an order the seed draws — the cascade documented on + /// [`PathOrder::Shuffled`]. 0 — fall through to the deterministic + /// tie-breaks — without a [`shape_key`](Self::shape_key). + fn tie(&self, parent: u64, range_id: u32, to_state: State) -> u64 { + match self.shape_key { + Some(key) => mix(parent ^ mix(key ^ ((range_id as u64) << 32) ^ to_state as u64)), + None => 0, + } + } + + /// Emits `window` from every path of a complete [`PathCache`], in the + /// order the search popped them: what a [`walk`](Self::walk) pass would + /// do, minus the search. + fn replay(&mut self, cache: &PathCache, window: &Range) -> Result { + let mut covered = 0usize; + + for path in cache.paths() { + self.emitter.execution_profile.assert_not_timed_out()?; + + let resolved = resolve(&self.range_pool, path); + covered = covered.saturating_add(self.emitter.emit_window(&resolved, path, window)?); + + if self.emitter.is_full() { + break; + } + } + + Ok(covered) + } +} + +impl Emitter { + #[inline] + fn is_full(&self) -> bool { + self.strings.len() >= self.limit + } - if !visited.contains(&element) { - visited.insert(element.clone()); - worklist.push_back(element); + /// Emits every combination of `ranges` that `offset` does not skip, in + /// ascending character order. Returns the number of combinations the path + /// holds. + fn emit_all(&mut self, ranges: &[&CharRange]) -> Result { + let range_lengths: Vec = ranges + .iter() + .map(|r| r.get_cardinality() as usize) + .collect(); + + let mut total_combinations = 1usize; + for &len in &range_lengths { + total_combinations = total_combinations.saturating_mul(len); + } + + if self.offset >= total_combinations { + self.offset -= total_combinations; + return Ok(total_combinations); + } + + self.emit_combinations(ranges, &range_lengths)?; + Ok(total_combinations) + } + + /// Walks the combinations depth-first over an explicit stack of range + /// cursors, one per position: recursing per character would overflow the + /// stack on the paths thousands of transitions long. + fn emit_combinations( + &mut self, + ranges: &[&CharRange], + range_lengths: &[usize], + ) -> Result<(), EngineError> { + if ranges.is_empty() { + // A single-combination path: `emit_all` either skipped it whole or + // arrived here with nothing left of the offset. + debug_assert_eq!(0, self.offset); + self.strings.insert(String::new()); + return Ok(()); + } + + // Combinations under a single character at each position: the product + // of the range lengths past it. + let mut sub_combinations = vec![1usize; ranges.len()]; + for position in (0..ranges.len() - 1).rev() { + sub_combinations[position] = + sub_combinations[position + 1].saturating_mul(range_lengths[position + 1]); + } + + let mut current_str = String::with_capacity(ranges.len()); + let mut cursors = Vec::with_capacity(ranges.len()); + cursors.push(self.descend(ranges[0], sub_combinations[0])); + + while let Some(cursor) = cursors.last_mut() { + let next = cursor.next(); + let position = cursors.len() - 1; + + let Some(ch) = next else { + // The range is exhausted: back up to the previous position and + // move it to its next character. + cursors.pop(); + current_str.pop(); + continue; + }; + + self.execution_profile.assert_not_timed_out()?; + + current_str.push(ch.to_char()); + if position + 1 == ranges.len() { + // A full combination; whatever `offset` had left to skip was + // consumed by the descents, so this string is on the page. + self.strings.insert(current_str.clone()); + current_str.pop(); + + if self.is_full() { + break; } + } else { + cursors.push(self.descend(ranges[position + 1], sub_combinations[position + 1])); + } + } + + Ok(()) + } + + /// A cursor over `range`, opened on the first combination `offset` does + /// not skip: the subtrees of `sub_combinations` strings each before it are + /// stepped over in one division, not walked character by character. + fn descend<'r>(&mut self, range: &'r CharRange, sub_combinations: usize) -> RangeCursor<'r> { + // Past the first emitted string the offset is zero and every cursor + // starts at its range's first character. `skip` stays within the + // range: the offset was left smaller than the previous position's + // subtree, which this whole range spans. (A saturated subtree count + // under-skips into the first character, never past the range.) + let skip = self.offset / sub_combinations; + self.offset -= skip * sub_combinations; + RangeCursor::new(range, skip as u32) + } + + /// Emits the combinations of `ranges` whose index falls inside `window`, + /// in the ascending order [`emit_all`](Self::emit_all) walks them in — or, + /// with a [`permuter`](Self::permuter), the path's own seeded permutation + /// of it (`path` holds the transition ids the ranges were resolved from). + /// Returns how many of them the window covered, the ones `offset` skipped + /// included. + fn emit_window( + &mut self, + ranges: &[&CharRange], + path: &[u32], + window: &Range, + ) -> Result { + let range_lengths: Vec = ranges.iter().map(|r| r.get_cardinality() as u128).collect(); + + // `None` once the product stops fitting: such a path holds more + // combinations than a window will ever reach into. + let total_combinations = range_lengths + .iter() + .try_fold(1u128, |total, &len| total.checked_mul(len)); + let bound = + total_combinations.map_or(usize::MAX, |total| total.min(usize::MAX as u128) as usize); + + let covered = window.end.min(bound) - window.start.min(bound); + if self.offset >= covered { + self.offset -= covered; + return Ok(covered); + } + + let first = window.start.min(bound) + self.offset; + self.offset = 0; + + let tweak = self + .permuter + .as_ref() + .map_or(0, |permuter| permuter.path_tweak(path)); + + for index in first..window.end.min(bound) { + self.execution_profile.assert_not_timed_out()?; + + // The permutation reorders `[0, bound)` onto itself, so the + // window still covers `covered` distinct combinations — just not + // the ascending ones. + let combination = match &self.permuter { + Some(permuter) => permuter.permute(index as u128, bound as u128, tweak), + None => index as u128, + }; + let string = sample_string(ranges, &range_lengths, combination)?; + self.strings.insert(string); + + if self.is_full() { + break; + } + } + + Ok(covered) + } +} + +/// Builds the combination of `ranges` at index `combination`, read as a +/// mixed-radix number whose least significant digit is the last character, +/// each digit indexing its range in ascending order: combinations come out in +/// the lexicographic order [`Emitter::emit_all`] walks them in. +/// +/// The mapping is a bijection over the combinations, which is what keeps the +/// sampled strings distinct and `offset` exact. +fn sample_string( + ranges: &[&CharRange], + range_lengths: &[u128], + mut combination: u128, +) -> Result { + let mut chars = Vec::with_capacity(ranges.len()); + + for (&range, &length) in ranges.iter().zip(range_lengths).rev() { + let index = (combination % length) as u32; + combination /= length; + let ch = char_at(range, index).ok_or(EngineError::InvalidCharacterInRegex)?; + chars.push(ch.to_char()); + } + + chars.reverse(); + Ok(chars.into_iter().collect()) +} + +/// A seeded family of permutations of `[0, bound)` for any `bound`, one per +/// `tweak`, evaluated point by point: a tweaked Feistel network over the +/// smallest even-width binary domain holding `bound`, cycle-walked back into +/// it. Being a bijection (at any fixed tweak) is what keeps +/// [`CharacterOrder::Shuffled`] strings distinct and `offset` exact, exactly +/// like the ascending order it stands in for; being a fixed function of the +/// seed is what lets a page be generated without materializing (or even +/// visiting) the combinations around it. +struct Permuter { + keys: [u64; 4], + tweak_key: u64, +} + +impl Permuter { + fn new(seed: u64) -> Self { + // SplitMix64: one independent-looking round key per Feistel round, + // nearby seeds included. + let mut state = seed; + let mut keys = [0u64; 4]; + for key in &mut keys { + state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + *key = mix(state); + } + let tweak_key = mix(state.wrapping_add(0x9E37_79B9_7F4A_7C15)); + Permuter { keys, tweak_key } + } + + /// A path's tweak: its transition ids folded through the seeded hash, + /// picking the path's own permutation out of the family. Without it, + /// same-shape alternation branches (`[a-z]{4}|[A-Z]{4}`) would emit the + /// same combination indices and mirror each other's strings. + fn path_tweak(&self, path: &[u32]) -> u64 { + path.iter() + .fold(self.tweak_key, |acc, &id| mix(acc ^ id as u64)) + } + + /// Where the `tweak`'s permutation of `[0, bound)` sends `index`; `index` + /// must be below `bound`. + fn permute(&self, index: u128, bound: u128, tweak: u64) -> u128 { + debug_assert!(index < bound); + if bound <= 1 { + return index; + } + + let bits = 128 - (bound - 1).leading_zeros(); + let half = bits.div_ceil(2); + let mask = (1u128 << half) - 1; + + // CYCLE-WALKING: encrypt until the value falls back under `bound`. + // The walk follows the cycle `index` itself sits on, so it terminates + // (on `index`, at worst), and distinct indices — on distinct cycles + // or ahead of one another on the same cycle — never land on the same + // value. The domain is under `4 * bound`, so it takes a few steps. + let mut value = index; + loop { + value = self.encrypt(value, half, mask, tweak); + if value < bound { + return value; } } + } - Ok(strings) + /// One pass of the 4-round Feistel network: a bijection over + /// `[0, 2^(2 * half))` for any fixed `tweak`, `half` at most 64. + fn encrypt(&self, value: u128, half: u32, mask: u128, tweak: u64) -> u128 { + let mut left = value >> half; + let mut right = value & mask; + for &key in &self.keys { + let round = (mix(right as u64 ^ key ^ tweak) as u128) & mask; + (left, right) = (right, left ^ round); + } + (left << half) | right } } +/// SplitMix64's finalizer: the avalanche behind the [`Permuter`]'s round keys +/// and round function. +fn mix(value: u64) -> u64 { + let mut z = value; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A cursor over the characters a [`CharRange`] holds, in order, opened at an +/// arbitrary ordinal: what the range's own iterator cannot do, and what lets +/// [`Emitter::descend`] skip an offset in one step. +struct RangeCursor<'a> { + /// The `(low, high)` bound pairs the range is made of. + bounds: &'a [Char], + /// Index of the current pair's low bound; past `bounds` once exhausted. + pair: usize, + /// [`scalar`] of the next character to hand out. + next_scalar: u32, +} + +impl<'a> RangeCursor<'a> { + /// A cursor whose first character is `range`'s `ordinal`-th; exhausted + /// from the start when `ordinal` is past the range's cardinality. + fn new(range: &'a CharRange, mut ordinal: u32) -> Self { + let bounds = range.0.as_slice(); + let mut pair = 0; + let mut next_scalar = 0; + + while pair < bounds.len() { + let (low, high) = (scalar(bounds[pair]), scalar(bounds[pair + 1])); + let length = high - low + 1; + if ordinal < length { + next_scalar = low + ordinal; + break; + } + ordinal -= length; + pair += 2; + } + + RangeCursor { + bounds, + pair, + next_scalar, + } + } + + fn next(&mut self) -> Option { + if self.pair >= self.bounds.len() { + return None; + } + + let ch = from_scalar(self.next_scalar); + if self.next_scalar == scalar(self.bounds[self.pair + 1]) { + // Past the current interval: on to the next one. + self.pair += 2; + if self.pair < self.bounds.len() { + self.next_scalar = scalar(self.bounds[self.pair]); + } + } else { + self.next_scalar += 1; + } + + ch + } +} + +/// The character `range` holds at `ordinal`, `None` past its cardinality. +fn char_at(range: &CharRange, ordinal: u32) -> Option { + RangeCursor::new(range, ordinal).next() +} + #[cfg(test)] mod tests { + use super::{CharacterOrder, GenerationOptions, PathOrder, Permuter, RangeCursor, char_at}; + use crate::CharRange; + use crate::cardinality::Cardinality; + use crate::{fast_automaton::FastAutomaton, regex::RegularExpression}; use regex::Regex; + use regex_charclass::{CharacterClass, char::Char, irange::range::AnyRange}; + + const AXES: [(PathOrder, CharacterOrder); 6] = [ + (PathOrder::Sweep, CharacterOrder::Ascending), + (PathOrder::Sweep, CharacterOrder::Shuffled), + (PathOrder::Interleave, CharacterOrder::Ascending), + (PathOrder::Interleave, CharacterOrder::Shuffled), + (PathOrder::Shuffled, CharacterOrder::Ascending), + (PathOrder::Shuffled, CharacterOrder::Shuffled), + ]; + + /// Every set of options the generation tests run through: all four axis + /// combinations, each of them once unrestricted and once over printable + /// ASCII, which is narrow enough to close off transitions in most of the + /// patterns. + fn all_options() -> Vec { + AXES.into_iter() + .flat_map(|axes| { + [ + GenerationOptions::from(axes), + GenerationOptions::from(axes).with_charset(printable_ascii()), + ] + }) + .collect() + } + + fn printable_ascii() -> CharRange { + CharRange::new_from_range_char(' '..='~') + } + + #[test] + fn test_generate_strings_1() -> Result<(), String> { + let automaton = RegularExpression::parse("((aad|..e.*|e.z)*|q)", false) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton = automaton.determinize().unwrap(); + for axes in AXES { + println!("{:?}", automaton.generate_strings(30, 0, axes).unwrap()); + } + + Ok(()) + } + + #[test] + fn test_generate_strings_2() -> Result<(), String> { + let automaton = RegularExpression::parse("(abc|de){2}", true) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton = automaton.determinize().unwrap(); + for axes in AXES { + let strings = automaton.generate_strings(2, 0, axes).unwrap(); + assert_eq!(2, strings.len()); - use crate::regex::RegularExpression; + let strings = automaton.generate_strings(2, 2, axes).unwrap(); + assert_eq!(2, strings.len()); + } + + Ok(()) + } #[test] - fn test_generate_strings() -> Result<(), String> { + fn test_generate_strings_3() -> Result<(), String> { + assert_generate_strings(r"<([A-Za-z][A-Za-z0-9]*)[^>]*?/>", 500); + assert_generate_strings("a{100}[a-z]", 100); + assert_generate_strings("(ab|cd)e", 100); + assert_generate_strings("[a-z]+", 100); + assert_generate_strings("[a-z]+@", 100); assert_generate_strings("ù", 1000); - assert_generate_strings("(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@", 500); - assert_generate_strings( - "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", - 500 - ); assert_generate_strings("[0-9]+[A-Z]*", 500); assert_generate_strings("a+(ba+)*", 200); assert_generate_strings("((a|bc)*|d)", 200); assert_generate_strings(".*", 50); assert_generate_strings("(ac|ads|a)*", 200); assert_generate_strings("((aad|ads|a)*|q)", 200); + + assert_generate_strings( + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + 1000, + ); + + assert_generate_strings("(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@", 500); + assert_generate_strings( + "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", + 500, + ); assert_generate_strings("((aad|ads|a)*abc.*uif(aad|ads|x)*|q)", 1000); - //((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,5} + Ok(()) } - fn assert_generate_strings(regex: &str, number: usize) { - println!(":{}", regex); - let automaton = RegularExpression::new(regex) + #[test] + fn test_generate_strings_offset() -> Result<(), String> { + assert_generate_strings_offset(".{900}"); + assert_generate_strings_offset("[a-z]+"); + assert_generate_strings_offset("[a-z]+@"); + + assert_generate_strings_offset("[0-9]+[A-Z]*"); + assert_generate_strings_offset("a+(ba+)*"); + assert_generate_strings_offset("((a|bc)*|d)"); + assert_generate_strings_offset(".*"); + assert_generate_strings_offset("(ac|ads|a)*"); + assert_generate_strings_offset("((aad|ads|a)*|q)"); + + assert_generate_strings_offset( + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + assert_generate_strings_offset("(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@"); + assert_generate_strings_offset( + "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", + ); + assert_generate_strings_offset("((aad|ads|a)*abc.*uif(aad|ads|x)*|q)"); + + Ok(()) + } + + /// The interleave order exists so that a pattern's *shapes* get covered: + /// what the sweep spends a million strings on (one path, every character + /// of its last range) has to fit in a handful of them. + #[test] + fn test_generate_strings_interleave_covers_the_whole_pattern() { + let automaton = automaton_of(".*abc.*").determinize().unwrap().into_owned(); + + let swept = automaton.generate_strings(20, 0, PathOrder::Sweep).unwrap(); + assert!( + swept.iter().all(|s| s.starts_with("abc")), + "the sweep stays on the first path it finds: {swept:?}" + ); + + let interleaved = automaton + .generate_strings(20, 0, PathOrder::Interleave) + .unwrap(); + assert!( + interleaved.iter().any(|s| !s.starts_with("abc")), + "the interleave order has to reach the strings with a prefix before `abc`: {interleaved:?}" + ); + assert!( + interleaved.iter().any(|s| !s.ends_with("abc")), + "the interleave order has to reach the strings with a suffix after `abc`: {interleaved:?}" + ); + assert!( + interleaved.iter().any(|s| s.len() > 5), + "the interleave order has to reach longer strings too: {interleaved:?}" + ); + } + + /// Interleaving still enumerates a finite language in full, given the + /// room: the passes dig deeper into every path until nothing is left to + /// cover. + #[test] + fn test_generate_strings_interleave_is_exhaustive_in_the_limit() { + let automaton = automaton_of("[a-z][0-9]"); + + let mut interleaved = automaton + .generate_strings(1000, 0, PathOrder::Interleave) + .unwrap(); + interleaved.sort(); + + let mut expected: Vec = ('a'..='z') + .flat_map(|letter| ('0'..='9').map(move |digit| format!("{letter}{digit}"))) + .collect(); + expected.sort(); + + assert_eq!(expected, interleaved); + } + + /// The path order chooses which strings come first, never the characters + /// they are made of: an interleaved string reaches for the same low end + /// of each range the sweep starts from, and a charset is how specific + /// characters are asked for. + #[test] + fn test_generate_strings_interleave_uses_the_same_characters_as_sweep() { + let automaton = automaton_of(".{3}"); + + let interleaved = automaton + .generate_strings(1, 0, PathOrder::Interleave) + .unwrap(); + let swept = automaton.generate_strings(1, 0, PathOrder::Sweep).unwrap(); + assert_eq!(swept, interleaved); + + let lowercase = CharRange::new_from_range_char('a'..='z'); + let options = GenerationOptions::from(PathOrder::Interleave).with_charset(lowercase); + assert_eq!( + vec!["aaa".to_string()], + automaton.generate_strings(1, 0, options).unwrap() + ); + } + + /// A single-path language has only one shape, so there is nothing for the + /// interleave order to spread over: within a path the characters come in + /// the lexicographic order the sweep walks. + #[test] + fn test_generate_strings_interleave_matches_sweep_on_a_single_path() { + let automaton = automaton_of("[a-z]{2}"); + + let swept = automaton.generate_strings(10, 0, PathOrder::Sweep).unwrap(); + let interleaved = automaton + .generate_strings(10, 0, PathOrder::Interleave) + .unwrap(); + + assert_eq!(swept, interleaved); + } + + /// The strongest form of "the axes choose which strings come first, never + /// what is generated": on a finite language, every axis combination — at + /// any seed — enumerates exactly the same set, including through + /// nondeterministic automata, multi-interval charsets, and ranges + /// straddling the surrogate hole. + #[test] + fn test_generate_strings_all_axes_agree_as_sets() { + let multi_interval = CharRange::new_from_ranges(&[ + AnyRange::from(Char::new('x')..=Char::new('z')), + AnyRange::from(Char::new('0')..=Char::new('1')), + ]); + let surrogate_straddle = CharRange::new_from_range_char('\u{d7fe}'..='\u{e001}'); + + let cases: Vec<(&str, Option)> = vec![ + ("(a|bc){0,3}", None), + // Nondeterministic: "ab" is reachable through both branches. + ("(ab|a)b{0,2}", None), + ("[a-e]{0,2}[0-3]?", None), + (".{0,2}", Some(multi_interval)), + (".", Some(surrogate_straddle)), + ]; + + for (pattern, charset) in cases { + for determinize in [false, true] { + let automaton = if determinize { + automaton_of(pattern).determinize().unwrap().into_owned() + } else { + automaton_of(pattern) + }; + + let options = |axes, seed| { + let options = GenerationOptions::from(axes).with_seed(seed); + match &charset { + Some(charset) => options.with_charset(charset.clone()), + None => options, + } + }; + + let mut baseline = automaton + .generate_strings(100_000, 0, options(AXES[0], 0)) + .unwrap(); + baseline.sort(); + + for axes in &AXES[1..] { + for seed in [0, 1, 42] { + let mut strings = automaton + .generate_strings(100_000, 0, options(*axes, seed)) + .unwrap(); + strings.sort(); + + assert_eq!( + baseline, strings, + "{pattern:?} {axes:?} seed {seed} (determinized: {determinize})" + ); + } + } + } + } + } + + /// Pages stay consistent at every chunk size under every axis + /// combination, not only at the window boundaries: an offset landing + /// mid-window or mid-path continues exactly where the previous page + /// stopped, never repeating a string. + #[test] + fn test_generate_strings_pages_at_any_boundary() { + for pattern in ["(a|bc){0,3}", "[a-c]{1,3}", "(x|yy)(0|11)?"] { + // Deterministic and minimal, so pages are exactly disjoint. + let mut automaton = automaton_of(pattern).determinize().unwrap().into_owned(); + automaton.minimize().unwrap(); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_seed(7); + assert_pages_match_bulk(&automaton, &options, 1..=7); + } + } + } + + /// Rebuilds a 60-string bulk page chunk by chunk at each of the given + /// chunk sizes, and asserts every rebuild matches the bulk exactly. + fn assert_pages_match_bulk( + automaton: &FastAutomaton, + options: &GenerationOptions, + chunks: impl IntoIterator, + ) { + let bulk = automaton.generate_strings(60, 0, options.clone()).unwrap(); + + for chunk in chunks { + let mut paged = vec![]; + loop { + let page = automaton + .generate_strings(chunk, paged.len(), options.clone()) + .unwrap(); + if page.is_empty() { + break; + } + paged.extend(page); + if paged.len() >= bulk.len() { + break; + } + } + paged.truncate(bulk.len()); + + assert_eq!(bulk, paged, "{options:?} at chunk size {chunk}"); + } + } + + /// A path holding more combinations than `u128` fits still emits the + /// ascending sequence: the decode consumes the index from the last + /// position, so the positions it never reaches keep their range's first + /// character. + #[test] + fn test_generate_strings_interleave_orders_huge_paths() { + let automaton = automaton_of(".{40}"); + + let interleaved = automaton + .generate_strings(3, 0, PathOrder::Interleave) + .unwrap(); + + assert_eq!( + vec![ + "\u{0}".repeat(40), + format!("{}\u{1}", "\u{0}".repeat(39)), + format!("{}\u{2}", "\u{0}".repeat(39)), + ], + interleaved + ); + } + + /// Shuffling keeps the interleave order's shape-first coverage, and + /// actually looks random: the strings of a wide range are spread over it, + /// not clustered at its low end the way ascending generation starts. + #[test] + fn test_generate_strings_shuffled_covers_shapes_and_spreads_characters() { + let automaton = automaton_of(".*abc.*").determinize().unwrap().into_owned(); + + let strings = automaton + .generate_strings(20, 0, (PathOrder::Shuffled, CharacterOrder::Shuffled)) + .unwrap(); + assert!( + strings.iter().any(|s| !s.starts_with("abc")) + && strings.iter().any(|s| !s.ends_with("abc")), + "shuffled paths still have to cover every shape of the pattern: {strings:?}" + ); + + let strings = automaton_of("[a-z]{20}") + .generate_strings(5, 0, CharacterOrder::Shuffled) + .unwrap(); + assert!( + strings + .iter() + .any(|s| s.chars().filter(|&ch| ch > 'm').count() > 5), + "the strings have to reach past the low end of the range: {strings:?}" + ); + } + + /// The seed is fixed, so shuffled generation is reproducible; a different + /// seed draws a different sequence of strings from the same pattern. + #[test] + fn test_generate_strings_shuffled_is_seeded() { + let automaton = automaton_of("[a-z]{8}"); + let options = |seed| GenerationOptions::from(CharacterOrder::Shuffled).with_seed(seed); + + let strings = automaton.generate_strings(10, 0, options(42)).unwrap(); + assert_eq!( + strings, + automaton.generate_strings(10, 0, options(42)).unwrap() + ); + assert_ne!( + strings, + automaton.generate_strings(10, 0, options(43)).unwrap() + ); + } + + /// Shuffled paths draw the *shapes* by seed, independently of the + /// characters: over ascending characters, which same-length paths a small + /// `limit` reaches depends on the seed instead of always being the same + /// ones — and the whole language still comes out, whatever the seed. + #[test] + fn test_generate_strings_shuffled_paths_draw_shapes_by_seed() { + let automaton = automaton_of("(aa|bb|cc|dd|ee|ff|gg|hh)"); + let options = |seed| GenerationOptions::from(PathOrder::Shuffled).with_seed(seed); + + let first = automaton.generate_strings(3, 0, options(1)).unwrap(); + assert_eq!(first, automaton.generate_strings(3, 0, options(1)).unwrap()); + assert!( + (2..20).any(|seed| automaton.generate_strings(3, 0, options(seed)).unwrap() != first), + "no seed reordered the shapes: {first:?}" + ); + + let mut all = automaton.generate_strings(100, 0, options(1)).unwrap(); + all.sort(); + assert_eq!(vec!["aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh"], all); + } + + /// The axes stay independent the other way around too: shuffling the + /// characters leaves the shape order alone. On single-combination paths + /// there is nothing for the character permutation to reorder, so + /// interleaved generation comes out identical with and without it. + #[test] + fn test_generate_strings_shuffled_characters_leave_the_shape_order_alone() { + let automaton = automaton_of("(aa|bb|cc|dd|ee|ff|gg|hh)"); + + let ascending = automaton + .generate_strings(8, 0, PathOrder::Interleave) + .unwrap(); + let shuffled = automaton + .generate_strings(8, 0, (PathOrder::Interleave, CharacterOrder::Shuffled)) + .unwrap(); + + assert_eq!(ascending, shuffled); + } + + /// Shuffled paths over ascending characters: a seed-drawn order of + /// shapes, each shown as its smallest witness. + #[test] + fn test_generate_strings_shuffled_paths_keep_ascending_witnesses() { + let automaton = automaton_of("(aa|bb|cc)[0-9]"); + + for seed in [0, 1, 42] { + let options = GenerationOptions::from(PathOrder::Shuffled).with_seed(seed); + let mut strings = automaton.generate_strings(3, 0, options).unwrap(); + + // Whatever order the seed drew the three shapes in, the first + // string of each is the low end of its ranges. + strings.sort(); + assert_eq!(vec!["aa0", "bb0", "cc0"], strings, "seed {seed}"); + } + } + + /// `with_max_length` bounds the generated string length: a deep offset + /// into `.*` pages within the bound instead of into arbitrarily long + /// strings — and comes back quickly, whatever the axes. + #[test] + fn test_generate_strings_max_length_bounds_deep_offsets() { + let automaton = automaton_of(".*"); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_max_length(5); + let strings = automaton + .generate_strings(5, 1_000_000_000, options) + .unwrap(); + + assert!(!strings.is_empty(), "{axes:?}"); + for string in &strings { + assert!(string.chars().count() <= 5, "{axes:?}: {string:?}"); + } + } + } + + /// The bounded language is a well-defined finite set: `(ab)*` under a + /// max of 5 stops at `abab`, and the page past it is empty, not endless. + #[test] + fn test_generate_strings_max_length_truncates_the_language() { + let automaton = automaton_of("(ab)*"); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_max_length(5); + let mut strings = automaton.generate_strings(100, 0, options.clone()).unwrap(); + strings.sort(); + assert_eq!(vec!["", "ab", "abab"], strings, "{axes:?}"); + + let past_the_end = automaton.generate_strings(10, 3, options).unwrap(); + assert!(past_the_end.is_empty(), "{axes:?}: {past_the_end:?}"); + } + } + + /// The bound is exactly what it says — a length: a finite language keeps + /// every string within it and loses every string past it. + #[test] + fn test_generate_strings_max_length_applies_to_finite_languages_too() { + let automaton = automaton_of("(ab){1,2}"); + + for axes in AXES { + let mut strings = automaton + .generate_strings(10, 0, GenerationOptions::from(axes).with_max_length(4)) + .unwrap(); + strings.sort(); + assert_eq!(vec!["ab", "abab"], strings, "{axes:?}"); + + assert_eq!( + vec!["ab".to_string()], + automaton + .generate_strings(10, 0, GenerationOptions::from(axes).with_max_length(3)) + .unwrap(), + "{axes:?}" + ); + } + } + + /// `with_min_length` leaves the short strings out: the enumeration + /// starts at the bound, and an empty band generates nothing. + #[test] + fn test_generate_strings_min_length_skips_short_strings() { + let automaton = automaton_of("(ab)*"); + + for axes in AXES { + let options = GenerationOptions::from(axes) + .with_min_length(3) + .with_max_length(8); + let mut strings = automaton.generate_strings(100, 0, options).unwrap(); + strings.sort(); + assert_eq!(vec!["abab", "ababab", "abababab"], strings, "{axes:?}"); + + let empty_band = GenerationOptions::from(axes) + .with_min_length(5) + .with_max_length(3); + assert!( + automaton + .generate_strings(10, 0, empty_band) + .unwrap() + .is_empty(), + "{axes:?}" + ); + } + } + + /// Length bounds compose with paging: `offset` never counts the strings + /// outside the band, so pages of the bounded language stay consistent at + /// any chunk size. + #[test] + fn test_generate_strings_length_bounds_page_consistently() { + // Deterministic and minimal, so pages are exactly disjoint. + let mut automaton = automaton_of("(a|bc){0,3}") + .determinize() .unwrap() - .to_automaton() + .into_owned(); + automaton.minimize().unwrap(); + + for axes in AXES { + let options = GenerationOptions::from(axes) + .with_seed(7) + .with_min_length(2) + .with_max_length(4); + + let bulk = automaton.generate_strings(60, 0, options.clone()).unwrap(); + assert!( + bulk.iter().all(|s| (2..=4).contains(&s.chars().count())), + "{axes:?}: {bulk:?}" + ); + + assert_pages_match_bulk(&automaton, &options, [1, 3]); + } + } + + /// The permuter maps `[0, bound)` onto itself one-to-one for any bound + /// and tweak — what "distinct strings across offsets" rests on. + #[test] + fn test_permuter_is_a_bijection() { + for seed in [0, 1, 42] { + let permuter = Permuter::new(seed); + for bound in [1u128, 2, 3, 7, 26, 100, 4096, 100_003] { + for tweak in [0, permuter.path_tweak(&[3, 1, 4])] { + let mut images: Vec = (0..bound) + .map(|index| permuter.permute(index, bound, tweak)) + .collect(); + images.sort_unstable(); + + assert!( + images.iter().enumerate().all(|(i, &v)| i as u128 == v), + "seed {seed}, bound {bound}, tweak {tweak}" + ); + } + } + } + } + + /// Every path draws its own permutation: alternation branches of the same + /// shape emit unrelated strings instead of mirroring each other's + /// combination indices ("knyn" next to "KNYN"). + #[test] + fn test_generate_strings_shuffled_decorrelates_same_shape_branches() { + let automaton = automaton_of("([a-z]{6}|[A-Z]{6})"); + + for seed in [0, 1, 42] { + let options = GenerationOptions::from(CharacterOrder::Shuffled) + .with_paths(PathOrder::Interleave) + .with_seed(seed); + let strings = automaton.generate_strings(2, 0, options).unwrap(); + + let [first, second] = strings.as_slice() else { + panic!("expected one string per branch: {strings:?}"); + }; + assert_ne!( + first.to_lowercase(), + second.to_lowercase(), + "seed {seed}: the branches drew the same combination" + ); + } + } + + /// A charset rules out whole paths, not single characters: a path that + /// needs a ruled-out character is dropped even when it only needs it + /// several transitions in, and the strings around it still come out. + #[test] + fn test_generate_strings_charset_drops_the_paths_it_closes() { + let automaton = automaton_of("(a[0-9]b|xyz)"); + let letters = CharRange::new_from_range_char('a'..='z'); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_charset(letters.clone()); + + assert_eq!( + vec!["xyz".to_string()], + automaton.generate_strings(10, 0, options).unwrap(), + "{axes:?}" + ); + } + } + + /// A charset that leaves the pattern nothing is an empty page, not an + /// error: the search prunes the start state like any other dead end. + #[test] + fn test_generate_strings_charset_can_leave_nothing() { + let automaton = automaton_of("[0-9]+"); + let letters = CharRange::new_from_range_char('a'..='z'); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_charset(letters.clone()); + + assert!( + automaton + .generate_strings(10, 0, options) + .unwrap() + .is_empty(), + "{axes:?}" + ); + } + } + + /// The charset narrows the ranges the strings are built from, so what is + /// generated is the language the pattern and the charset agree on. + #[test] + fn test_generate_strings_charset_narrows_every_range() { + let automaton = automaton_of(".{2}"); + let vowels = CharRange::new_from_ranges(&[ + AnyRange::from(Char::new('a')..=Char::new('a')), + AnyRange::from(Char::new('e')..=Char::new('e')), + ]); + + for axes in AXES { + let options = GenerationOptions::from(axes).with_charset(vowels.clone()); + + let mut strings = automaton.generate_strings(10, 0, options).unwrap(); + strings.sort(); + + assert_eq!(vec!["aa", "ae", "ea", "ee"], strings, "{axes:?}"); + } + } + + /// The offset steps over whole subtrees at once: a page from deep inside a + /// large language comes back without walking everything before it. + #[test] + fn test_generate_strings_offset_reaches_deep_pages() { + let automaton = automaton_of("[a-z]{5}"); + let total = 26usize.pow(5); + + let strings = automaton + .generate_strings(2, total - 2, PathOrder::Sweep) .unwrap(); - println!("{}", automaton.get_number_of_states()); - //automaton.to_dot(); + + assert_eq!(vec!["zzzzy".to_string(), "zzzzz".to_string()], strings); + } + + /// The cursor hands out exactly the characters past its opening ordinal, + /// across intervals and over the surrogate hole, like the plain iterator. + #[test] + fn test_range_cursor_opens_at_any_ordinal() { + let range = CharRange::new_from_ranges(&[ + AnyRange::from(Char::new('0')..=Char::new('9')), + AnyRange::from(Char::new('\u{d7fe}')..=Char::new('\u{e001}')), + ]); + + for start in 0..=range.get_cardinality() { + let expected: Vec = range + .iter() + .skip(start as usize) + .map(|c| c.to_char()) + .collect(); + + let mut cursor = RangeCursor::new(&range, start); + let mut walked = vec![]; + while let Some(ch) = cursor.next() { + walked.push(ch.to_char()); + } + + assert_eq!(expected, walked, "start {start}"); + } + } + + /// A huge `limit` means "everything the language holds", not "reserve this + /// much memory": it must not size an allocation before generation starts. + #[test] + fn test_generate_strings_limit_does_not_preallocate() { + let automaton = automaton_of("[ab]{2}"); + + for axes in AXES { + let mut strings = automaton.generate_strings(usize::MAX, 0, axes).unwrap(); + strings.sort(); + + assert_eq!(vec!["aa", "ab", "ba", "bb"], strings, "{axes:?}"); + } + } + + /// Combination emission walks an explicit stack, not the call stack: a + /// path tens of thousands of transitions long emits without overflowing. + #[test] + fn test_generate_strings_very_long_string() { + let automaton = automaton_of("[ab]{20000}"); + + for axes in AXES { + let strings = automaton.generate_strings(2, 0, axes).unwrap(); + assert_eq!(2, strings.len(), "{axes:?}"); + for string in &strings { + assert_eq!(20_000, string.len(), "{axes:?}"); + } + } + } + + /// The surrogate block is not a character, so the whole alphabet is one + /// character shorter than its last code point suggests. + #[test] + fn test_char_at_walks_over_the_surrogate_block() { + let total = CharRange::total(); + + assert_eq!('\u{0}', char_at(&total, 0).unwrap().to_char()); + assert_eq!('\u{d7ff}', char_at(&total, 0xd7ff).unwrap().to_char()); + assert_eq!('\u{e000}', char_at(&total, 0xd800).unwrap().to_char()); + + let cardinality = total.get_cardinality(); + assert_eq!( + '\u{10ffff}', + char_at(&total, cardinality - 1).unwrap().to_char() + ); + assert!(char_at(&total, cardinality).is_none()); + } + + fn automaton_of(regex: &str) -> FastAutomaton { + RegularExpression::parse(regex, false) + .unwrap() + .to_automaton() + .unwrap() + } + + fn assert_generate_strings_offset(regex: &str) { + println!("regex: {regex}"); + let automaton = automaton_of(regex); + + for options in all_options() { + // Generate 30 strings at once + let all_strings = automaton.generate_strings(30, 0, options.clone()).unwrap(); + + // Generate the same 30 strings in chunks of 10 + let chunk1 = automaton.generate_strings(10, 0, options.clone()).unwrap(); + let chunk2 = automaton.generate_strings(10, 10, options.clone()).unwrap(); + let chunk3 = automaton.generate_strings(10, 20, options.clone()).unwrap(); + + assert_eq!( + all_strings.len(), + 30, + "Should generate exactly 30 strings ({options:?})" + ); + assert_eq!(chunk1.len(), 10, "{options:?}"); + assert_eq!(chunk2.len(), 10, "{options:?}"); + assert_eq!(chunk3.len(), 10, "{options:?}"); + + // Combine the chunks + let mut combined = chunk1; + combined.extend(chunk2); + combined.extend(chunk3); + + // Prove that generating in chunks perfectly matches the bulk generation + assert_eq!( + all_strings, combined, + "Chunked generation did not match bulk generation ({options:?})" + ); + + let cardinality = automaton.cardinality().unwrap(); + + // A charset only ever leaves fewer strings than the automaton + // holds, so its count is past the end of the restricted language + // too. + if let Cardinality::Integer(count) = cardinality { + let empty_chunk = automaton + .generate_strings(10, count as usize, options.clone()) + .unwrap(); + assert!( + empty_chunk.is_empty(), + "Chunk past limits should be empty ({options:?})" + ); + } + } + } + + fn assert_generate_strings(regex: &str, number: usize) { + println!(":{}", regex); + let automaton = automaton_of(regex); + let re = Regex::new(&format!("(?s)^{}$", regex)).unwrap(); - let strings = automaton.generate_strings(number).unwrap(); - let mut strings: Vec<_> = strings.iter().collect(); - strings.sort_unstable(); - println!("nb of strings: {}/{}", strings.len(), number); - assert!(number >= strings.len()); - for string in strings { - if !re.is_match(string) { - for byte in string.as_bytes() { - print!("{:02x} ", byte); + for options in all_options() { + let strings = automaton + .generate_strings(number, 0, options.clone()) + .unwrap(); + println!("nb of strings ({options:?}): {}/{}", strings.len(), number); + assert!(number >= strings.len()); + + let distinct: std::collections::HashSet<_> = strings.iter().collect(); + assert_eq!( + distinct.len(), + strings.len(), + "the same string came up twice ({options:?})" + ); + + for string in strings { + if let Some(charset) = options.charset() { + assert!( + string.chars().all(|ch| charset.contains(Char::new(ch))), + "'{string}' uses characters outside the charset" + ); + } + if !re.is_match(&string) { + for byte in string.as_bytes() { + print!("{:02x} ", byte); + } + panic!("'{string}' ({options:?})") } - panic!("'{string}'") } - assert!(re.is_match(string), "'{string}'"); } } } diff --git a/src/fast_automaton/mod.rs b/src/fast_automaton/mod.rs index 6d6fcbc..bdad488 100644 --- a/src/fast_automaton/mod.rs +++ b/src/fast_automaton/mod.rs @@ -1,29 +1,64 @@ -use crate::Range; +use crate::error::EngineError; use ahash::{AHashMap, HashSetExt}; use condition::Condition; use regex_charclass::CharacterClass; use spanning_set::SpanningSet; -use std::collections::hash_map::Entry; use std::collections::VecDeque; +use std::collections::hash_map::Entry; use std::fmt::Display; -use crate::{IntMap, IntSet}; +use super::*; -pub(crate) type State = usize; pub(crate) type Transitions = IntMap; +/// The identifier of a state in a [`FastAutomaton`]. +pub type State = usize; + mod analyze; mod builder; +/// Transition labels: the bitvector [`Condition`] type over an automaton's +/// spanning set of disjoint character ranges. pub mod condition; mod convert; mod generate; mod operation; -#[cfg(feature = "serde")] -mod serializer; +/// The [`SpanningSet`]: an automaton's partition of the alphabet into disjoint +/// character ranges, over which transition conditions are defined. pub mod spanning_set; -/// Represent a finite state automaton. +pub use generate::{CharacterOrder, GenerationOptions, PathOrder}; + +/// The block of code points `char` cannot hold: [`regex_charclass::char::Char`] +/// values skip it, so scalar values have to be shifted down past it to be +/// counted. +const SURROGATES: std::ops::Range = 0xD800..0xE000; + +/// The index of `ch` among all the characters, the surrogate block excluded. +/// Consecutive scalars are consecutive `Char`s, so `+ 1` arithmetic in scalar +/// space cannot land inside the surrogate hole. +#[inline] +fn scalar(ch: regex_charclass::char::Char) -> u32 { + let code = ch.to_u32(); + if code >= SURROGATES.end { + code - (SURROGATES.end - SURROGATES.start) + } else { + code + } +} + +/// The inverse of [`scalar`]. +#[inline] +fn from_scalar(index: u32) -> Option { + regex_charclass::char::Char::from_u32(if index >= SURROGATES.start { + index + (SURROGATES.end - SURROGATES.start) + } else { + index + }) +} + +/// Represents a finite-state automaton. #[derive(Clone, Debug, PartialEq, Eq)] +#[must_use = "non-`_mut` operations return a new automaton"] pub struct FastAutomaton { transitions: Vec, transitions_in: IntMap>, @@ -32,37 +67,49 @@ pub struct FastAutomaton { removed_states: IntSet, spanning_set: SpanningSet, deterministic: bool, - cyclic: bool, + minimal: bool, +} + +/// Returned by [`FastAutomaton::try_add_transition`] when adding the requested +/// condition would turn a DFA into an NFA. The automaton is left unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeterminismLost; + +impl std::fmt::Display for DeterminismLost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "adding the transition would introduce overlapping conditions" + ) + } } +impl std::error::Error for DeterminismLost {} + impl Display for FastAutomaton { fn fmt(&self, sb: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(sb, "digraph Automaton {{")?; writeln!(sb, "\trankdir = LR;")?; - for from_state in self.transitions_iter() { - write!(sb, "\t{}", from_state)?; + for from_state in self.states() { + write!(sb, "\t{from_state}")?; if self.accept_states.contains(&from_state) { - writeln!(sb, "\t[shape=doublecircle,label=\"{}\"];", from_state)?; + writeln!(sb, "\t[shape=doublecircle,label=\"{from_state}\"];")?; } else { - writeln!(sb, "\t[shape=circle,label=\"{}\"];", from_state)?; + writeln!(sb, "\t[shape=circle,label=\"{from_state}\"];")?; } if self.start_state == from_state { writeln!(sb, "\tinitial [shape=plaintext,label=\"\"];")?; - writeln!(sb, "\tinitial -> {}", from_state)?; + writeln!(sb, "\tinitial -> {from_state}")?; } - for (to_state, cond) in self.transitions_from_state_enumerate_iter(&from_state) { - writeln!( - sb, - "\t{} -> {} [label=\"{}\"]", - from_state, - to_state, - cond.to_range(&self.spanning_set) - .expect("Cannot convert condition to range.") - .to_regex() - .replace('\\', "\\\\") - .replace('"', "\\\"") - )?; + for (cond, to_state) in self.transitions_from(from_state) { + // The automata most worth printing are the broken ones: + // never panic mid-format, label desynced conditions instead. + let label = match cond.to_range(&self.spanning_set) { + Ok(range) => range.to_regex().replace('\\', "\\\\").replace('"', "\\\""), + Err(_) => String::from(""), + }; + writeln!(sb, "\t{from_state} -> {to_state} [label=\"{label}\"]")?; } } write!(sb, "}}") @@ -73,86 +120,104 @@ impl FastAutomaton { #[inline] fn assert_state_exists(&self, state: State) { if !self.has_state(state) { - panic!("The state {} does not exist", state); + panic!("The state {state} does not exist"); } } + /// Returns the number of transitions to the provided state. #[inline] pub fn in_degree(&self, state: State) -> usize { - self.transitions_in - .get(&state) - .unwrap_or(&IntSet::new()) - .len() + self.transitions_in.get(&state).map_or(0, IntSet::len) } + /// Returns the number of transitions from the provided state. + /// Returns `0` if the state does not exist. #[inline] pub fn out_degree(&self, state: State) -> usize { - self.transitions[state].len() - } - - pub fn in_transitions(&self, state: State) -> Vec<(usize, Condition)> { - let mut in_transitions = vec![]; - for from_state in self.transitions_in.get(&state).unwrap_or(&IntSet::new()) { - for (to_state, condition) in self.transitions_from_state_enumerate_vec(from_state) { - if to_state == state { - in_transitions.push((*from_state, condition)); - } - } + if !self.has_state(state) { + return 0; } - in_transitions - } - - pub fn in_states(&self, state: State) -> IntSet { - self.transitions_in - .get(&state) - .unwrap_or(&IntSet::new()) - .clone() + self.transitions[state].len() } + /// Returns an iterator over the automaton’s states. #[inline] - pub fn transitions_iter(&self) -> impl Iterator + '_ { + pub fn states(&self) -> impl Iterator + '_ { (0..self.transitions.len()).filter(|s| !self.removed_states.contains(s)) } + /// Returns a vector containing the automaton’s states. #[inline] - pub fn transitions_vec(&self) -> Vec { - self.transitions_iter().collect() + pub fn states_vec(&self) -> Vec { + self.states().collect() } + /// Returns an iterator over states directly reachable from the given state in one transition. + /// Returns an empty iterator if the state does not exist. #[inline] - pub fn transitions_from_state_enumerate_iter( - &self, - from_state: &State, - ) -> impl Iterator { - self.transitions[*from_state] - .iter() - .filter(|s| !self.removed_states.contains(s.0)) + pub fn direct_states(&self, state: State) -> impl Iterator + '_ { + self.transitions.get(state).into_iter().flat_map(move |t| { + t.keys() + .copied() + .filter(|s| !self.removed_states.contains(s)) + }) } + /// Returns a vector of states directly reachable from the given state in one transition. #[inline] - pub fn transitions_from_state_enumerate_iter_mut( - &mut self, - from_state: &State, - ) -> impl Iterator { - self.transitions[*from_state] - .iter_mut() - .filter(|s| !self.removed_states.contains(s.0)) + pub fn direct_states_vec(&self, state: State) -> Vec { + self.direct_states(state).collect() } + /// Returns a vector of transitions to the given state. + pub fn transitions_to_vec(&self, state: State) -> Vec<(State, Condition)> { + if !self.has_state(state) { + return vec![]; + } + let Some(predecessors) = self.transitions_in.get(&state) else { + return vec![]; + }; + let mut in_transitions = vec![]; + for from_state in predecessors { + if !self.has_state(*from_state) { + continue; + } + if let Some(condition) = self.condition(*from_state, state) { + in_transitions.push((*from_state, condition.clone())); + } + } + in_transitions + } + + /// Returns a vector of transitions from the given state. + /// Returns an empty vector if the state does not exist. #[inline] - pub fn transitions_from_state_enumerate_vec( - &self, - from_state: &State, - ) -> Vec<(State, Condition)> { - self.transitions[*from_state] - .iter() - .map(|(s, c)| (*s, c.clone())) - .filter(|s| !self.removed_states.contains(&s.0)) - .collect() + pub fn transitions_from_vec(&self, state: State) -> Vec<(Condition, State)> { + self.transitions + .get(state) + .map(|t| { + t.iter() + .map(|(s, c)| (c.clone(), *s)) + .filter(|s| !self.removed_states.contains(&s.1)) + .collect() + }) + .unwrap_or_default() + } + + /// Returns an iterator over transitions from the given state. + /// Returns an empty iterator if the state does not exist. + #[inline] + pub fn transitions_from(&self, state: State) -> impl Iterator { + self.transitions.get(state).into_iter().flat_map(move |t| { + t.iter() + .map(|(s, c)| (c, s)) + .filter(|s| !self.removed_states.contains(s.1)) + }) } + /// Returns `true` if there is a directed transition from `from_state` to `to_state`. #[inline] - pub fn does_transition_exists(&self, from_state: State, to_state: State) -> bool { + pub fn has_transition(&self, from_state: State, to_state: State) -> bool { if !self.has_state(from_state) || !self.has_state(to_state) { return false; } @@ -173,126 +238,118 @@ impl FastAutomaton { .collect() } + /// Returns the number of states in the automaton. #[inline] - pub fn transitions_from_state_enumerate_into_iter( - &self, - from_state: &State, - ) -> impl Iterator + '_ { - self.transitions - .get(*from_state) // Assume transitions is a map; adjust accordingly. - .into_iter() // Creates an iterator over Option<&V> - .flat_map(|transitions| transitions.iter()) // Flattens into Iterator - .filter(move |(state, _)| !self.removed_states.contains(state)) // Filters out removed states - .map(|(state, condition)| (*state, condition.clone())) // Creates owned data; adjust if cloning is expensive - } - - #[inline] - pub fn transitions_from_state_iter( - &self, - from_state: &State, - ) -> impl Iterator + '_ { - self.transitions[*from_state] - .keys() - .cloned() - .filter(|s| !self.removed_states.contains(s)) - } - - #[inline] - pub fn transitions_from_state(&self, from_state: &State) -> Vec { - self.transitions_from_state_iter(from_state).collect() - } - - #[inline] - pub fn transitions_from_state_into_iter<'a>( - &'a self, - from_state: &State, - ) -> impl Iterator + 'a { - self.transitions[*from_state] - .clone() - .into_iter() - .filter(|s| !self.removed_states.contains(&s.0)) - } - - #[inline] - pub fn get_number_of_states(&self) -> usize { + pub fn number_of_states(&self) -> usize { self.transitions.len() - self.removed_states.len() } + /// Returns a reference to the condition of the directed transition between the two states, if any. + /// Returns `None` if either state does not exist. #[inline] - pub fn get_condition(&self, from_state: &State, to_state: &State) -> Option<&Condition> { - self.transitions[*from_state].get(to_state) + pub fn condition(&self, from_state: State, to_state: State) -> Option<&Condition> { + self.transitions + .get(from_state) + .and_then(|t| t.get(&to_state)) } + /// Returns the start state. #[inline] - pub fn get_start_state(&self) -> State { + pub fn start_state(&self) -> State { self.start_state } + /// Returns a reference to the set of accept (final) states. #[inline] - pub fn get_removed_states(&self) -> &IntSet { - &self.removed_states - } - - #[inline] - pub fn get_accept_states(&self) -> &IntSet { + pub fn accept_states(&self) -> &IntSet { &self.accept_states } + /// Returns a reference to the automaton's spanning set. #[inline] - pub fn get_spanning_set(&self) -> &SpanningSet { + pub fn spanning_set(&self) -> &SpanningSet { &self.spanning_set } + /// Returns `true` if the given state is one of the accept states. #[inline] - pub fn is_accepted(&self, state: &State) -> bool { - self.accept_states.contains(state) + pub fn is_accepted(&self, state: State) -> bool { + self.accept_states.contains(&state) } + /// Returns `true` if the automaton is deterministic. + /// + /// Note: this flag degrades monotonically. Once `add_transition` introduces + /// an overlapping condition, the flag flips to `false` and is not + /// re-checked by `remove_transition` or `remove_state`. The automaton may + /// in fact be deterministic again after such removals; call + /// [`determinize`](Self::determinize) if you need a fresh DFA. #[inline] - pub fn is_determinitic(&self) -> bool { + pub fn is_deterministic(&self) -> bool { self.deterministic } + /// Returns `true` if the automaton is minimal. #[inline] - pub fn is_cyclic(&self) -> bool { - self.cyclic + pub fn is_minimal(&self) -> bool { + self.minimal } + /// Returns `true` if the automaton contains the given state. #[inline] pub fn has_state(&self, state: State) -> bool { !(state >= self.transitions.len() || self.removed_states.contains(&state)) } - pub fn match_string(&self, input: &str) -> bool { - let mut worklist = VecDeque::with_capacity(self.get_number_of_states()); - worklist.push_back((0, &self.start_state)); + /// Returns `true` if the automaton matches the given string. + #[tracing::instrument(level = "debug", skip(self, string), fields(states = self.number_of_states(), string_len=string.len()))] + pub fn is_match(&self, string: &str) -> bool { + let mut current: IntSet = IntSet::default(); + current.insert(self.start_state); - while let Some((position, current_state)) = worklist.pop_back() { - if input.len() == position { - if self.accept_states.contains(current_state) { - return true; - } - continue; + let mut next: IntSet = IntSet::default(); + for c in string.chars() { + if current.is_empty() { + return false; } - let curr_char = input.chars().nth(position).unwrap() as u32; - for (to_state, cond) in self.transitions_from_state_enumerate_iter(current_state) { - if cond.has_character(&curr_char, &self.spanning_set).unwrap() { - if position + 1 == input.len() { - if self.accept_states.contains(to_state) { - return true; + let c_u32 = c as u32; + next.clear(); + for &state in ¤t { + for (cond, to_state) in self.transitions_from(state) { + // A condition/spanning-set mismatch is a broken internal + // invariant; make it loud in debug builds instead of + // silently treating the transition as non-matching. + let matches = match cond.has_character(&c_u32, &self.spanning_set) { + Ok(matches) => matches, + Err(error) => { + debug_assert!( + false, + "condition desynchronized from spanning set: {error}" + ); + false } - } else { - worklist.push_back((position + 1, to_state)); + }; + if matches { + next.insert(*to_state); } } } + std::mem::swap(&mut current, &mut next); } - false + + current.iter().any(|s| self.accept_states.contains(s)) } + /// Returns the automaton's DOT representation. #[inline] - pub fn to_dot(&self) { - println!("{}", self); + pub fn to_dot(&self) -> String { + format!("{self}") + } + + /// Prints the automaton's DOT representation. + #[inline] + pub fn print_dot(&self) { + println!("{self}"); } } @@ -315,4 +372,82 @@ mod tests { assert!(automaton.is_total()); Ok(()) } + + fn assert_send() {} + fn assert_sync() {} + + #[test] + fn test_traits() -> Result<(), String> { + assert_send::(); + assert_sync::(); + + Ok(()) + } + + // Read-only query methods must return gracefully (0 / None / empty + // iterator) on out-of-range or unknown states, not index out of bounds. + #[test] + fn out_degree_safe_on_unknown_state() { + let a = FastAutomaton::new_total(); + assert_eq!(a.out_degree(999), 0); + } + + #[test] + fn condition_safe_on_unknown_state() { + let a = FastAutomaton::new_total(); + assert!(a.condition(999, 0).is_none()); + assert!(a.condition(0, 999).is_none()); + } + + #[test] + fn direct_states_safe_on_unknown_state() { + let a = FastAutomaton::new_total(); + assert_eq!(a.direct_states(999).count(), 0); + assert_eq!(a.transitions_from(999).count(), 0); + assert!(a.transitions_from_vec(999).is_empty()); + assert!(a.direct_states_vec(999).is_empty()); + } + + #[test] + #[should_panic(expected = "does not exist")] + fn remove_states_panics_clearly_on_out_of_range() { + let mut a = FastAutomaton::new_total(); + let mut states = IntSet::default(); + states.insert(999); + a.remove_states(&states); + } + + #[test] + #[should_panic(expected = "does not exist")] + fn remove_states_panics_clearly_on_tombstoned_id() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + // Remove the trailing state so it becomes a tombstone. + let mut first = IntSet::default(); + first.insert(s2); + a.remove_states(&first); + // Removing it again must fail cleanly, not panic on an OOB index. + let mut again = IntSet::default(); + again.insert(s2); + a.remove_states(&again); + let _ = s1; + } + + // A valid multi-state removal (including a trailing id) still works. + #[test] + fn remove_states_removes_valid_ids() { + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + let s3 = a.new_state(); + let mut states = IntSet::default(); + states.insert(s1); + states.insert(s3); // trailing + a.remove_states(&states); + assert!(a.has_state(0)); + assert!(a.has_state(s2)); + assert!(!a.has_state(s1)); + assert!(!a.has_state(s3)); + } } diff --git a/src/fast_automaton/operation/alternation.rs b/src/fast_automaton/operation/alternation.rs deleted file mode 100644 index 06c386e..0000000 --- a/src/fast_automaton/operation/alternation.rs +++ /dev/null @@ -1,274 +0,0 @@ -use std::hash::BuildHasherDefault; - -use condition::converter::ConditionConverter; - -use crate::error::EngineError; - -use super::*; - -impl FastAutomaton { - pub fn union(&self, that: &FastAutomaton) -> Result { - let mut union = self.clone(); - union.alternate(that)?; - Ok(union) - } - - pub fn alternation(automatons: Vec) -> Result { - if automatons.len() == 1 { - return Ok(automatons[0].clone()); - } - let mut new_automaton = FastAutomaton::new_empty(); - if automatons.is_empty() { - return Ok(new_automaton); - } - for automaton in automatons { - new_automaton.alternate(&automaton)?; - } - Ok(new_automaton) - } - - fn prepare_start_states( - &mut self, - other: &FastAutomaton, - new_states: &mut IntMap, - condition_converter: &ConditionConverter, - ) -> Result, EngineError> { - let mut imcomplete_states = IntSet::with_capacity(other.out_degree(other.start_state) + 1); - let self_start_state_in_degree = self.in_degree(self.start_state); - let other_start_state_in_degree = other.in_degree(other.start_state); - if self_start_state_in_degree == 0 && other_start_state_in_degree == 0 { - // The start states can be the same state without any consequence - new_states.insert(other.start_state, self.start_state); - imcomplete_states.insert(self.start_state); - } else { - if self_start_state_in_degree != 0 { - let new_state = self.new_state(); - if self.is_accepted(&self.start_state) { - self.accept(new_state); - } - - for (to_state, cond) in self.transitions_from_state_enumerate_vec(&self.start_state) - { - self.add_transition_to(new_state, to_state, &cond); - } - self.start_state = new_state; - } - if other_start_state_in_degree != 0 { - let new_state = self.new_state(); - if other.is_accepted(&other.start_state) { - self.accept(new_state); - self.accept(self.start_state); - } - - new_states.insert(other.start_state, new_state); - imcomplete_states.insert(new_state); - - for (other_to_state, cond) in - other.transitions_from_state_enumerate_vec(&other.start_state) - { - let cond = condition_converter.convert(&cond)?; - let to_state = match new_states.entry(other_to_state) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - let new_state = self.new_state(); - imcomplete_states.insert(new_state); - v.insert(new_state); - new_state - } - }; - self.add_transition_to(self.start_state, to_state, &cond); - } - } - } - Ok(imcomplete_states) - } - - fn prepare_accept_states( - &mut self, - other: &FastAutomaton, - new_states: &mut IntMap, - imcomplete_states: &IntSet, - ) { - let mut self_accept_states_without_outgoing_edges = vec![]; - for &state in &self.accept_states { - if self.out_degree(state) == 0 && !imcomplete_states.contains(&state) { - self_accept_states_without_outgoing_edges.push(state); - } - } - let accept_state_without_outgoing_edges = - match self_accept_states_without_outgoing_edges.len() { - 1 => self_accept_states_without_outgoing_edges[0], - n if n > 1 => { - let new_state = self.new_state(); - self.accept(new_state); - - for &accept_state in &self_accept_states_without_outgoing_edges { - for (from_state, condition) in self.in_transitions(accept_state) { - self.add_transition_to(from_state, new_state, &condition); - } - self.remove_state(accept_state); - } - new_state - } - _ => { - let new_state = self.new_state(); - self.accept(new_state); - new_state - } - }; - - for &state in &other.accept_states { - if other.out_degree(state) == 0 { - new_states - .entry(state) - .or_insert(accept_state_without_outgoing_edges); - } else if new_states.get(&state).is_none() { - let new_accept_state = self.new_state(); - self.accept(new_accept_state); - new_states.insert(state, new_accept_state); - } - } - } - - /* Important things to remember before modifying this method: - * - the start states can't be merged if they have incoming edges - * - the accept states can't be merged if they have outgoing edges - */ - fn alternate(&mut self, other: &FastAutomaton) -> Result<(), EngineError> { - if other.is_empty() || self.is_total() { - return Ok(()); - } else if other.is_total() { - self.make_total(); - return Ok(()); - } else if self.is_empty() { - self.apply_model(other); - return Ok(()); - } - - let new_spanning_set = &self.spanning_set.merge(&other.spanning_set); - self.apply_new_spanning_set(new_spanning_set)?; - let condition_converter = ConditionConverter::new(&other.spanning_set, new_spanning_set)?; - - let mut new_states: IntMap = IntMap::with_capacity_and_hasher( - other.get_number_of_states(), - BuildHasherDefault::default(), - ); - - let imcomplete_states = - self.prepare_start_states(other, &mut new_states, &condition_converter)?; - self.prepare_accept_states(other, &mut new_states, &imcomplete_states); - - for from_state in other.transitions_iter() { - let new_from_state = match new_states.entry(from_state) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - let new_state = self.new_state(); - v.insert(new_state); - new_state - } - }; - for (to_state, condition) in other.transitions_from_state_enumerate_iter(&from_state) { - let new_condition = condition_converter.convert(condition)?; - let new_to_state = match new_states.entry(*to_state) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - let new_state = self.new_state(); - v.insert(new_state); - new_state - } - }; - self.add_transition_to(new_from_state, new_to_state, &new_condition); - } - } - self.cyclic = self.cyclic || other.cyclic; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::regex::RegularExpression; - - #[test] - fn test_simple_alternation_regex_1() -> Result<(), String> { - let automaton = RegularExpression::new("(abc|ac|aaa)") - .unwrap() - .to_automaton() - .unwrap(); - assert!(automaton.match_string("abc")); - assert!(automaton.match_string("ac")); - assert!(automaton.match_string("aaa")); - assert!(!automaton.match_string("abcd")); - assert!(!automaton.match_string("ab")); - assert!(!automaton.match_string("acc")); - assert!(!automaton.match_string("a")); - assert!(!automaton.match_string("aaaa")); - assert!(!automaton.match_string("aa")); - assert!(!automaton.match_string("")); - Ok(()) - } - - #[test] - fn test_simple_alternation_regex_2() -> Result<(), String> { - let automaton = RegularExpression::new("(b?|b{2})") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("b")); - assert!(automaton.match_string("bb")); - assert!(!automaton.match_string("bbb")); - assert!(!automaton.match_string("bbbb")); - Ok(()) - } - - #[test] - fn test_simple_alternation_regex_3() -> Result<(), String> { - let automaton = RegularExpression::new("((a|bc)*|d)") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("abcaaabcbc")); - assert!(automaton.match_string("d")); - assert!(!automaton.match_string("ad")); - assert!(!automaton.match_string("abcd")); - Ok(()) - } - - #[test] - fn test_simple_alternation_regex_4() -> Result<(), String> { - let automaton = RegularExpression::new("(a+(ba+)*|ca*c)") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("cc")); - assert!(automaton.match_string("caaac")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aababa")); - Ok(()) - } - - #[test] - fn test_simple_alternation_regex_5() -> Result<(), String> { - let automaton = RegularExpression::new("((aad|ads|a)*|q)") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("q")); - assert!(automaton.match_string("aad")); - assert!(automaton.match_string("ads")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aadadsaaa")); - assert!(!automaton.match_string("aaaas")); - assert!(!automaton.match_string("ad")); - assert!(!automaton.match_string("adsq")); - assert!(!automaton.match_string("qq")); - Ok(()) - } -} diff --git a/src/fast_automaton/operation/concat.rs b/src/fast_automaton/operation/concat.rs new file mode 100644 index 0000000..804ad84 --- /dev/null +++ b/src/fast_automaton/operation/concat.rs @@ -0,0 +1,701 @@ +use std::hash::BuildHasherDefault; + +use condition::converter::ConditionConverter; + +use crate::error::EngineError; + +use super::*; + +impl FastAutomaton { + /// Computes the concatenation between `self` and `other`. + pub fn concat(&self, other: &FastAutomaton) -> Result { + let mut new_automaton = self.clone(); + new_automaton.concat_mut(other)?; + Ok(new_automaton) + } + + /// Computes the concatenation of all automata in the given iterator. + #[tracing::instrument(level = "debug", skip_all)] + pub fn concat_all<'a, I: IntoIterator>( + automata: I, + ) -> Result { + // Each operand's degenerate checks run once, on the operand: running + // them per fold step, on the growing result, made long concatenations + // quadratic. + let mut new_automaton = FastAutomaton::new_empty_string(); + let mut seeded = false; + for automaton in automata { + if automaton.is_empty() { + // ∅ annihilates the whole concatenation. + return Ok(FastAutomaton::new_empty()); + } + if automaton.is_empty_string() { + // {""} is the identity. + continue; + } + if seeded { + new_automaton.concat_mut_nondegenerate(automaton, false)?; + } else { + new_automaton.apply_model(automaton); + seeded = true; + } + } + + Ok(new_automaton) + } + + pub(crate) fn concat_mut(&mut self, other: &FastAutomaton) -> Result<(), EngineError> { + self.concat_mut_with(other, false) + } + + /// Concatenation where `force_no_merge` prevents merging `other`'s start + /// state into `self`'s accept states, always introducing a fresh start + /// state for `other` reached by epsilon transitions. Used by `repeat` to + /// keep accept states "clean" when they must remain accepting (so they do + /// not inherit the next copy's transitions). + /// + /// On error, `self` may be left partially mutated. The fallible steps can + /// only fail on a pre-existing condition/spanning-set desync (a broken + /// internal invariant), so a corrupted input is the only way to observe + /// a half-concatenated result. + pub(crate) fn concat_mut_with( + &mut self, + other: &FastAutomaton, + force_no_merge: bool, + ) -> Result<(), EngineError> { + ExecutionProfile::get().assert_not_timed_out()?; + + if other.is_empty() { + self.make_empty(); + return Ok(()); + } else if other.is_empty_string() { + return Ok(()); + } + + if self.is_empty() { + self.make_empty(); + return Ok(()); + } else if self.is_empty_string() { + self.apply_model(other); + return Ok(()); + } + + self.concat_mut_nondegenerate(other, force_no_merge) + } + + /// The concatenation core: both operands must be neither the empty + /// language `∅` nor the empty-string language `{""}`. Callers looping over + /// a growing accumulator (`repeat_mut`, [`concat_all`](Self::concat_all)) + /// establish that invariant once and call this directly: the degenerate + /// checks of [`concat_mut_with`](Self::concat_mut_with) walk the whole + /// automaton, and re-running them on every iteration made those loops + /// quadratic. + pub(crate) fn concat_mut_nondegenerate( + &mut self, + other: &FastAutomaton, + force_no_merge: bool, + ) -> Result<(), EngineError> { + self.assert_nondegenerate_operation_fits(other, || { + self.concat_state_count_nondegenerate(other, force_no_merge) + })?; + + // Equal spanning sets (the dominant case: `repeat_mut` concatenates + // the same operand over and over) skip the quadratic merge and treat + // every condition conversion as identity. + let new_spanning_set; + let condition_converter = if self.spanning_set == other.spanning_set { + None + } else { + new_spanning_set = self.spanning_set.merge(&other.spanning_set); + self.apply_new_spanning_set(&new_spanning_set)?; + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut new_states: IntMap = IntMap::with_capacity_and_hasher( + other.number_of_states(), + BuildHasherDefault::default(), + ); + + let start_state_and_accept_states_not_mergeable = force_no_merge + || (other.in_degree(other.start_state) > 0 + && self + .accept_states + .iter() + .cloned() + .any(|s| self.out_degree(s) > 0)); + + let accept_states = self.accept_states.iter().cloned().collect::>(); + + self.accept_states.clear(); + + if other.accept_states.contains(&other.start_state) { + for &accept_state in accept_states.iter() { + self.accept(accept_state); + } + } + + if start_state_and_accept_states_not_mergeable { + let new_start_state = new_states + .entry(other.start_state) + .or_insert(self.new_state()); + if other.accept_states.contains(&other.start_state) { + self.accept(*new_start_state); + } + } + + // `other`'s start maps to all of `self`'s accept states (when merging); + // every other state maps to exactly one. Borrowing a slice for both + // shapes keeps the per-transition loop allocation-free. + for from_state in other.states() { + let single_from; + let new_from_states: &[usize] = match new_states.entry(from_state) { + Entry::Occupied(o) => { + single_from = [*o.get()]; + &single_from + } + Entry::Vacant(v) => { + if from_state == other.start_state { + &accept_states + } else { + let new_state = self.new_state(); + if other.accept_states.contains(&from_state) { + self.accept(new_state); + } + v.insert(new_state); + single_from = [new_state]; + &single_from + } + } + }; + + for (condition, to_state) in other.transitions_from(from_state) { + let single_to; + let new_to_states: &[usize] = match new_states.entry(*to_state) { + Entry::Occupied(o) => { + single_to = [*o.get()]; + &single_to + } + Entry::Vacant(v) => { + if *to_state == other.start_state { + &accept_states + } else { + let new_state = self.new_state(); + if other.accept_states.contains(to_state) { + self.accept(new_state); + } + v.insert(new_state); + single_to = [new_state]; + &single_to + } + } + }; + let projected_condition = + convert_condition(condition_converter.as_ref(), condition)?; + for new_from_state in new_from_states.iter() { + for new_to_state in new_to_states.iter() { + self.add_transition(*new_from_state, *new_to_state, &projected_condition); + } + } + } + } + + if start_state_and_accept_states_not_mergeable + && let Some(&other_start_state) = new_states.get(&other.start_state) + { + for accept_state in &accept_states { + self.add_epsilon_transition(*accept_state, other_start_state); + } + } + + self.minimal = false; + Ok(()) + } + + /// `force_no_merge` must match the flag later passed to + /// [`concat_mut_with`](Self::concat_mut_with): a forced concatenation + /// keeps `other`'s start as a fresh state, costing one more state than + /// the merging estimate. + pub(crate) fn concat_state_count_heuristic( + &self, + other: &FastAutomaton, + force_no_merge: bool, + ) -> usize { + if other.is_empty() { + return 1; + } else if other.is_empty_string() { + return self.number_of_states(); + } + + if self.is_empty() { + return 1; + } else if self.is_empty_string() { + return other.number_of_states(); + } + + self.concat_state_count_nondegenerate(other, force_no_merge) + } + + /// [`concat_state_count_heuristic`](Self::concat_state_count_heuristic) + /// for operands already known to be non-degenerate: no emptiness walks. + fn concat_state_count_nondegenerate( + &self, + other: &FastAutomaton, + force_no_merge: bool, + ) -> usize { + // Determine if we are forced to create a new state to avoid unintended loops + let start_state_and_accept_states_not_mergeable = force_no_merge + || (other.in_degree(other.start_state) > 0 + && self + .accept_states + .iter() + .cloned() + .any(|s| self.out_degree(s) > 0)); + + let v1 = self.number_of_states(); + let v2 = other.number_of_states(); + + // Apply the heuristic + if start_state_and_accept_states_not_mergeable { + v1 + v2 + } else { + v1 + v2 - 1 + } + } +} + +#[cfg(test)] +mod tests { + use crate::{fast_automaton::FastAutomaton, regex::RegularExpression}; + + #[test] + fn bug_concat_empty_left() { + let e = FastAutomaton::new_empty(); + let t = FastAutomaton::new_total(); + let r = e.concat(&t).unwrap(); + assert!(r.is_empty(), "∅ · Σ* must be ∅, got something non-empty"); + } + + #[test] + fn bug_concat_empty_right() { + let e = FastAutomaton::new_empty(); + let t = FastAutomaton::new_total(); + let r = t.concat(&e).unwrap(); + assert!(r.is_empty(), "Σ* · ∅ must be ∅, got something non-empty"); + } + + #[test] + fn bug_term_concat_with_empty() { + use crate::Term; + let a = Term::from_automaton( + RegularExpression::parse("abc", false) + .unwrap() + .to_automaton() + .unwrap(), + ); + let e = Term::from_automaton(FastAutomaton::new_empty()); + let r = a.concat(&[e]).unwrap(); + assert!(r.is_empty().unwrap(), "'abc' · ∅ must be ∅"); + } + + #[test] + fn test_simple_concatenation_regex() -> Result<(), String> { + let automaton = RegularExpression::parse("abc", false) + .unwrap() + .to_automaton() + .unwrap(); + + automaton.print_dot(); + assert!(automaton.is_match("abc")); + assert!(!automaton.is_match("abcd")); + assert!(!automaton.is_match("ab")); + assert!(!automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_simple_concat_alternation_regex() -> Result<(), String> { + let automaton = RegularExpression::parse("0101(abc|ac|aaa)", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("0101abc")); + assert!(automaton.is_match("0101ac")); + assert!(automaton.is_match("0101aaa")); + assert!(!automaton.is_match("abc")); + assert!(!automaton.is_match("0101abcd")); + assert!(!automaton.is_match("ab")); + assert!(!automaton.is_match("acc")); + assert!(!automaton.is_match("a")); + assert!(!automaton.is_match("aaaa")); + assert!(!automaton.is_match("aa")); + assert!(!automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_simple_concat_repeat_regex() -> Result<(), String> { + let automaton = RegularExpression::parse("A+B*", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("AAABBB")); + assert!(automaton.is_match("AA")); + assert!(automaton.is_match("AB")); + assert!(!automaton.is_match("B")); + assert!(!automaton.is_match("ABA")); + assert!(!automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_01() -> Result<(), String> { + let automaton = RegularExpression::parse("a+", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aa")); + assert!(automaton.is_match("aaaaaaa")); + assert!(!automaton.is_match("ab")); + assert!(!automaton.is_match("")); + + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_02() -> Result<(), String> { + let automaton = RegularExpression::parse("a*c", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("c")); + assert!(automaton.is_match("ac")); + assert!(automaton.is_match("aac")); + assert!(automaton.is_match("aaaaaaac")); + assert!(!automaton.is_match("abc")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_03() -> Result<(), String> { + let automaton = RegularExpression::parse("(ab){3,4}", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("ababab")); + assert!(automaton.is_match("abababab")); + assert!(!automaton.is_match("ab")); + assert!(!automaton.is_match("abab")); + assert!(!automaton.is_match("ababababab")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_04() -> Result<(), String> { + let automaton = RegularExpression::parse("a{3,}", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("aaa")); + assert!(automaton.is_match("aaaaa")); + assert!(!automaton.is_match("a")); + assert!(!automaton.is_match("aa")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_05() -> Result<(), String> { + let automaton = RegularExpression::parse("a?", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(!automaton.is_match("aa")); + assert!(!automaton.is_match("aaa")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_06() -> Result<(), String> { + let automaton = RegularExpression::parse("a{0,2}", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aa")); + assert!(!automaton.is_match("aaa")); + assert!(!automaton.is_match("aaaa")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_07() -> Result<(), String> { + let automaton = RegularExpression::parse("a{1,3}", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(!automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aa")); + assert!(automaton.is_match("aaa")); + assert!(!automaton.is_match("aaaa")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_08() -> Result<(), String> { + let automaton = RegularExpression::parse("a+(ba+)*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(!automaton.is_match("")); + assert!(!automaton.is_match("aab")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aaa")); + assert!(automaton.is_match("aba")); + assert!(automaton.is_match("aaba")); + assert!(automaton.is_match("aabaaa")); + assert!(automaton.is_match("aaabaaabaaba")); + assert!(!automaton.is_match("aaabbaa")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_09() -> Result<(), String> { + let automaton = RegularExpression::parse("(ac|ads|a)*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("ac")); + assert!(automaton.is_match("ads")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("acaadsac")); + assert!(automaton.is_match("adsaaaaaaaacaa")); + assert!(!automaton.is_match("as")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("c")); + assert!(!automaton.is_match("ds")); + assert!(!automaton.is_match("d")); + assert!(!automaton.is_match("s")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_10() -> Result<(), String> { + let automaton = RegularExpression::parse("(ef|ads|a)+", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(!automaton.is_match("")); + assert!(automaton.is_match("ef")); + assert!(automaton.is_match("ads")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("efadsa")); + assert!(automaton.is_match("aaadsefef")); + assert!(!automaton.is_match("as")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("e")); + assert!(!automaton.is_match("ds")); + assert!(!automaton.is_match("d")); + assert!(!automaton.is_match("s")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_11() -> Result<(), String> { + let automaton = RegularExpression::parse("(a|bc)*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("bc")); + assert!(automaton.is_match("abcbca")); + assert!(automaton.is_match("bcabcbcaaaa")); + assert!(!automaton.is_match("b")); + assert!(!automaton.is_match("c")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_12() -> Result<(), String> { + let automaton = RegularExpression::parse("([ab]*a)?", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aa")); + assert!(automaton.is_match("ba")); + assert!(automaton.is_match("aba")); + assert!(automaton.is_match("abbaabbaba")); + assert!(!automaton.is_match("b")); + assert!(!automaton.is_match("abab")); + Ok(()) + } + + #[test] + fn test_simple_repeat_regex_13() -> Result<(), String> { + let automaton = RegularExpression::parse("([ab]*a)*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aa")); + assert!(automaton.is_match("ba")); + assert!(automaton.is_match("aba")); + assert!(automaton.is_match("abbaabbaba")); + assert!(!automaton.is_match("b")); + assert!(!automaton.is_match("abab")); + Ok(()) + } + + #[test] + fn test_simple_repeat_right_number_of_states_1() -> Result<(), String> { + let automaton = RegularExpression::parse("a*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert_eq!(1, automaton.number_of_states()); + Ok(()) + } + + #[test] + fn test_simple_concat_right_number_of_states_2() -> Result<(), String> { + let automaton = RegularExpression::parse("(a*bc)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert_eq!(3, automaton.number_of_states()); + Ok(()) + } + + #[test] + fn test_heuristic() -> Result<(), String> { + assert_heuristic(".{900}", "[a-z]+"); + + assert_heuristic("[a-z]+@", "[0-9]+[A-Z]*"); + + assert_heuristic("a+(ba+)*", "((a|bc)*|d)"); + + assert_heuristic(".*", "(ac|ads|a)*"); + + assert_heuristic( + "((aad|ads|a)*|q)", + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + assert_heuristic( + "(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@", + "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", + ); + + assert_heuristic("((aad|ads|a)*abc.*uif(aad|ads|x)*|q)", ".*"); + + assert_heuristic( + ".{900}", + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + Ok(()) + } + + fn assert_heuristic(regex1: &str, regex2: &str) { + println!( + "Testing concat heuristic for: '{}' and '{}'", + regex1, regex2 + ); + + let automaton1 = RegularExpression::parse(regex1, false) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton2 = RegularExpression::parse(regex2, false) + .unwrap() + .to_automaton() + .unwrap(); + + // Helper closure to run the test and assert + let test_pair = |a1: &FastAutomaton, a2: &FastAutomaton, desc: &str| { + let mut actual_concat = a1.clone(); + + // Execute the actual mutation + actual_concat.concat_mut(a2).unwrap(); + + let actual_states = actual_concat.number_of_states(); + let heuristic_states = a1.concat_state_count_heuristic(a2, false); + + // The forced variant must be exact too. + let mut forced_concat = a1.clone(); + forced_concat.concat_mut_with(a2, true).unwrap(); + assert_eq!( + forced_concat.number_of_states(), + a1.concat_state_count_heuristic(a2, true), + "force_no_merge heuristic mismatch for {desc}", + ); + + assert_eq!( + actual_states, heuristic_states, + "Mismatch for {}.\nExpected (heuristic): {}\nActual (computed): {}", + desc, heuristic_states, actual_states + ); + }; + + // Test 1: regex1 + regex2 + test_pair( + &automaton1, + &automaton2, + &format!("'{}' + '{}'", regex1, regex2), + ); + + // Test 2: regex2 + regex1 (Reverse order) + test_pair( + &automaton2, + &automaton1, + &format!("'{}' + '{}'", regex2, regex1), + ); + + // Test 3: regex1 + regex1 (Self-concatenation, crucial for your repeat logic) + test_pair( + &automaton1, + &automaton1, + &format!("'{}' + '{}' (Self)", regex1, regex1), + ); + + // Test 4 & 5: Empty automaton edge cases + let empty_automaton = FastAutomaton::new_empty(); + + test_pair( + &empty_automaton, + &automaton2, + &format!("Empty + '{}'", regex2), + ); + test_pair( + &automaton1, + &empty_automaton, + &format!("'{}' + Empty", regex1), + ); + } +} +//(a|bc)* diff --git a/src/fast_automaton/operation/concatenate.rs b/src/fast_automaton/operation/concatenate.rs deleted file mode 100644 index 3741e01..0000000 --- a/src/fast_automaton/operation/concatenate.rs +++ /dev/null @@ -1,520 +0,0 @@ -use std::hash::BuildHasherDefault; - -use condition::converter::ConditionConverter; - -use crate::error::EngineError; - -use super::*; - -impl FastAutomaton { - pub fn concatenate(automatons: Vec) -> Result { - if automatons.len() == 1 { - return Ok(automatons[0].clone()); - } - let mut new_automaton = FastAutomaton::new_empty_string(); - if automatons.is_empty() { - return Ok(new_automaton); - } - for automaton in automatons { - new_automaton.concat(&automaton)?; - } - - Ok(new_automaton) - } - - pub fn repeat(&mut self, min: u32, max_opt: Option) -> Result<(), EngineError> { - if let Some(max) = max_opt { - if min > max { - self.make_empty(); - return Ok(()); - } - } - - let automaton_to_repeat = self.clone(); - - if min == 0 && self.in_degree(self.start_state) != 0 { - let new_state = self.new_state(); - if self.is_accepted(&self.start_state) { - self.accept(new_state); - } - - for to_state in self.transitions_from_state(&self.start_state) { - self.add_epsilon(new_state, to_state); - } - self.start_state = new_state; - - if max_opt.is_none() { - for accept_state in self.accept_states.clone() { - self.add_epsilon(accept_state, self.start_state); - } - self.accept(self.start_state); - return Ok(()); - } - } - - if let Some(max) = max_opt { - if min <= 1 && max == 1 { - if min == 0 { - self.accept_states.insert(self.start_state); - } - return Ok(()); - } - } - - let iter = if min == 0 { 0..0 } else { 0..min - 1 }; - for _ in iter { - self.concat(&automaton_to_repeat)?; - } - - if max_opt.is_none() { - let mut automaton_to_repeat = automaton_to_repeat.clone(); - - let accept_state = *automaton_to_repeat.accept_states.iter().next().unwrap(); - if automaton_to_repeat.accept_states.len() == 1 - && automaton_to_repeat.out_degree(accept_state) == 0 - && automaton_to_repeat.in_degree(automaton_to_repeat.start_state) == 0 - { - automaton_to_repeat.add_epsilon(accept_state, automaton_to_repeat.start_state); - let old_start_state = automaton_to_repeat.start_state; - automaton_to_repeat.start_state = accept_state; - automaton_to_repeat.remove_state(old_start_state); - } else { - let t = Self::transitions_from_state_set( - &automaton_to_repeat.transitions, - automaton_to_repeat.start_state, - ); - let transitions = - Self::transitions_from_state_enumerate(&t, &automaton_to_repeat.removed_states); - - for state in automaton_to_repeat.accept_states.clone() { - for &(to_state, condition) in &transitions { - automaton_to_repeat.add_transition_to(state, *to_state, condition); - } - } - - automaton_to_repeat.accept(automaton_to_repeat.get_start_state()); - } - automaton_to_repeat.cyclic = true; - - if min == 0 { - self.apply_model(&automaton_to_repeat); - } else { - self.concat(&automaton_to_repeat)?; - } - - return Ok(()); - } - - let mut end_states = self.accept_states.iter().cloned().collect::>(); - for _ in cmp::max(min, 1)..max_opt.unwrap() { - self.concat(&automaton_to_repeat)?; - end_states.extend(self.accept_states.iter()); - } - self.accept_states.extend(end_states); - if min == 0 { - self.accept(self.start_state); - } - Ok(()) - } - - fn concat(&mut self, other: &FastAutomaton) -> Result<(), EngineError> { - if other.is_empty() { - return Ok(()); - } - if self.is_empty() { - self.apply_model(other); - return Ok(()); - } - - let new_spanning_set = &self.spanning_set.merge(&other.spanning_set); - self.apply_new_spanning_set(new_spanning_set)?; - let condition_converter = ConditionConverter::new(&other.spanning_set, new_spanning_set)?; - - let mut new_states: IntMap = IntMap::with_capacity_and_hasher( - other.get_number_of_states(), - BuildHasherDefault::default(), - ); - - let start_state_and_accept_states_not_mergeable = other.in_degree(other.start_state) > 0 - && self - .accept_states - .iter() - .cloned() - .any(|s| self.out_degree(s) > 0); - - let accept_states = self.accept_states.iter().cloned().collect::>(); - - self.accept_states.clear(); - - if other.accept_states.contains(&other.start_state) { - for &accept_state in accept_states.iter() { - self.accept(accept_state); - } - } - - if start_state_and_accept_states_not_mergeable { - let new_start_state = new_states - .entry(other.start_state) - .or_insert(self.new_state()); - if other.accept_states.contains(&other.start_state) { - self.accept(*new_start_state); - } - } - - for from_state in other.transitions_iter() { - let new_from_states = match new_states.entry(from_state) { - Entry::Occupied(o) => { - vec![*o.get()] - } - Entry::Vacant(v) => { - if from_state == other.start_state { - accept_states.clone() - } else { - let new_state = self.new_state(); - if other.accept_states.contains(&from_state) { - self.accept(new_state); - } - v.insert(new_state); - vec![new_state] - } - } - }; - - for (to_state, condition) in other.transitions_from_state_enumerate_iter(&from_state) { - let new_to_states = match new_states.entry(*to_state) { - Entry::Occupied(o) => { - vec![*o.get()] - } - Entry::Vacant(v) => { - if *to_state == other.start_state { - accept_states.clone() - } else { - let new_state = self.new_state(); - if other.accept_states.contains(to_state) { - self.accept(new_state); - } - v.insert(new_state); - vec![new_state] - } - } - }; - let projected_condition = condition_converter.convert(condition)?; - for new_from_state in new_from_states.iter() { - for new_to_state in new_to_states.iter() { - self.add_transition_to( - *new_from_state, - *new_to_state, - &projected_condition, - ); - } - } - } - } - - if start_state_and_accept_states_not_mergeable { - if let Some(&other_start_state) = new_states.get(&other.start_state) { - for accept_state in &accept_states { - self.add_epsilon(*accept_state, other_start_state); - } - } - } - self.cyclic = self.cyclic || other.cyclic; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::regex::RegularExpression; - - #[test] - fn test_simple_concatenation_regex() -> Result<(), String> { - let automaton = RegularExpression::new("abc") - .unwrap() - .to_automaton() - .unwrap(); - - automaton.to_dot(); - assert!(automaton.match_string("abc")); - assert!(!automaton.match_string("abcd")); - assert!(!automaton.match_string("ab")); - assert!(!automaton.match_string("")); - Ok(()) - } - - #[test] - fn test_simple_concat_alternation_regex() -> Result<(), String> { - let automaton = RegularExpression::new("0101(abc|ac|aaa)") - .unwrap() - .to_automaton() - .unwrap(); - assert!(automaton.match_string("0101abc")); - assert!(automaton.match_string("0101ac")); - assert!(automaton.match_string("0101aaa")); - assert!(!automaton.match_string("abc")); - assert!(!automaton.match_string("0101abcd")); - assert!(!automaton.match_string("ab")); - assert!(!automaton.match_string("acc")); - assert!(!automaton.match_string("a")); - assert!(!automaton.match_string("aaaa")); - assert!(!automaton.match_string("aa")); - assert!(!automaton.match_string("")); - Ok(()) - } - - #[test] - fn test_simple_concat_repeat_regex() -> Result<(), String> { - let automaton = RegularExpression::new("A+B*") - .unwrap() - .to_automaton() - .unwrap(); - assert!(automaton.match_string("AAABBB")); - assert!(automaton.match_string("AA")); - assert!(automaton.match_string("AB")); - assert!(!automaton.match_string("B")); - assert!(!automaton.match_string("ABA")); - assert!(!automaton.match_string("")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_01() -> Result<(), String> { - let automaton = RegularExpression::new("a+") - .unwrap() - .to_automaton() - .unwrap(); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aa")); - assert!(automaton.match_string("aaaaaaa")); - assert!(!automaton.match_string("ab")); - assert!(!automaton.match_string("")); - - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_02() -> Result<(), String> { - let automaton = RegularExpression::new("a*c") - .unwrap() - .to_automaton() - .unwrap(); - assert!(automaton.match_string("c")); - assert!(automaton.match_string("ac")); - assert!(automaton.match_string("aac")); - assert!(automaton.match_string("aaaaaaac")); - assert!(!automaton.match_string("abc")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_03() -> Result<(), String> { - let automaton = RegularExpression::new("(ab){3,4}") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("ababab")); - assert!(automaton.match_string("abababab")); - assert!(!automaton.match_string("ab")); - assert!(!automaton.match_string("abab")); - assert!(!automaton.match_string("ababababab")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_04() -> Result<(), String> { - let automaton = RegularExpression::new("a{3,}") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("aaa")); - assert!(automaton.match_string("aaaaa")); - assert!(!automaton.match_string("a")); - assert!(!automaton.match_string("aa")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_05() -> Result<(), String> { - let automaton = RegularExpression::new("a?") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(!automaton.match_string("aa")); - assert!(!automaton.match_string("aaa")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_06() -> Result<(), String> { - let automaton = RegularExpression::new("a{0,2}") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aa")); - assert!(!automaton.match_string("aaa")); - assert!(!automaton.match_string("aaaa")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_07() -> Result<(), String> { - let automaton = RegularExpression::new("a{1,3}") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(!automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aa")); - assert!(automaton.match_string("aaa")); - assert!(!automaton.match_string("aaaa")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_08() -> Result<(), String> { - let automaton = RegularExpression::new("a+(ba+)*") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(!automaton.match_string("")); - assert!(!automaton.match_string("aab")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aaa")); - assert!(automaton.match_string("aba")); - assert!(automaton.match_string("aaba")); - assert!(automaton.match_string("aabaaa")); - assert!(automaton.match_string("aaabaaabaaba")); - assert!(!automaton.match_string("aaabbaa")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_09() -> Result<(), String> { - let automaton = RegularExpression::new("(ac|ads|a)*") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("ac")); - assert!(automaton.match_string("ads")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("acaadsac")); - assert!(automaton.match_string("adsaaaaaaaacaa")); - assert!(!automaton.match_string("as")); - assert!(!automaton.match_string("ad")); - assert!(!automaton.match_string("c")); - assert!(!automaton.match_string("ds")); - assert!(!automaton.match_string("d")); - assert!(!automaton.match_string("s")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_10() -> Result<(), String> { - let automaton = RegularExpression::new("(ef|ads|a)+") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(!automaton.match_string("")); - assert!(automaton.match_string("ef")); - assert!(automaton.match_string("ads")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("efadsa")); - assert!(automaton.match_string("aaadsefef")); - assert!(!automaton.match_string("as")); - assert!(!automaton.match_string("ad")); - assert!(!automaton.match_string("e")); - assert!(!automaton.match_string("ds")); - assert!(!automaton.match_string("d")); - assert!(!automaton.match_string("s")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_11() -> Result<(), String> { - let automaton = RegularExpression::new("(a|bc)*") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("bc")); - assert!(automaton.match_string("abcbca")); - assert!(automaton.match_string("bcabcbcaaaa")); - assert!(!automaton.match_string("b")); - assert!(!automaton.match_string("c")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_12() -> Result<(), String> { - let automaton = RegularExpression::new("([ab]*a)?") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aa")); - assert!(automaton.match_string("ba")); - assert!(automaton.match_string("aba")); - assert!(automaton.match_string("abbaabbaba")); - assert!(!automaton.match_string("b")); - assert!(!automaton.match_string("abab")); - Ok(()) - } - - #[test] - fn test_simple_repeat_regex_13() -> Result<(), String> { - let automaton = RegularExpression::new("([ab]*a)*") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert!(automaton.match_string("")); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("aa")); - assert!(automaton.match_string("ba")); - assert!(automaton.match_string("aba")); - assert!(automaton.match_string("abbaabbaba")); - assert!(!automaton.match_string("b")); - assert!(!automaton.match_string("abab")); - Ok(()) - } - - #[test] - fn test_simple_repeat_right_number_of_states_1() -> Result<(), String> { - let automaton = RegularExpression::new("a*") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert_eq!(1, automaton.get_number_of_states()); - Ok(()) - } - - #[test] - fn test_simple_concat_right_number_of_states_2() -> Result<(), String> { - let automaton = RegularExpression::new("(a*bc)") - .unwrap() - .to_automaton() - .unwrap(); - automaton.to_dot(); - assert_eq!(3, automaton.get_number_of_states()); - Ok(()) - } -} -//(a|bc)* diff --git a/src/fast_automaton/operation/determinize.rs b/src/fast_automaton/operation/determinize.rs index 3d4057b..82786fa 100644 --- a/src/fast_automaton/operation/determinize.rs +++ b/src/fast_automaton/operation/determinize.rs @@ -1,98 +1,141 @@ -use ahash::HashMapExt; +use bit_set::BitSet; -use crate::{execution_profile::ThreadLocalParams, EngineError}; +use crate::{EngineError, execution_profile::ExecutionProfile}; use super::*; impl FastAutomaton { - pub fn determinize(&self) -> Result { + /// [`determinize`](Self::determinize) on behalf of an operation that + /// requires a deterministic automaton: when the execution profile + /// disables implicit determinization, a non-deterministic input is + /// rejected with [`EngineError::DeterministicAutomatonRequired`] instead + /// of being converted. Already-deterministic automata always pass. + pub(crate) fn determinize_implicit(&self) -> Result, EngineError> { + if !self.deterministic { + ExecutionProfile::get().assert_implicit_determinization_allowed()?; + } + self.determinize() + } + + /// Determinizes the automaton and returns the result. + #[tracing::instrument(level = "debug", skip_all, fields(states = self.number_of_states(), deterministic = self.is_deterministic()))] + pub fn determinize(&self) -> Result, EngineError> { if self.deterministic { - return Ok(self.clone()); + return Ok(Cow::Borrowed(self)); } - let execution_profile = ThreadLocalParams::get_execution_profile(); + let execution_profile = ExecutionProfile::get(); - let ranges = self.get_ranges()?; + let bases = self.spanning_bases()?; - let initial_vec = VecDeque::from(vec![self.start_state]); + let mut worklist = VecDeque::with_capacity(self.number_of_states()); - let mut worklist = VecDeque::with_capacity(self.get_number_of_states()); + let map_capacity = (self.number_of_states() as f64 / 0.75).ceil() as usize; + let mut new_states = AHashMap::with_capacity(map_capacity); - let map_capacity = (self.get_number_of_states() as f64 / 0.75).ceil() as usize; - let mut new_states = IntMap::with_capacity(map_capacity); + let mut accept_states = BitSet::new(); + for &state in &self.accept_states { + accept_states.insert(state); + } let mut new_automaton = FastAutomaton::new_empty(); new_automaton.spanning_set = self.spanning_set.clone(); - worklist.push_back((vec![self.start_state], new_automaton.start_state)); - new_states.insert(Self::simple_hash(&initial_vec), new_automaton.start_state); + let mut initial_state = BitSet::new(); + initial_state.insert(self.start_state); + + worklist.push_back((initial_state.clone(), new_automaton.start_state)); + new_states.insert(initial_state, new_automaton.start_state); - let mut new_states_to_add = VecDeque::with_capacity(self.get_number_of_states()); + // Per-base successor subsets, reused across popped subsets. Base `b` + // of the spanning set is exactly bit `b` of a condition, so one sweep + // over the subset's transitions distributes each target into the + // bases its condition covers. + let mut base_targets: Vec = vec![BitSet::new(); bases.len()]; while let Some((states, r)) = worklist.pop_front() { execution_profile.assert_not_timed_out()?; + execution_profile.assert_max_number_of_states(new_states.len())?; - for state in &states { - if self.accept_states.contains(state) { - new_automaton.accept_states.insert(r); - break; - } + if !states.is_disjoint(&accept_states) { + new_automaton.accept(r); } - for base in &ranges { - for from_state in &states { - for (to_state, cond) in self.transitions_from_state_enumerate_iter(from_state) { - if cond.has_intersection(base) { - match new_states_to_add.binary_search(to_state) { - Ok(_) => {} // element already in vector @ `pos` - Err(pos) => new_states_to_add.insert(pos, *to_state), - }; - } + for from_state in &states { + for (cond, to_state) in self.transitions_from(from_state) { + for base_index in cond.iter_set_bits() { + base_targets[base_index].insert(*to_state); } } - if !new_states_to_add.is_empty() { - let q = match new_states.entry(Self::simple_hash(&new_states_to_add)) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - let new_q = new_automaton.new_state(); - worklist - .push_back((new_states_to_add.iter().cloned().collect(), new_q)); - v.insert(new_q); - new_q - } - }; - - new_automaton.add_transition_to(r, q, base); + } + + // Base index order keeps the resulting state numbering + // deterministic. + for (targets, base) in base_targets.iter_mut().zip(&bases) { + if targets.is_empty() { + continue; + } + // Once the construction converges, the subset usually + // already exists: look it up first so the hit path pays + // no `BitSet` clone (the entry API would need an owned + // key), and only clone-free-insert on a miss. + if let Some(&q) = new_states.get(targets) { + targets.clear(); + + new_automaton.add_transition(r, q, base); + } else { + let new_q = new_automaton.new_state(); + let subset = std::mem::take(targets); + new_states.insert(subset.clone(), new_q); + worklist.push_back((subset, new_q)); + + new_automaton.add_transition(r, new_q, base); } - new_states_to_add.clear(); } } - Ok(new_automaton) - } - fn simple_hash(list: &VecDeque) -> u64 { - let mut hasher = AHasher::default(); - for &item in list { - hasher.write_usize(item); - } - hasher.finish() + Ok(Cow::Owned(new_automaton)) } } #[cfg(test)] mod tests { + use crate::CharRange; + use crate::fast_automaton::FastAutomaton; + use crate::fast_automaton::condition::Condition; + use crate::fast_automaton::spanning_set::SpanningSet; use crate::regex::RegularExpression; + use regex_charclass::char::Char; + // Subset construction iterates `spanning_bases`, which must include the + // spanning set's "rest" range: otherwise a transition whose condition + // lies in the rest range would be dropped, giving a DFA with a smaller + // language than the input NFA. #[test] - fn test_determinize_1() -> Result<(), String> { - let automaton = RegularExpression::new(".*ab") - .unwrap() - .to_automaton() - .unwrap(); - - let deterministic_automaton = automaton.determinize().unwrap(); - - assert!(deterministic_automaton.is_determinitic()); - - Ok(()) + fn determinize_keeps_rest_range_transitions() { + let rng = |c: char| { + let c = Char::new(c); + CharRange::new_from_range(c..=c) + }; + let ss = SpanningSet::compute_spanning_set(&[rng('a'), rng('b')]); + let rest = ss.rest().clone(); + + let mut a = FastAutomaton::new_empty(); + a.apply_new_spanning_set(&ss).unwrap(); + a.new_state(); + a.add_transition(0, 1, &Condition::from_range(&rest, &ss).unwrap()); // 0 -[^ab]-> 1 + a.add_transition(1, 0, &Condition::from_range(&rng('a'), &ss).unwrap()); + a.add_transition(1, 1, &Condition::from_range(&rng('a'), &ss).unwrap()); // nondeterministic + a.accept(1); + + assert!(!a.is_deterministic()); + assert!(a.is_match("\u{0}"), "a should accept a [^ab] character"); + + let d = a.determinize().unwrap(); + assert!(d.is_deterministic()); + assert!( + d.is_match("\u{0}"), + "determinize dropped the [^ab] transition" + ); + assert!(a.equivalent(&d).unwrap()); } #[test] @@ -112,22 +155,22 @@ mod tests { fn assert_determinization(regex: &str) { println!(":{}", regex); - let automaton = RegularExpression::new(regex) + let automaton = RegularExpression::parse(regex, false) .unwrap() .to_automaton() .unwrap(); - //automaton.compute_determinization_cost(); - //println!("Determinization Cost: {:?}", automaton.determinisation_cost); - println!("States Before: {}", automaton.get_number_of_states()); + println!("States Before: {}", automaton.number_of_states()); let deterministic_automaton = automaton.determinize().unwrap(); println!( "States After: {}", - deterministic_automaton.get_number_of_states() + deterministic_automaton.number_of_states() + ); + assert!(deterministic_automaton.is_deterministic()); + assert!( + automaton + .difference(&deterministic_automaton) + .unwrap() + .is_empty() ); - assert!(deterministic_automaton.is_determinitic()); - assert!(automaton - .subtraction(&deterministic_automaton) - .unwrap() - .is_empty()); } } diff --git a/src/fast_automaton/operation/difference.rs b/src/fast_automaton/operation/difference.rs new file mode 100644 index 0000000..b5410c6 --- /dev/null +++ b/src/fast_automaton/operation/difference.rs @@ -0,0 +1,116 @@ +use std::hash::BuildHasherDefault; + +use crate::EngineError; + +use super::*; + +impl FastAutomaton { + /// Totalize the automaton. Precondition: `self.deterministic` is true + /// (the only caller, `complement`, determinizes first). + fn totalize(&mut self) -> Result<(), EngineError> { + debug_assert!(self.deterministic, "totalize requires a DFA"); + + let crash_state = self.new_state(); + let mut transitions_to_crash_state: IntMap = + IntMap::with_capacity_and_hasher( + self.number_of_states(), + BuildHasherDefault::default(), + ); + + let mut ranges = Vec::with_capacity(self.number_of_states()); + for from_state in self.states() { + let mut new_condition = Condition::empty(&self.spanning_set); + for (condition, _) in self.transitions_from(from_state) { + new_condition = new_condition.union(condition); + ranges.push(condition.to_range(self.spanning_set())?); + } + + new_condition = new_condition.complement(); + + transitions_to_crash_state.insert(from_state, new_condition); + } + + for (from_state, condition) in &transitions_to_crash_state { + self.add_transition(*from_state, crash_state, condition); + ranges.push(condition.to_range(self.spanning_set())?); + } + + let new_spanning_set = SpanningSet::compute_spanning_set(&ranges); + self.apply_new_spanning_set(&new_spanning_set)?; + + if self.in_degree(crash_state) == 1 { + // Only the self-loop points to crash; nothing else needs it. + self.remove_state(crash_state); + } + Ok(()) + } + + /// Complements the automaton. + /// + /// If `self` is non-deterministic, it is determinized in place first, + /// unless the execution profile disables implicit determinization, in + /// which case [`EngineError::DeterministicAutomatonRequired`] is + /// returned. + #[tracing::instrument(level = "debug", skip_all, fields(states = self.number_of_states(), deterministic = self.is_deterministic()))] + pub fn complement(&mut self) -> Result<(), EngineError> { + if !self.deterministic { + *self = self.determinize_implicit()?.into_owned(); + } + self.totalize()?; + + let mut new_accept_states = IntSet::default(); + for state in self.states() { + if self.accept_states.contains(&state) { + continue; + } + new_accept_states.insert(state); + } + + self.accept_states = new_accept_states; + Ok(()) + } + + /// Computes the difference between `self` and `other`. + /// + /// If `other` is non-deterministic, it is determinized first, unless + /// the execution profile disables implicit determinization, in which + /// case [`EngineError::DeterministicAutomatonRequired`] is returned. + #[tracing::instrument(level = "debug", skip_all, fields(self_states = self.number_of_states(), self_deterministic = self.is_deterministic(), other_states = other.number_of_states(), other_deterministic = other.is_deterministic()))] + pub fn difference(&self, other: &FastAutomaton) -> Result { + let mut complement = other.determinize_implicit()?.into_owned(); + complement.complement()?; + self.intersection(&complement) + } +} + +#[cfg(test)] +mod tests { + use crate::fast_automaton::FastAutomaton; + use crate::regex::RegularExpression; + + // `totalize` adds a `crash_state` with a total self-loop, so the complement + // of a finite language is infinite (it matches arbitrarily long strings via + // the crash-state loop). + #[test] + fn complement_of_finite_is_infinite() { + let mut a = RegularExpression::parse("abc", false) + .unwrap() + .to_automaton() + .unwrap(); + + a.complement().unwrap(); + + assert!(!a.is_match("abc"), "complement must not match 'abc'"); + assert!(a.is_match("x")); + assert!(a.is_match("xx")); + assert!(a.is_match("xxxxxxxxxx")); + } + + // empty.complement() = Σ*. + #[test] + fn complement_of_empty_is_total() { + let mut a = FastAutomaton::new_empty(); + a.complement().unwrap(); + assert!(a.is_total(), "complement of ∅ must be Σ*"); + } +} diff --git a/src/fast_automaton/operation/intersection.rs b/src/fast_automaton/operation/intersection.rs index 96007e6..f03c112 100644 --- a/src/fast_automaton/operation/intersection.rs +++ b/src/fast_automaton/operation/intersection.rs @@ -1,32 +1,113 @@ +use std::borrow::Cow; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + use condition::converter::ConditionConverter; -use crate::{error::EngineError, execution_profile::ThreadLocalParams}; +use crate::{error::EngineError, execution_profile::ExecutionProfile}; use super::*; impl FastAutomaton { - pub fn intersection(&self, other: &FastAutomaton) -> Result { + /// Computes the intersection between `self` and `other`. + pub fn intersection(&self, other: &FastAutomaton) -> Result { + FastAutomaton::intersection_all([self, other]) + } + + /// Computes the intersection of all automata in the given iterator. + #[tracing::instrument(level = "debug", skip_all)] + pub fn intersection_all<'a, I: IntoIterator>( + automata: I, + ) -> Result { + let mut result: Cow<'a, FastAutomaton> = Cow::Owned(FastAutomaton::new_total()); + + for automaton in automata { + result = result.intersection_internal(automaton)?; + + if result.is_empty() { + break; + } + } + + Ok(result.into_owned()) + } + + /// Computes in parallel the intersection of all automata in the given iterator. + /// + /// Only available with the `parallel` feature (enabled by default). + #[cfg(feature = "parallel")] + #[tracing::instrument(level = "debug", skip_all)] + pub fn intersection_all_par<'a, I: IntoParallelIterator>( + automata: I, + ) -> Result { + let execution_profile = ExecutionProfile::get(); + + let total = FastAutomaton::new_total(); + + automata + .into_par_iter() + .try_fold( + || total.clone(), + |acc, next| { + execution_profile.apply(|| Ok(acc.intersection_internal(next)?.into_owned())) + }, + ) + .try_reduce( + || total.clone(), + |acc, next| { + execution_profile.apply(|| Ok(acc.intersection_internal(&next)?.into_owned())) + }, + ) + } + + fn intersection_internal<'a>( + &self, + other: &'a FastAutomaton, + ) -> Result, EngineError> { if self.is_empty() || other.is_empty() { - return Ok(Self::new_empty()); + return Ok(Cow::Owned(Self::new_empty())); } else if self.is_total() { - return Ok(other.clone()); + return Ok(Cow::Borrowed(other)); } else if other.is_total() { - return Ok(self.clone()); + return Ok(Cow::Owned(self.clone())); } - let execution_profile = ThreadLocalParams::get_execution_profile(); - - let new_spanning_set = self.spanning_set.merge(&other.spanning_set); - - let condition_converter_self_to_new = - ConditionConverter::new(&self.spanning_set, &new_spanning_set)?; - let condition_converter_other_to_new = - ConditionConverter::new(&other.spanning_set, &new_spanning_set)?; + let execution_profile = ExecutionProfile::get(); + + // Equal spanning sets (the dominant case in operation chains) need no + // merge and no condition projection at all. + let same_spanning_set = self.spanning_set == other.spanning_set; + let new_spanning_set = if same_spanning_set { + self.spanning_set.clone() + } else { + self.spanning_set.merge(&other.spanning_set) + }; + + let condition_converter_self_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &self.spanning_set, + &new_spanning_set, + )?) + }; + let condition_converter_other_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut projected_self: IntMap> = IntMap::default(); + let mut projected_other: IntMap> = IntMap::default(); let mut new_automaton = FastAutomaton::new_empty(); let mut worklist = - VecDeque::with_capacity(self.get_number_of_states() + other.get_number_of_states()); + VecDeque::with_capacity(self.number_of_states() + other.number_of_states()); let mut new_states: AHashMap<(usize, usize), (usize, usize, usize), _> = - AHashMap::with_capacity(self.get_number_of_states() + other.get_number_of_states()); + AHashMap::with_capacity(self.number_of_states() + other.number_of_states()); let initial_pair = ( new_automaton.start_state, @@ -39,60 +120,89 @@ impl FastAutomaton { while let Some(p) = worklist.pop_front() { execution_profile.assert_not_timed_out()?; + execution_profile.assert_max_number_of_states(new_states.len())?; if self.accept_states.contains(&p.1) && other.accept_states.contains(&p.2) { new_automaton.accept(p.0); } - let transitions_1 = - self.get_projected_transitions(p.1, &condition_converter_self_to_new)?; - let transitions_2 = - other.get_projected_transitions(p.2, &condition_converter_other_to_new)?; - - for (n1, condition_1) in transitions_1 { - for (n2, condition_2) in &transitions_2 { + let transitions_1 = self.projected_transitions( + &mut projected_self, + p.1, + condition_converter_self_to_new.as_ref(), + )?; + let transitions_2 = other.projected_transitions( + &mut projected_other, + p.2, + condition_converter_other_to_new.as_ref(), + )?; + + for (condition_1, n1) in transitions_1 { + for (condition_2, n2) in transitions_2 { let intersection = condition_1.intersection(condition_2); if intersection.is_empty() { continue; } - let k = (n1, *n2); + let k = (*n1, *n2); let r = match new_states.get(&k) { Some(new_r) => *new_r, None => { - let new_r = (new_automaton.new_state(), n1, *n2); + let new_r = (new_automaton.new_state(), *n1, *n2); worklist.push_back(new_r); new_states.insert(k, new_r); new_r } }; - new_automaton.add_transition_to(p.0, r.0, &intersection); + new_automaton.add_transition(p.0, r.0, &intersection); } } } new_automaton.spanning_set = new_spanning_set; - new_automaton.remove_dead_transitions(); - Ok(new_automaton) + new_automaton.remove_dead_states(); + Ok(Cow::Owned(new_automaton)) } + /// Returns `true` if the two automata have a non-empty intersection. + #[tracing::instrument(level = "debug", skip_all, fields(self_states = self.number_of_states(), other_states = other.number_of_states()))] pub fn has_intersection(&self, other: &FastAutomaton) -> Result { if self.is_empty() || other.is_empty() { return Ok(false); } else if self.is_total() || other.is_total() { return Ok(true); } - let execution_profile = ThreadLocalParams::get_execution_profile(); - - let new_spanning_set = self.spanning_set.merge(&other.spanning_set); - - let condition_converter_self_to_new = - ConditionConverter::new(&self.spanning_set, &new_spanning_set)?; - let condition_converter_other_to_new = - ConditionConverter::new(&other.spanning_set, &new_spanning_set)?; + let execution_profile = ExecutionProfile::get(); + + let same_spanning_set = self.spanning_set == other.spanning_set; + let new_spanning_set = if same_spanning_set { + self.spanning_set.clone() + } else { + self.spanning_set.merge(&other.spanning_set) + }; + + let condition_converter_self_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &self.spanning_set, + &new_spanning_set, + )?) + }; + let condition_converter_other_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut projected_self: IntMap> = IntMap::default(); + let mut projected_other: IntMap> = IntMap::default(); let mut new_automaton = FastAutomaton::new_empty(); let mut worklist = - VecDeque::with_capacity(self.get_number_of_states() + other.get_number_of_states()); + VecDeque::with_capacity(self.number_of_states() + other.number_of_states()); let mut new_states: AHashMap<(usize, usize), (usize, usize, usize), _> = - AHashMap::with_capacity(self.get_number_of_states() + other.get_number_of_states()); + AHashMap::with_capacity(self.number_of_states() + other.number_of_states()); let initial_pair = ( new_automaton.start_state, @@ -105,52 +215,69 @@ impl FastAutomaton { while let Some(p) = worklist.pop_front() { execution_profile.assert_not_timed_out()?; + execution_profile.assert_max_number_of_states(new_states.len())?; if self.accept_states.contains(&p.1) && other.accept_states.contains(&p.2) { return Ok(true); } - let transitions_1 = - self.get_projected_transitions(p.1, &condition_converter_self_to_new)?; - let transitions_2 = - other.get_projected_transitions(p.2, &condition_converter_other_to_new)?; - - for (n1, condition_1) in transitions_1 { - for (n2, condition_2) in &transitions_2 { + let transitions_1 = self.projected_transitions( + &mut projected_self, + p.1, + condition_converter_self_to_new.as_ref(), + )?; + let transitions_2 = other.projected_transitions( + &mut projected_other, + p.2, + condition_converter_other_to_new.as_ref(), + )?; + + for (condition_1, n1) in transitions_1 { + for (condition_2, n2) in transitions_2 { let intersection = condition_1.intersection(condition_2); if intersection.is_empty() { continue; } - let k = (n1, *n2); + let k = (*n1, *n2); let r = match new_states.get(&k) { Some(new_r) => *new_r, None => { - let new_r = (new_automaton.new_state(), n1, *n2); + let new_r = (new_automaton.new_state(), *n1, *n2); worklist.push_back(new_r); new_states.insert(k, new_r); new_r } }; - new_automaton.add_transition_to(p.0, r.0, &intersection); + new_automaton.add_transition(p.0, r.0, &intersection); } } } Ok(false) } - fn get_projected_transitions( + /// Returns `state`'s outgoing transitions projected on the operation's + /// spanning set (`condition_converter` is `None` when both inputs already + /// share it), memoized in `cache`: a component state participates in up + /// to |other| product pairs, and projecting it once instead of once per + /// pair keeps the product construction's inner loop allocation-free. + fn projected_transitions<'m>( &self, + cache: &'m mut IntMap>, state: State, - condition_converter: &ConditionConverter, - ) -> Result, EngineError> { - let transitions_1: Result, EngineError> = self - .transitions_from_state_enumerate_iter(&state) - .map(|(&s, c)| match condition_converter.convert(c) { - Ok(condition) => Ok((s, condition)), - Err(err) => Err(err), - }) - .collect(); - - transitions_1 + condition_converter: Option<&ConditionConverter>, + ) -> Result<&'m Vec<(Condition, State)>, EngineError> { + match cache.entry(state) { + Entry::Occupied(o) => Ok(o.into_mut()), + Entry::Vacant(v) => { + let transitions: Result, EngineError> = self + .transitions_from(state) + .map(|(c, &s)| match condition_converter { + Some(converter) => converter.convert(c).map(|c| (c, s)), + None => Ok((c.clone(), s)), + }) + .collect(); + Ok(v.insert(transitions?)) + } + } } } @@ -158,102 +285,143 @@ impl FastAutomaton { mod tests { use crate::regex::RegularExpression; + // `has_intersection` must enforce the state budget like `intersection`, + // so the product pair map cannot grow unchecked. + #[test] + fn has_intersection_respects_state_budget() { + use crate::error::EngineError; + use crate::execution_profile::ExecutionProfileBuilder; + + let a = RegularExpression::parse("abcd", false) + .unwrap() + .to_automaton() + .unwrap(); + let b = RegularExpression::parse("abcd", false) + .unwrap() + .to_automaton() + .unwrap(); + + let result = ExecutionProfileBuilder::new() + .max_number_of_states(2) + .build() + .run(|| a.has_intersection(&b)); + assert!(matches!( + result, + Err(EngineError::AutomatonHasTooManyStates) + )); + } + + // a* ∩ a* = a*: the intersection keeps the (infinite) looping language. + #[test] + fn intersection_keeps_infinite_language() { + let a_star = RegularExpression::parse("a*", false) + .unwrap() + .to_automaton() + .unwrap(); + + let inter = a_star.intersection(&a_star).unwrap(); + assert!(inter.is_match("")); + assert!(inter.is_match("aaaaaaaa")); + assert!(!inter.is_match("b")); + assert!(inter.equivalent(&a_star).unwrap()); + } + #[test] fn test_simple_intersection_regex_1() -> Result<(), String> { - let automaton1 = RegularExpression::new("(abc|ac|aaa)") + let automaton1 = RegularExpression::parse("(abc|ac|aaa)", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("(abcd|ac|aba)") + let automaton2 = RegularExpression::parse("(abcd|ac|aba)", false) .unwrap() .to_automaton() .unwrap(); let intersection = automaton1.intersection(&automaton2).unwrap(); - assert!(intersection.match_string("ac")); - assert!(!intersection.match_string("abc")); - assert!(!intersection.match_string("aaa")); - assert!(!intersection.match_string("abcd")); - assert!(!intersection.match_string("aba")); + assert!(intersection.is_match("ac")); + assert!(!intersection.is_match("abc")); + assert!(!intersection.is_match("aaa")); + assert!(!intersection.is_match("abcd")); + assert!(!intersection.is_match("aba")); Ok(()) } #[test] fn test_simple_intersection_regex_2() -> Result<(), String> { - let automaton1 = RegularExpression::new("a*") + let automaton1 = RegularExpression::parse("a*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("b*") + let automaton2 = RegularExpression::parse("b*", false) .unwrap() .to_automaton() .unwrap(); let intersection = automaton1.intersection(&automaton2).unwrap(); - assert!(intersection.match_string("")); - assert!(!intersection.match_string("a")); - assert!(!intersection.match_string("b")); + assert!(intersection.is_match("")); + assert!(!intersection.is_match("a")); + assert!(!intersection.is_match("b")); Ok(()) } #[test] fn test_simple_intersection_regex_3() -> Result<(), String> { - let automaton1 = RegularExpression::new("x*") + let automaton1 = RegularExpression::parse("x*", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("(xxx)*") + let automaton2 = RegularExpression::parse("(xxx)*", false) .unwrap() .to_automaton() .unwrap(); let intersection = automaton1.intersection(&automaton2).unwrap(); - assert!(intersection.match_string("")); - assert!(intersection.match_string("xxx")); - assert!(intersection.match_string("xxxxxx")); - assert!(!intersection.match_string("xx")); - assert!(!intersection.match_string("xxxx")); + assert!(intersection.is_match("")); + assert!(intersection.is_match("xxx")); + assert!(intersection.is_match("xxxxxx")); + assert!(!intersection.is_match("xx")); + assert!(!intersection.is_match("xxxx")); Ok(()) } #[test] fn test_complex_intersection_regex_1() -> Result<(), String> { - let automaton1 = RegularExpression::new(".*(abc|ac|aaa)") + let automaton1 = RegularExpression::parse(".*(abc|ac|aaa)", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("(abcd|ac|aba)") + let automaton2 = RegularExpression::parse("(abcd|ac|aba)", false) .unwrap() .to_automaton() .unwrap(); let intersection = automaton1.intersection(&automaton2).unwrap(); - assert!(intersection.match_string("ac")); - assert!(!intersection.match_string("aaac")); - assert!(!intersection.match_string("abc")); - assert!(!intersection.match_string("aaa")); - assert!(!intersection.match_string("abcd")); - assert!(!intersection.match_string("aba")); + assert!(intersection.is_match("ac")); + assert!(!intersection.is_match("aaac")); + assert!(!intersection.is_match("abc")); + assert!(!intersection.is_match("aaa")); + assert!(!intersection.is_match("abcd")); + assert!(!intersection.is_match("aba")); Ok(()) } #[test] fn test_complex_intersection_regex_2() -> Result<(), String> { - let automaton1 = RegularExpression::new("(?:[a-z0-9]+(?:\\.[a-z0-9]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])") + let automaton1 = RegularExpression::parse("(?:[a-z0-9]+(?:\\.[a-z0-9]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])", false) .unwrap() .to_automaton().unwrap(); - let automaton2 = RegularExpression::new("avb@.*") + let automaton2 = RegularExpression::parse("avb@.*", false) .unwrap() .to_automaton() .unwrap(); - automaton1.to_dot(); - automaton2.to_dot(); + automaton1.print_dot(); + automaton2.print_dot(); let intersection = automaton1.intersection(&automaton2).unwrap(); assert!(!intersection.is_empty()); - assert!(intersection.match_string("avb@gmail.com")); + assert!(intersection.is_match("avb@gmail.com")); Ok(()) } } diff --git a/src/fast_automaton/operation/minimize.rs b/src/fast_automaton/operation/minimize.rs new file mode 100644 index 0000000..f33f555 --- /dev/null +++ b/src/fast_automaton/operation/minimize.rs @@ -0,0 +1,283 @@ +use crate::execution_profile::ExecutionProfile; + +use super::*; + +impl FastAutomaton { + /// Minimizes the automaton using Hopcroft's Algorithm. + /// + /// If `self` is non-deterministic, it is determinized in place first, + /// unless the [`ExecutionProfile`] disables implicit determinization, in + /// which case [`EngineError::DeterministicAutomatonRequired`] is + /// returned. + #[tracing::instrument(level = "debug", skip_all, fields(states = self.number_of_states(), deterministic = self.is_deterministic(), minimal = self.is_minimal()))] + pub fn minimize(&mut self) -> Result<(), EngineError> { + // The `minimal` flag is conservatively cleared on every mutation, so + // it can be trusted here; this also keeps the + // `minimize_after_determinization` profile from paying a second + // Hopcroft pass when callers minimize an already-minimized result. + if self.minimal { + return Ok(()); + } + if !self.deterministic { + *self = self.determinize_implicit()?.into_owned(); + } + let execution_profile = ExecutionProfile::get(); + + // Drop states unreachable from the start. A minimal automaton has none, + // and downstream invariants rely on it; in particular `is_empty`'s + // fast path treats any minimal automaton with an accept state as + // non-empty, which only holds if every accept state is reachable. + let reachable = self.forward_reachable_states(); + let unreachable: IntSet = self.states().filter(|s| !reachable.contains(s)).collect(); + if !unreachable.is_empty() { + self.remove_states(&unreachable); + } + + self.remove_dead_states(); + + let max_states = self.transitions.len(); + + let all_states: IntSet = self.states().collect(); + let accept_states: IntSet = self.accept_states().iter().cloned().collect(); + let non_accept_states: IntSet = + all_states.difference(&accept_states).cloned().collect(); + + let mut partitions: Vec> = vec![accept_states, non_accept_states]; + partitions.retain(|p| !p.is_empty()); + + let mut state_to_partition = vec![0; max_states]; + for (i, partition) in partitions.iter().enumerate() { + for &state in partition { + state_to_partition[state] = i; + } + } + + // `in_worklist` is indexed by *partition* id and must stay aligned + // with `partitions` (it grows by one per split, below). Sizing it by + // state count would leave every split-created partition with a stale + // `true` at its slot, breaking Hopcroft's "smaller half" rule and + // degrading the splitter work toward O(n²). + let mut worklist: Vec = (0..partitions.len()).collect(); + let mut in_worklist: Vec = vec![true; partitions.len()]; + + let bases = self.spanning_bases()?; + + // One forward sweep with direct indexing; going through + // `transitions_to_vec` per state would pay two hash lookups per edge. + let mut inverse_transitions: Vec> = vec![Vec::new(); max_states]; + for from_state in self.states() { + for (condition, &to_state) in self.transitions_from(from_state) { + inverse_transitions[to_state].push((from_state, condition.clone())); + } + } + + let mut x = IntSet::with_capacity(self.number_of_states()); + + let mut intersection_states: Vec> = vec![Vec::new(); max_states]; + let mut touched_partitions: Vec = Vec::with_capacity(max_states); + + while let Some(a_idx) = worklist.pop() { + execution_profile.assert_not_timed_out()?; + in_worklist[a_idx] = false; + + let a = partitions[a_idx].clone(); + + for base in &bases { + x.clear(); + + // Find states that transition into partition 'A' on 'base' + for &to_state in &a { + for (from_state, condition) in &inverse_transitions[to_state] { + if base.has_intersection(condition) { + x.insert(*from_state); + } + } + } + + if x.is_empty() { + continue; + } + + // TARGETED SPLITTING: Only evaluate partitions we know overlap with 'x' + for &state in &x { + let p_idx = state_to_partition[state]; + if intersection_states[p_idx].is_empty() { + touched_partitions.push(p_idx); + } + intersection_states[p_idx].push(state); + } + + // Process only the affected partitions + for &p_idx in &touched_partitions { + let int_states = &mut intersection_states[p_idx]; + let y_len = partitions[p_idx].len(); + + // If the partition is fully contained in 'x', no split happens. + if int_states.len() == y_len { + int_states.clear(); + continue; + } + + // A split happens! 'int_states' becomes the new partition. + let new_idx = partitions.len(); + let mut new_part = IntSet::with_capacity(int_states.len()); + + for &state in int_states.iter() { + partitions[p_idx].remove(&state); // Remove from original (forming the difference) + new_part.insert(state); // Add to new partition (forming the intersection) + state_to_partition[state] = new_idx; // Update the lookup array + } + + let diff_len = partitions[p_idx].len(); + let int_len = new_part.len(); + + partitions.push(new_part); + in_worklist.push(false); + + // Worklist update + if in_worklist[p_idx] || int_len <= diff_len { + worklist.push(new_idx); + in_worklist[new_idx] = true; + } else { + worklist.push(p_idx); + in_worklist[p_idx] = true; + } + + int_states.clear(); + } + touched_partitions.clear(); + } + } + + if partitions.len() == all_states.len() { + self.minimal = true; + return Ok(()); + } + + self.rebuild_automaton_from_partition(&partitions)?; + + self.minimal = true; + Ok(()) + } + + fn rebuild_automaton_from_partition( + &mut self, + partitions: &[IntSet], + ) -> Result<(), EngineError> { + let mut state_to_rep = vec![0; self.transitions.len()]; + let mut representatives = Vec::with_capacity(partitions.len()); + + for partition in partitions { + let representative = if partition.contains(&self.start_state()) { + self.start_state() + } else { + *partition + .iter() + .next() + .expect("A partition cannot be empty") + }; + + representatives.push(representative); + + for &state in partition { + state_to_rep[state] = representative; + } + } + + let mut transitions_to_update = Vec::new(); + for &rep in &representatives { + for (condition, old_target) in self.transitions_from_vec(rep) { + let new_target = state_to_rep[old_target]; + transitions_to_update.push((rep, condition, new_target)); + } + } + + for partition in partitions { + for &state in partition { + let rep = state_to_rep[state]; + if state != rep { + self.remove_state(state); + } + } + } + + for (from, condition, to) in transitions_to_update { + self.add_transition(from, to, &condition); + } + + self.recompute_minimal_spanning_set() + } +} + +#[cfg(test)] +mod tests { + use crate::regex::RegularExpression; + + #[test] + fn test_minimize_various_regexes() -> Result<(), String> { + let test_cases = [ + "a", + "a|b", + "ab", + "a|a", + "a(b|c)d|a(b|c)d", + "(ab|ab|ab)", + "a*|b*", + "(a|b)*a(a|b)*", + "(abc|de)", + "a(b|c)*d", + "((a|b)c|(a|b)d)", + "a+b?", + "(a+b)*", + ]; + + for regex in test_cases { + assert_minimize(regex)?; + } + + Ok(()) + } + + fn assert_minimize(regex: &str) -> Result<(), String> { + println!("{regex}"); + let automaton = RegularExpression::parse(regex, false) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton = automaton.determinize().unwrap().into_owned(); + let mut minimized_automaton = automaton.clone(); + minimized_automaton.minimize().unwrap(); + + assert!(automaton.equivalent(&minimized_automaton).unwrap()); + + assert!(minimized_automaton.is_deterministic()); + assert!(minimized_automaton.is_minimal()); + Ok(()) + } + + #[test] + fn test_minimize_union_complement_total() -> Result<(), String> { + let automaton = RegularExpression::parse("(abc|de)", false) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton = automaton.determinize().unwrap(); + let mut complement = automaton.clone().into_owned(); + complement.complement().unwrap(); + + let union = automaton.union(&complement).unwrap(); + let mut union = union.determinize().unwrap().into_owned(); + + assert!(union.is_deterministic()); + assert!(!union.is_minimal()); + + union.minimize().unwrap(); + assert!(union.is_total()); + + assert!(union.is_deterministic()); + assert!(union.is_minimal()); + Ok(()) + } +} diff --git a/src/fast_automaton/operation/mod.rs b/src/fast_automaton/operation/mod.rs index 7c7c0f1..de8dcc8 100644 --- a/src/fast_automaton/operation/mod.rs +++ b/src/fast_automaton/operation/mod.rs @@ -1,23 +1,61 @@ -use std::{cmp, hash::Hasher}; - -use ahash::AHasher; +use std::cmp; use super::*; +use condition::converter::ConditionConverter; + +/// Projects `condition` through `converter`, or borrows it unchanged when no +/// projection is needed (`None`: both operands already share the spanning +/// set, the dominant case in operation chains). +fn convert_condition<'c>( + converter: Option<&ConditionConverter<'_, '_>>, + condition: &'c Condition, +) -> Result, EngineError> { + Ok(match converter { + Some(converter) => std::borrow::Cow::Owned(converter.convert(condition)?), + None => std::borrow::Cow::Borrowed(condition), + }) +} -mod alternation; -mod concatenate; +mod concat; mod determinize; +mod difference; mod intersection; -mod subtraction; +mod minimize; +mod repeat; +mod union; impl FastAutomaton { - pub fn remove_dead_transitions(&mut self) { + /// The shared preamble of the non-degenerate operation cores + /// (`concat_mut_nondegenerate`, `union_mut_nondegenerate`): a cheap + /// necessary condition for the caller-guaranteed invariant (the full + /// degenerate checks are exactly what the cores exist to avoid re-running), + /// the timeout check, and — only when a state limit is configured — the + /// predicted-size check. + fn assert_nondegenerate_operation_fits( + &self, + other: &FastAutomaton, + predicted_states: impl FnOnce() -> usize, + ) -> Result<(), crate::error::EngineError> { + debug_assert!(!self.accept_states.is_empty() && !other.accept_states.is_empty()); + + let execution_profile = crate::execution_profile::ExecutionProfile::get(); + execution_profile.assert_not_timed_out()?; + if execution_profile.limits_number_of_states() { + execution_profile.assert_max_number_of_states(predicted_states())?; + } + Ok(()) + } + + /// Removes "dead" states (those that cannot reach any accept state), since + /// they never contribute to the language. If the language is empty the whole + /// automaton collapses to the canonical empty automaton. + pub fn remove_dead_states(&mut self) { if !self.is_empty() { - let reacheable_states = self.get_reacheable_states(); + let live_states = self.live_states(); let mut dead_states = IntSet::default(); - for from_state in self.transitions_iter() { - if !reacheable_states.contains(&from_state) { + for from_state in self.states() { + if !live_states.contains(&from_state) { dead_states.insert(from_state); } } @@ -34,17 +72,17 @@ mod tests { #[test] fn test_remove_dead_states() -> Result<(), String> { - let automaton1 = RegularExpression::new("(abc|ac|aaa)") + let automaton1 = RegularExpression::parse("(abc|ac|aaa)", false) .unwrap() .to_automaton() .unwrap(); - let automaton2 = RegularExpression::new("(abcd|ac|aba)") + let automaton2 = RegularExpression::parse("(abcd|ac|aba)", false) .unwrap() .to_automaton() .unwrap(); let intersection = automaton1.intersection(&automaton2).unwrap(); - assert_eq!(3, intersection.get_number_of_states()); - assert_eq!(3, intersection.get_reacheable_states().len()); + assert_eq!(3, intersection.number_of_states()); + assert_eq!(3, intersection.live_states().len()); Ok(()) } } diff --git a/src/fast_automaton/operation/repeat.rs b/src/fast_automaton/operation/repeat.rs new file mode 100644 index 0000000..24df7de --- /dev/null +++ b/src/fast_automaton/operation/repeat.rs @@ -0,0 +1,527 @@ +use super::*; + +impl FastAutomaton { + /// Computes the repetition of the automaton between `min` and `max_opt` times; if `max_opt` is `None`, the repetition is unbounded. + #[tracing::instrument(level = "debug", skip(self), fields(states = self.number_of_states(), deterministic = self.is_deterministic(), min = min, max_opt = tracing::field::debug(max_opt)))] + pub fn repeat(&self, min: u32, max_opt: Option) -> Result { + let mut automaton = self.clone(); + automaton.repeat_mut(min, max_opt)?; + Ok(automaton) + } + + pub(crate) fn repeat_mut(&mut self, min: u32, max_opt: Option) -> Result<(), EngineError> { + let execution_profile = ExecutionProfile::get(); + execution_profile.assert_not_timed_out()?; + execution_profile + .assert_max_number_of_states(self.repeat_state_count_heuristic(min, max_opt))?; + + if let Some(max) = max_opt + && min > max + { + self.make_empty(); + return Ok(()); + } + + // r⁰ = {""} for any language (max == 0 implies min == 0 here, since + // min > max already returned above). Without this, the general path + // below would leave the original language reachable and return + // L ∪ {""} instead of just {""}. + if max_opt == Some(0) { + self.make_empty_string(); + return Ok(()); + } + + // The empty-string language is a fixpoint of repetition: {""}{m,n} = {""} + // for any valid m ≤ n. Returning early also avoids the unbounded + // construction below, whose single-state "tight loop" branch would + // otherwise try to remove the start state and panic. + if self.is_empty_string() { + return Ok(()); + } + + // Empty language: ∅⁰ = {""}, ∅ⁿ = ∅ for n ≥ 1. The general algorithm + // below assumes a non-empty language; bail out before it can panic. + // This must be the semantic `is_empty()` check, not just + // `accept_states.is_empty()`: an automaton whose accept states are + // all unreachable is the empty language too, and the construction + // below breaks on it (concatenation prunes the dead accepts, leaving + // stale state ids in the accept frontier). + if self.is_empty() { + if min == 0 { + // ∅⁰ is exactly {""}: replace the whole automaton instead + // of marking the start accepting: a dead automaton can still + // have reachable transitions (e.g. a self-loop on a + // non-accepting start), and an accepting start would wrongly + // revive them into (label)*. + self.make_empty_string(); + } + return Ok(()); + } + + let automaton_to_repeat = self.clone(); + + if min == 0 && self.in_degree(self.start_state) != 0 { + let new_state = self.new_state(); + if self.is_accepted(self.start_state) { + self.accept(new_state); + } + + self.add_epsilon_transition(new_state, self.start_state); + self.start_state = new_state; + + if max_opt.is_none() { + for accept_state in self.accept_states.clone() { + self.add_epsilon_transition(accept_state, self.start_state); + } + self.accept(self.start_state); + return Ok(()); + } + } + + if let Some(max) = max_opt + && min <= 1 + && max == 1 + { + if min == 0 { + // Through `accept()`, not a direct insert: the language + // changes (it gains ""), so the `minimal` flag must clear. + self.accept(self.start_state); + } + return Ok(()); + } + + // From here on `self` and `automaton_to_repeat` are known to be + // neither ∅ nor {""} (checked above), and concatenating two such + // languages preserves that: the loops call the concatenation core + // directly, since re-checking the growing chain on every iteration is + // quadratic. + let iter = if min == 0 { 0..0 } else { 0..min - 1 }; + for _ in iter { + self.concat_mut_nondegenerate(&automaton_to_repeat, false)?; + } + + if max_opt.is_none() { + if min == 0 { + // r* with a start state that has no incoming edges (the + // in_degree > 0 case already returned above): loop the single + // copy in place by letting each accept state re-enter the + // start, and make the start accepting. + let mut star = automaton_to_repeat.clone(); + + let accept_state = *star.accept_states.iter().next().unwrap(); + if star.accept_states.len() == 1 + && star.out_degree(accept_state) == 0 + && star.in_degree(star.start_state) == 0 + { + star.add_epsilon_transition(accept_state, star.start_state); + let old_start_state = star.start_state; + star.start_state = accept_state; + star.remove_state(old_start_state); + } else { + let t = Self::transitions_from_state_set(&star.transitions, star.start_state); + let transitions = + Self::transitions_from_state_enumerate(&t, &star.removed_states); + + for state in star.accept_states.clone() { + for &(to_state, condition) in &transitions { + star.add_transition(state, *to_state, condition); + } + } + + star.accept(star.start_state()); + } + + self.apply_model(&star); + } else { + // r{min,} = rᵐⁱⁿ · r*. Build the star part via recursion rather + // than looping `automaton_to_repeat` in place: when the start + // state has incoming edges, `repeat(0, None)` introduces a + // clean accepting start instead of marking the looping start + // accepting, which would otherwise accept partial copies + // (e.g. `(a*b)+` matching "aaba"). + // The star of a non-degenerate language is non-degenerate: it + // keeps every string of `r` and gains "". + let star = automaton_to_repeat.repeat(0, None)?; + self.concat_mut_nondegenerate(&star, false)?; + } + + return Ok(()); + } + + // Finite maximum: append the optional copies one at a time, keeping + // `self` with a single accept frontier so the chain stays linear, and + // collect each copy boundary in `end_states` to mark accepting at the + // end (stopping after any copy in `min..=max` is valid). + // + // When the copy's start state has incoming edges, merging it into the + // previous copy's accept state would let that (re-marked accepting) + // junction inherit the copy's own transitions and accept partial + // copies (e.g. `(a*b){1,3}` matching "ba"). In that case we force a + // non-merging concatenation so each boundary is a clean accept state + // reached by an epsilon transition. + let force_no_merge = automaton_to_repeat.in_degree(automaton_to_repeat.start_state) > 0; + let mut end_states = self.accept_states.iter().cloned().collect::>(); + for _ in cmp::max(min, 1)..max_opt.unwrap() { + self.concat_mut_nondegenerate(&automaton_to_repeat, force_no_merge)?; + end_states.extend(self.accept_states.iter()); + } + for end_state in end_states { + self.accept(end_state); + } + if min == 0 { + self.accept(self.start_state); + } + Ok(()) + } + + /// Computes the expected number of states after calling `repeat_mut`, + /// reusing the concatenation heuristic to determine loop costs. + fn repeat_state_count_heuristic(&self, min: u32, max_opt: Option) -> usize { + // 1. Invalid range clears the automaton + if let Some(max) = max_opt + && min > max + { + return 0; + } + + // 1b. r⁰ = {""} (a single state); see `repeat_mut`. + if max_opt == Some(0) { + return 1; + } + + let v_original = self.number_of_states(); + if v_original == 0 { + return 0; + } + + let mut current_states = v_original; + let in_deg_start = self.in_degree(self.start_state) > 0; + + // --- REUSE CONCAT HEURISTIC HERE --- + // Calculate the state delta for a single concatenation. The concat + // heuristic short-circuits to a *smaller* value than `v_original` + // for degenerate languages (∅ → 1, {""} → the operand size), so the + // delta must saturate: `repeat_mut` early-returns for those inputs + // right after this estimate anyway. + let concat_cost = self + .concat_state_count_heuristic(self, false) + .saturating_sub(v_original); + + // 2. Early state allocation for 0-minimum repeats with incoming start edges + if min == 0 && in_deg_start { + current_states += 1; + if max_opt.is_none() { + return current_states; + } + } + + // 3. Simple cases: 0..=1 or 1..=1 repetitions + if let Some(max) = max_opt + && min <= 1 + && max == 1 + { + return current_states; + } + + // 4. Minimum repetitions loop + let min_iters = if min == 0 { 0 } else { min - 1 }; + current_states += min_iters as usize * concat_cost; + + // 5. Infinite repetition (max_opt is None) + if max_opt.is_none() { + if min == 0 { + // In-place looped r*: a single accept state with no outgoing + // edges and an incoming-edge-free start drops the old start + // state (`v_original - 1`); otherwise the state count is + // unchanged (the `min == 0 && in_deg_start` case already + // returned in step 2). + let mut v_modified = v_original; + if self.accept_states.len() == 1 { + let accept_state = *self.accept_states.iter().next().unwrap(); + if self.out_degree(accept_state) == 0 && !in_deg_start { + v_modified -= 1; + } + } + return v_modified; + } else { + // r{min,} = rᵐⁱⁿ · r*. `current_states` already accounts for + // the rᵐⁱⁿ part. The star's size and whether its start has + // incoming edges follow directly from the `repeat_mut(0, None)` + // construction, so derive them here rather than building the + // star automaton just to measure it: + // - start with incoming edges → fresh accepting start + // (v + 1 states); epsilon "transitions" copy outgoing edges, + // so nothing ever points into the fresh start; + // - single dead-end accept with an incoming-edge-free start → + // in-place loop rooted at the old accept (v - 1 states), + // which keeps incoming edges (its surviving predecessors, or + // the self-loop copied from a direct start→accept edge); + // - otherwise → in-place loop (v states) keeping the + // incoming-edge-free start. + let acc_out_gt_0 = self.accept_states.iter().any(|&s| self.out_degree(s) > 0); + let (star_states, star_start_has_in_edges) = if in_deg_start { + (v_original + 1, false) + } else if self.accept_states.len() == 1 + && self + .accept_states + .iter() + .next() + .is_some_and(|&s| self.out_degree(s) == 0) + { + (v_original - 1, true) + } else { + (v_original, false) + }; + let not_mergeable = star_start_has_in_edges && acc_out_gt_0; + let final_concat_cost = if not_mergeable { + star_states + } else { + star_states.saturating_sub(1) + }; + return current_states + final_concat_cost; + } + } + + // 6. Finite maximum repetition loop + // + // The mandatory copies (handled above) merge as plain `r`. Each + // optional tail copy merges as well (`v - 1` new states), except when + // the start state has an incoming edge: the non-merging concatenation + // then introduces a fresh start state, costing `v` per copy. + let max = max_opt.unwrap(); + let loop_start = if min > 1 { min } else { 1 }; + let max_iters = max.saturating_sub(loop_start); + + let optional_states = v_original + if in_deg_start { 1 } else { 0 }; + current_states += max_iters as usize * (optional_states - 1); + + current_states + } +} + +#[cfg(test)] +mod tests { + // Building a large bounded repetition must stay linear in the bound: the + // per-copy concatenations run against a growing chain, and re-checking + // that chain's emptiness on every copy made this quadratic (~10 s in + // debug builds at this size, milliseconds when linear). + #[test] + fn repeat_large_bounded_stays_linear() { + let automaton = crate::regex::RegularExpression::parse("[ab]{5000}", false) + .unwrap() + .to_automaton() + .unwrap(); + + assert_eq!(5001, automaton.number_of_states()); + assert!(automaton.is_match(&"ab".repeat(2500))); + assert!(!automaton.is_match(&"ab".repeat(2499))); + } + + // Repeating an empty-language automaton must respect ∅* = {""} and + // ∅ⁿ = ∅ even when the emptiness comes from unreachable accept states or + // dead-but-reachable transitions (rather than an absent accept set): the + // repeat must not revive those dead transitions. + #[test] + fn repeat_of_unreachable_accept_empty_language() { + let mut a = crate::fast_automaton::FastAutomaton::new_empty(); + let s1 = a.new_state(); + a.accept(s1); // unreachable accept: the language is ∅ + assert!(a.is_empty()); + + let star = a.repeat(0, None).unwrap(); // ∅* = {""} + assert!(star.is_match("")); + assert!(!star.is_match("a")); + + assert!(a.repeat(1, Some(2)).unwrap().is_empty()); // ∅{1,2} = ∅ + assert!(a.repeat(2, None).unwrap().is_empty()); // ∅{2,} = ∅ + + // A dead automaton with REACHABLE transitions: ∅* must still be + // exactly {""}, without reviving the dead self-loop into b*. + let range_b = crate::CharRange::new_from_range( + regex_charclass::char::Char::new('b')..=regex_charclass::char::Char::new('b'), + ); + let mut dead_loop = crate::fast_automaton::FastAutomaton::new_empty(); + dead_loop.add_transition_from_range(0, 0, &range_b).unwrap(); + assert!(dead_loop.is_empty()); + + let star = dead_loop.repeat(0, None).unwrap(); + assert!(star.is_match("")); + assert!(!star.is_match("b"), "∅* must not contain \"b\""); + assert!(dead_loop.repeat(1, None).unwrap().is_empty()); + } + + // Repeating a multi-state empty-language automaton must not underflow the + // state-count heuristic (the concat heuristic short-circuits ∅ to 1) in + // the public `repeat` before the empty-language early-return runs. + #[test] + fn repeat_of_multi_state_empty_language_does_not_underflow() { + let mut a = crate::fast_automaton::FastAutomaton::new_empty(); + a.new_state(); // ≥ 2 states, no accept states: the empty language + + let star = a.repeat(0, None).unwrap(); // ∅* = {""} + assert!(star.is_match("")); + assert!(!star.is_match("a")); + + let plus = a.repeat(1, None).unwrap(); // ∅⁺ = ∅ + assert!(plus.is_empty()); + + let bounded = a.repeat(2, Some(3)).unwrap(); // ∅{2,3} = ∅ + assert!(bounded.is_empty()); + } + + // The r{0,1} fast path changes the language (it gains ""), so it must go + // through `accept()` and clear the `minimal` flag; otherwise `minimize()` + // (which trusts the flag) would refuse to minimize the mutated automaton. + #[test] + fn repeat_zero_or_one_clears_the_minimal_flag() { + let mut a = crate::regex::RegularExpression::new("ab") + .unwrap() + .to_automaton() + .unwrap(); + a.minimize().unwrap(); + assert!(a.is_minimal()); + assert!(!a.is_match("")); + + a.repeat_mut(0, Some(1)).unwrap(); + assert!(a.is_match("")); + assert!(a.is_match("ab")); + assert!(!a.is_minimal(), "the language changed: the flag must clear"); + } + + use crate::fast_automaton::FastAutomaton; + use crate::regex::RegularExpression; + + // r⁰ must be exactly {""} for a non-empty language, not L ∪ {""}. + #[test] + fn bug_repeat_zero_zero_on_non_empty() { + let a = RegularExpression::parse("abc", false) + .unwrap() + .to_automaton() + .unwrap(); + let r = a.repeat(0, Some(0)).unwrap(); + assert!(r.is_match(""), "L^0 must contain \"\""); + assert!( + !r.is_match("abc"), + "L^0 must NOT contain L (got 'abc' match)" + ); + } + + // {""} is a fixpoint of repetition: every bound must return {""} without + // panicking (in particular the unbounded "tight loop" branch must not try + // to remove the single state while it is still the start state). + #[test] + fn repeat_of_empty_string_is_fixpoint() { + let empty_string = FastAutomaton::new_empty_string(); + for (min, max) in [ + (0, None), + (1, None), + (3, None), + (0, Some(1)), + (2, Some(5)), + (0, Some(0)), + ] { + let r = empty_string.repeat(min, max).unwrap(); + assert!(r.is_match(""), "{{\"\"}}{{{min},{max:?}}} must match \"\""); + assert!( + !r.is_match("a"), + "{{\"\"}}{{{min},{max:?}}} must match only \"\"" + ); + } + } + + // Unbounded repetition of the empty language must not panic (the branch + // must not assume an accept state exists): ∅* = {""} and ∅⁺ = ∅. + #[test] + fn empty_repeat_unbounded_does_not_panic() { + let empty = FastAutomaton::new_empty(); + // Expected: ∅* = {""}. + let r = empty.repeat(0, None).expect("should not error"); + assert!(r.is_match("")); + assert!(!r.is_match("a")); + + // Expected: ∅⁺ = ∅. + let r = empty.repeat(1, None).expect("should not error"); + assert!(!r.is_match("")); + assert!(!r.is_match("a")); + } + + #[test] + fn test_repeat_1() -> Result<(), String> { + let automaton = RegularExpression::parse("(a*,a*)?", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("")); + assert!(automaton.is_match(",")); + assert!(automaton.is_match("aaa,")); + assert!(automaton.is_match("aaaa,aa")); + assert!(!automaton.is_match("a")); + assert!(!automaton.is_match("aa")); + Ok(()) + } + + #[test] + fn test_heuristic() -> Result<(), String> { + assert_heuristic("b*a"); + assert_heuristic("a*b"); + assert_heuristic("ba*"); + assert_heuristic(".{900}"); + assert_heuristic("[a-z]+"); + assert_heuristic("[a-z]+@"); + + assert_heuristic("[0-9]+[A-Z]*"); + assert_heuristic("a+(ba+)*"); + assert_heuristic("((a|bc)*|d)"); + assert_heuristic(".*"); + assert_heuristic("(ac|ads|a)*"); + assert_heuristic("((aad|ads|a)*|q)"); + + assert_heuristic( + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + assert_heuristic("(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@"); + assert_heuristic( + "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", + ); + assert_heuristic("((aad|ads|a)*abc.*uif(aad|ads|x)*|q)"); + Ok(()) + } + + fn assert_heuristic(regex: &str) { + println!("Testing regex: {regex}"); + + let automaton = RegularExpression::parse(regex, false) + .unwrap() + .to_automaton() + .unwrap(); + + // A matrix of test cases covering all edge cases in the repeat logic + let test_cases = vec![ + (0, Some(0)), // Zero-repeat + (0, Some(1)), // Optional once + (1, Some(1)), // Exactly once + (5, Some(10)), // Standard finite range + (0, None), // Zero or more (Kleene star) + (1, None), // One or more (Kleene plus) + (3, None), // Finite minimum, infinite maximum + ]; + + for (min, max_opt) in test_cases { + // Clone the original automaton to avoid mutating it across iterations + let mut actual_automaton = automaton.clone(); + + // Execute the actual mutation (assuming repeat_mut is the core method) + actual_automaton.repeat_mut(min, max_opt).unwrap(); + + let actual_states = actual_automaton.number_of_states(); + let heuristic_states = automaton.repeat_state_count_heuristic(min, max_opt); + + assert_eq!( + actual_states, heuristic_states, + "Mismatch for regex '{}' with min={}, max={:?}.\nExpected (heuristic): {}\nActual (computed): {}", + regex, min, max_opt, heuristic_states, actual_states + ); + } + } +} diff --git a/src/fast_automaton/operation/subtraction.rs b/src/fast_automaton/operation/subtraction.rs deleted file mode 100644 index d513fbb..0000000 --- a/src/fast_automaton/operation/subtraction.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::hash::BuildHasherDefault; - -use crate::EngineError; - -use super::*; - -impl FastAutomaton { - fn totalize(&mut self) -> Result<(), EngineError> { - if !self.is_determinitic() { - return Err(EngineError::AutomatonShouldBeDeterministic); - } - let crash_state = self.new_state(); - let mut transitions_to_crash_state: IntMap = - IntMap::with_capacity_and_hasher( - self.get_number_of_states(), - BuildHasherDefault::default(), - ); - - let mut ranges = Vec::with_capacity(self.get_number_of_states()); - for from_state in self.transitions_iter() { - let mut new_condition = Condition::empty(&self.spanning_set); - for (_, condition) in self.transitions_from_state_enumerate_iter(&from_state) { - new_condition = new_condition.union(condition); - ranges.push(condition.to_range(self.get_spanning_set())?); - } - - new_condition = new_condition.complement(); - - transitions_to_crash_state.insert(from_state, new_condition); - } - - for (from_state, condition) in &transitions_to_crash_state { - self.add_transition_to(*from_state, crash_state, condition); - ranges.push(condition.to_range(self.get_spanning_set())?); - } - - let new_spanning_set = SpanningSet::compute_spanning_set(&ranges); - self.apply_new_spanning_set(&new_spanning_set)?; - - if self.in_degree(crash_state) == 1 { - self.remove_state(crash_state); - } - Ok(()) - } - - pub fn complement(&mut self) -> Result<(), EngineError> { - self.totalize()?; - - let mut new_accept_states = IntSet::default(); - for state in self.transitions_iter() { - if self.accept_states.contains(&state) { - continue; - } - new_accept_states.insert(state); - } - - self.accept_states = new_accept_states; - Ok(()) - } - - pub fn subtraction(&self, other: &FastAutomaton) -> Result { - let mut complement = other.clone(); - match complement.complement() { - Ok(()) => self.intersection(&complement), - Err(err) => Err(err), - } - } -} - -#[cfg(test)] -mod tests {} diff --git a/src/fast_automaton/operation/union.rs b/src/fast_automaton/operation/union.rs new file mode 100644 index 0000000..284807c --- /dev/null +++ b/src/fast_automaton/operation/union.rs @@ -0,0 +1,723 @@ +use std::hash::BuildHasherDefault; + +use condition::converter::ConditionConverter; +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crate::{error::EngineError, execution_profile::ExecutionProfile}; + +use super::*; + +impl FastAutomaton { + /// Computes the union between `self` and `other`. + pub fn union(&self, other: &FastAutomaton) -> Result { + let mut new_automaton = self.clone(); + new_automaton.union_mut(other)?; + Ok(new_automaton) + } + + /// Computes the union of all automata in the given iterator. + #[tracing::instrument(level = "debug", skip_all)] + pub fn union_all<'a, I: IntoIterator>( + automata: I, + ) -> Result { + // Each operand's degenerate checks run once, on the operand: running + // them per fold step, on the growing result, made large alternations + // quadratic. + let mut new_automaton = FastAutomaton::new_empty(); + let mut seeded = false; + for automaton in automata { + if automaton.is_empty() { + // ∅ is the identity. + continue; + } + if automaton.is_total() { + // Σ* absorbs the whole union. + new_automaton.make_total(); + return Ok(new_automaton); + } + if seeded { + new_automaton.union_mut_nondegenerate(automaton)?; + } else { + new_automaton.apply_model(automaton); + seeded = true; + } + } + Ok(new_automaton) + } + + /// Computes in parallel the union of all automata in the given iterator. + /// + /// Only available with the `parallel` feature (enabled by default). + #[cfg(feature = "parallel")] + #[tracing::instrument(level = "debug", skip_all)] + pub fn union_all_par<'a, I: IntoParallelIterator>( + automata: I, + ) -> Result { + let execution_profile = ExecutionProfile::get(); + + let empty = FastAutomaton::new_empty(); + + automata + .into_par_iter() + .try_fold( + || empty.clone(), + |mut acc, next| { + execution_profile.apply(|| { + acc.union_mut(next)?; + Ok(acc) + }) + }, + ) + .try_reduce( + || empty.clone(), + |mut acc, next| { + execution_profile.apply(|| { + acc.union_mut(&next)?; + Ok(acc) + }) + }, + ) + } + + fn prepare_start_states( + &mut self, + other: &FastAutomaton, + new_states: &mut IntMap, + condition_converter: Option<&ConditionConverter>, + ) -> Result, EngineError> { + let mut imcomplete_states = IntSet::with_capacity(other.out_degree(other.start_state) + 1); + // If `other` accepts the empty string we must make the union's *entry* + // state accepting, but only after the start state is finalized below. + // Marking the current start eagerly is wrong when it has incoming edges + // (e.g. a self-loop) and is about to be demoted behind a fresh start: + // the demoted state would then wrongly accept the strings on its loop. + let self_start_state_in_degree = self.in_degree(self.start_state); + let other_start_state_in_degree = other.in_degree(other.start_state); + if self_start_state_in_degree == 0 && other_start_state_in_degree == 0 { + // The start states can be the same state without any consequence + new_states.insert(other.start_state, self.start_state); + imcomplete_states.insert(self.start_state); + } else { + if self_start_state_in_degree != 0 { + let new_state = self.new_state(); + + self.add_epsilon_transition(new_state, self.start_state); + self.start_state = new_state; + new_states.insert(other.start_state, self.start_state); + imcomplete_states.insert(self.start_state); + } + if other_start_state_in_degree != 0 { + let new_state = self.new_state(); + if other.is_accepted(other.start_state) { + self.accept(new_state); + } + + new_states.insert(other.start_state, new_state); + imcomplete_states.insert(new_state); + + for (cond, other_to_state) in other.transitions_from(other.start_state) { + let cond = convert_condition(condition_converter, cond)?; + let to_state = match new_states.entry(*other_to_state) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + let new_state = self.new_state(); + imcomplete_states.insert(new_state); + v.insert(new_state); + new_state + } + }; + self.add_transition(self.start_state, to_state, &cond); + } + } + } + // Now that `self.start_state` is the final entry state, record `other`'s + // empty-string acceptance there. `self`'s own empty-string acceptance is + // preserved by the start handling above (a freshly created start + // inherits it through the epsilon transition). + if other.is_accepted(other.start_state) { + self.accept(self.start_state); + } + Ok(imcomplete_states) + } + + fn prepare_accept_states( + &mut self, + other: &FastAutomaton, + new_states: &mut IntMap, + imcomplete_states: &IntSet, + ) { + let mut self_accept_states_without_outgoing_edges = vec![]; + for &state in &self.accept_states { + // The start state must never be a merge candidate: the n > 1 + // branch below removes the merged states, and removing the start + // state panics (e.g. an accepting start with no outgoing edges, + // unioned with an operand whose start has incoming edges). + if self.out_degree(state) == 0 + && !imcomplete_states.contains(&state) + && state != self.start_state + { + self_accept_states_without_outgoing_edges.push(state); + } + } + let accept_state_without_outgoing_edges = + match self_accept_states_without_outgoing_edges.len() { + 1 => Some(self_accept_states_without_outgoing_edges[0]), + n if n > 1 => { + let new_state = self.new_state(); + self.accept(new_state); + + for &accept_state in &self_accept_states_without_outgoing_edges { + for (from_state, condition) in self.transitions_to_vec(accept_state) { + self.add_transition(from_state, new_state, &condition); + } + self.remove_state(accept_state); + } + Some(new_state) + } + _ => None, + }; + + for &state in &other.accept_states { + // Resolve the self-state that represents `state`, allocating one if + // it is not mapped yet, then mark it accepting. The accept flag must + // be applied even when `state` was already mapped during + // `prepare_start_states` (e.g. a start state with incoming edges + // whose outgoing edges reach this accept state); otherwise the + // union would silently drop `other`'s acceptance. + let mapped = match accept_state_without_outgoing_edges { + Some(accept_state) if other.out_degree(state) == 0 => { + *new_states.entry(state).or_insert(accept_state) + } + _ => match new_states.get(&state) { + Some(&mapped) => mapped, + None => { + let new_accept_state = self.new_state(); + new_states.insert(state, new_accept_state); + new_accept_state + } + }, + }; + self.accept(mapped); + } + } + + /* Important things to remember before modifying this method: + * - the start states can't be merged if they have incoming edges + * - the accept states can't be merged if they have outgoing edges + */ + pub(crate) fn union_mut(&mut self, other: &FastAutomaton) -> Result<(), EngineError> { + ExecutionProfile::get().assert_not_timed_out()?; + + if other.is_empty() || self.is_total() { + return Ok(()); + } else if other.is_total() { + self.make_total(); + return Ok(()); + } else if self.is_empty() { + self.apply_model(other); + return Ok(()); + } + + self.union_mut_nondegenerate(other) + } + + /// The union core: neither operand may be the empty language `∅` or all + /// strings `Σ*`. Callers folding many operands + /// ([`union_all`](Self::union_all)) establish that invariant per operand + /// and call this directly: the degenerate checks of + /// [`union_mut`](Self::union_mut) walk the whole automaton, and re-running + /// them on the growing result at every fold step made large alternations + /// quadratic. + fn union_mut_nondegenerate(&mut self, other: &FastAutomaton) -> Result<(), EngineError> { + self.assert_nondegenerate_operation_fits(other, || { + self.union_state_count_nondegenerate(other) + })?; + + // Equal spanning sets (the dominant case in `union_all` folds) skip + // the quadratic merge and treat every condition conversion as identity. + let new_spanning_set; + let condition_converter = if self.spanning_set == other.spanning_set { + None + } else { + new_spanning_set = self.spanning_set.merge(&other.spanning_set); + self.apply_new_spanning_set(&new_spanning_set)?; + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut new_states: IntMap = IntMap::with_capacity_and_hasher( + other.number_of_states(), + BuildHasherDefault::default(), + ); + + let imcomplete_states = + self.prepare_start_states(other, &mut new_states, condition_converter.as_ref())?; + self.prepare_accept_states(other, &mut new_states, &imcomplete_states); + + for from_state in other.states() { + let new_from_state = match new_states.entry(from_state) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + let new_state = self.new_state(); + v.insert(new_state); + new_state + } + }; + for (condition, to_state) in other.transitions_from(from_state) { + let new_condition = convert_condition(condition_converter.as_ref(), condition)?; + let new_to_state = match new_states.entry(*to_state) { + Entry::Occupied(o) => *o.get(), + Entry::Vacant(v) => { + let new_state = self.new_state(); + v.insert(new_state); + new_state + } + }; + self.add_transition(new_from_state, new_to_state, &new_condition); + } + } + self.minimal = false; + Ok(()) + } + + /// Computes the expected number of states after calling `union_mut`. + /// Kept as the specification of the union's state growth; the exactness + /// tests validate it against `union_mut`, and the non-degenerate half + /// backs the state-limit check in the union core. + #[cfg(test)] + fn union_state_count_heuristic(&self, other: &FastAutomaton) -> usize { + // Edge cases + if other.is_empty() || self.is_total() { + return self.number_of_states(); + } else if other.is_total() || self.is_empty() { + return other.number_of_states(); + } + + self.union_state_count_nondegenerate(other) + } + + /// [`union_state_count_heuristic`](Self::union_state_count_heuristic) for + /// operands already known to be non-degenerate: no emptiness walks. + fn union_state_count_nondegenerate(&self, other: &FastAutomaton) -> usize { + let v1 = self.number_of_states(); + let v2 = other.number_of_states(); + + let self_in = self.in_degree(self.start_state); + let other_in = other.in_degree(other.start_state); + + let mut total_delta: i32 = 0; + + // --- 1. Start States Math --- + if self_in == 0 && other_in == 0 { + total_delta -= 1; + } else if self_in != 0 && other_in != 0 { + total_delta += 1; + } + + // Track which 'other' states are already mapped in the start phase + // so we don't double-count them when calculating accept state savings. + let mut mapped_other_states = IntSet::new(); + mapped_other_states.insert(other.start_state); + + if other_in != 0 { + for (_, to_state) in other.transitions_from(other.start_state) { + mapped_other_states.insert(*to_state); + } + } + + // --- 2. Accept States Math --- + // Gather self's accept states. If other.start_state is accepted, + // it virtually triggers self.accept(self.start_state) early. + let mut self_accepts: IntSet = self.accept_states.iter().cloned().collect(); + + if other.is_accepted(other.start_state) { + self_accepts.insert(self.start_state); + } + + let mut n = 0; + + for &state in &self_accepts { + // Mirror `prepare_accept_states`: a state that is (still) the + // start after the start-state phase is never a merge candidate. + // When `self_in != 0` the original start gets demoted behind a + // fresh start, so it *does* participate. + let is_excluded = self_in == 0 && state == self.start_state; + if self.out_degree(state) == 0 && !is_excluded { + n += 1; + } + } + + let has_acc_target = n >= 1; + + // If n > 1, we replace `n` states with exactly 1 unified state. + if n > 1 { + total_delta += 1 - n; + } + + // Calculate mappings for other's accept states + if has_acc_target { + for &state in &other.accept_states { + if other.out_degree(state) == 0 && !mapped_other_states.contains(&state) { + total_delta -= 1; + } + } + } + + (v1 as i32 + v2 as i32 + total_delta) as usize + } +} + +#[cfg(test)] +mod tests { + use crate::{Term, fast_automaton::FastAutomaton, regex::RegularExpression}; + + // Unioning with the empty-string language must keep the other operand's + // acceptance: when `other`'s start state has incoming edges, the accept + // states reachable from it are mapped early, and they must still be + // marked accepting, so `union({""}, "a+")` matches "" and "a", "aa", ... + #[test] + fn union_with_empty_string_keeps_other_accepts() { + let empty_string = RegularExpression::parse("", false) + .unwrap() + .to_automaton() + .unwrap(); + let a_plus = RegularExpression::parse("a+", false) + .unwrap() + .to_automaton() + .unwrap(); + + let u = empty_string.union(&a_plus).unwrap(); + assert!(u.is_match(""), "union must keep \"\""); + assert!( + u.is_match("a"), + "union dropped the other operand's language" + ); + assert!(u.is_match("aaa")); + + // It must be equivalent regardless of operand order. + let u2 = a_plus.union(&empty_string).unwrap(); + assert!( + Term::from_automaton(u) + .equivalent(&Term::from_automaton(u2)) + .unwrap() + ); + } + + // `prepare_accept_states` merges accept states without outgoing edges and + // removes the originals; it must not put `self`'s accepting start (no + // outgoing edges) in the merge list, since removing the start state would + // panic. + #[test] + fn union_does_not_remove_accepting_start() { + use crate::CharRange; + use crate::fast_automaton::condition::Condition; + use crate::fast_automaton::spanning_set::SpanningSet; + use regex_charclass::char::Char; + + let rng = |c: char| { + let c = Char::new(c); + CharRange::new_from_range(c..=c) + }; + let ss = SpanningSet::compute_spanning_set(&[rng('a'), rng('b')]); + + // a: two accepting states without outgoing edges, one being the start. + let mut a = FastAutomaton::new_empty(); + a.apply_new_spanning_set(&ss).unwrap(); + a.new_state(); + a.accept(0); + a.accept(1); + + // b: start has an incoming edge (1 -a-> 0) but no outgoing edges. + let mut b = FastAutomaton::new_empty(); + b.apply_new_spanning_set(&ss).unwrap(); + b.new_state(); + b.add_transition(1, 0, &Condition::from_range(&rng('a'), &ss).unwrap()); + b.accept(0); + + let u = a.union(&b).unwrap(); + assert!(u.is_match(""), "union must keep the empty string"); + } + + // When a language whose start state has a self-loop is unioned with the + // empty string, the empty-string acceptance must land on the union's + // entry state, not on the looping start — otherwise `a*b | ""` would + // wrongly match "a", "aa", ... + #[test] + fn union_with_empty_string_does_not_over_accept() { + let a_star_b = RegularExpression::parse("a*b", false) + .unwrap() + .to_automaton() + .unwrap(); + let empty_string = RegularExpression::parse("", false) + .unwrap() + .to_automaton() + .unwrap(); + + let u = a_star_b.union(&empty_string).unwrap(); + assert!(u.is_match(""), "(a*b)? must match \"\""); + assert!(u.is_match("b")); + assert!(u.is_match("ab")); + assert!(u.is_match("aab")); + assert!( + !u.is_match("a"), + "union wrongly accepted 'a' (looping start marked accepting)" + ); + assert!(!u.is_match("aa")); + } + + #[test] + fn test_simple_alternation_regex_1() -> Result<(), String> { + let automaton = RegularExpression::parse("(abc|ac|aaa)", false) + .unwrap() + .to_automaton() + .unwrap(); + assert!(automaton.is_match("abc")); + assert!(automaton.is_match("ac")); + assert!(automaton.is_match("aaa")); + assert!(!automaton.is_match("abcd")); + assert!(!automaton.is_match("ab")); + assert!(!automaton.is_match("acc")); + assert!(!automaton.is_match("a")); + assert!(!automaton.is_match("aaaa")); + assert!(!automaton.is_match("aa")); + assert!(!automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_2() -> Result<(), String> { + let automaton = RegularExpression::parse("(b?|b{2})", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("b")); + assert!(automaton.is_match("bb")); + assert!(!automaton.is_match("bbb")); + assert!(!automaton.is_match("bbbb")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_3() -> Result<(), String> { + let automaton = RegularExpression::parse("((a|bc)*|d)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("abcaaabcbc")); + assert!(automaton.is_match("d")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("abcd")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_3b() -> Result<(), String> { + let automaton = RegularExpression::parse("(d|(a|bc)*)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("abcaaabcbc")); + assert!(automaton.is_match("d")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("abcd")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_3t() -> Result<(), String> { + let automaton = RegularExpression::parse("(d*|(a|bc)*)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("abcaaabcbc")); + assert!(automaton.is_match("d")); + assert!(automaton.is_match("ddd")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("abcd")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_4() -> Result<(), String> { + let automaton = RegularExpression::parse("(a+(ba+)*|ca*c)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("cc")); + assert!(automaton.is_match("caaac")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aababa")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_5() -> Result<(), String> { + let automaton = RegularExpression::parse("((aad|ads|a)*|q)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("q")); + assert!(automaton.is_match("aad")); + assert!(automaton.is_match("ads")); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("aadadsaaa")); + assert!(!automaton.is_match("aaaas")); + assert!(!automaton.is_match("ad")); + assert!(!automaton.is_match("adsq")); + assert!(!automaton.is_match("qq")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_6() -> Result<(), String> { + let automaton = RegularExpression::parse("(ab|)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("ab")); + assert!(automaton.is_match("")); + assert!(!automaton.is_match("a")); + assert!(!automaton.is_match("b")); + assert!(!automaton.is_match("aab")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_7() -> Result<(), String> { + let automaton = RegularExpression::parse("(d|a?|ab)", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("d")); + assert!(automaton.is_match("ab")); + assert!(automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_simple_alternation_regex_8() -> Result<(), String> { + let automaton = RegularExpression::parse("((d|a?|ab)u)*", false) + .unwrap() + .to_automaton() + .unwrap(); + automaton.print_dot(); + assert!(automaton.is_match("au")); + assert!(automaton.is_match("du")); + assert!(automaton.is_match("abu")); + assert!(automaton.is_match("u")); + assert!(automaton.is_match("")); + Ok(()) + } + + #[test] + fn test_heuristic() -> Result<(), String> { + assert_heuristic(".{900}", "[a-z]+"); + + assert_heuristic("[a-z]+@", "[0-9]+[A-Z]*"); + + assert_heuristic("a+(ba+)*", "((a|bc)*|d)"); + + assert_heuristic(".*", "(ac|ads|a)*"); + + assert_heuristic( + "((aad|ads|a)*|q)", + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + assert_heuristic( + "(?:A+(?:\\.[AB]+)*|\"(?:C|\\\\D)*\")@", + "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@", + ); + + assert_heuristic("((aad|ads|a)*abc.*uif(aad|ads|x)*|q)", ".*"); + + assert_heuristic( + ".{900}", + r"john[!#-'\*\+\-/-9=\?\^-\u{007e}]*(\.[!#-'\*\+\-/-9=\?\^-\u{007e}](\.?[!#-'\*\+\-/-9=\?\^-\u{007e}])*)?\.?doe@example\.com", + ); + + Ok(()) + } + + fn assert_heuristic(regex1: &str, regex2: &str) { + println!("Testing union heuristic for: '{}' | '{}'", regex1, regex2); + + let automaton1 = RegularExpression::parse(regex1, false) + .unwrap() + .to_automaton() + .unwrap(); + + let automaton2 = RegularExpression::parse(regex2, false) + .unwrap() + .to_automaton() + .unwrap(); + + let test_pair = |a1: &FastAutomaton, a2: &FastAutomaton, desc: &str| { + let mut actual_union = a1.clone(); + actual_union.union_mut(a2).unwrap(); + + let actual_states = actual_union.number_of_states(); + let heuristic_states = a1.union_state_count_heuristic(a2); + + assert_eq!( + actual_states, heuristic_states, + "Mismatch for {}.\nExpected (heuristic): {}\nActual (computed): {}", + desc, heuristic_states, actual_states + ); + }; + + // Test standard union: A | B + test_pair( + &automaton1, + &automaton2, + &format!("'{}' | '{}'", regex1, regex2), + ); + + // Test reverse union: B | A + test_pair( + &automaton2, + &automaton1, + &format!("'{}' | '{}'", regex2, regex1), + ); + + // Test self-union: A | A + test_pair( + &automaton1, + &automaton1, + &format!("'{}' | '{}' (Self)", regex1, regex1), + ); + + // Test Empty states + let empty_automaton = FastAutomaton::new_empty(); + + test_pair( + &empty_automaton, + &automaton2, + &format!("Empty | '{}'", regex2), + ); + test_pair( + &automaton1, + &empty_automaton, + &format!("'{}' | Empty", regex1), + ); + } +} diff --git a/src/fast_automaton/serializer.rs b/src/fast_automaton/serializer.rs deleted file mode 100644 index 017341b..0000000 --- a/src/fast_automaton/serializer.rs +++ /dev/null @@ -1,225 +0,0 @@ -use super::*; -use lazy_static::lazy_static; -use rand::Rng; -use serde::{de, ser, Deserializer, Serializer}; -use serde::{Deserialize, Serialize}; -use std::env; -use z85::{decode, encode}; -use crate::tokenizer::Tokenizer; - -use sha2::{Digest, Sha256}; - -use aes_gcm_siv::{ - aead::{Aead, KeyInit}, - Aes256GcmSiv, Nonce, -}; -use flate2::read::ZlibDecoder; -use flate2::write::ZlibEncoder; -use flate2::Compression; -use std::io::prelude::*; - -use crate::tokenizer::token::{automaton_token::AutomatonToken, Token}; - -pub struct FastAutomatonReader { - cipher: Aes256GcmSiv, -} - -impl FastAutomatonReader { - pub fn new() -> Self { - let env_var = env::var("RS_FAIR_SECRET_KEY").unwrap_or("DEFAULT PASSKEY".to_string()); - let key = Sha256::digest(env_var.as_bytes()); - FastAutomatonReader { - cipher: Aes256GcmSiv::new(&key), - } - } - - pub fn random_nonce() -> [u8; 12] { - let mut nonce = [0u8; 12]; - rand::thread_rng().fill(&mut nonce); - nonce - } -} - -lazy_static! { - static ref SINGLETON_INSTANCE: FastAutomatonReader = FastAutomatonReader::new(); -} - -fn get_fast_automaton_reader() -> &'static FastAutomatonReader { - &SINGLETON_INSTANCE -} - -#[derive(Serialize, Deserialize, Debug)] -struct SerializedAutomaton(Vec, SpanningSet); - -impl serde::Serialize for FastAutomaton { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let tokenizer = Tokenizer::new(self); - match AutomatonToken::to_fair_tokens(&tokenizer.to_embedding()) { - Ok(tokens) => { - let serialized_automaton = - SerializedAutomaton(tokens, self.get_spanning_set().clone()); - - let mut serialized = Vec::with_capacity(self.get_number_of_states() * 8); - if let Err(err) = ciborium::into_writer(&serialized_automaton, &mut serialized) { - return Err(ser::Error::custom(err.to_string())); - } - - serialized = compress_data(&serialized); - - let nonce = FastAutomatonReader::random_nonce(); - - match get_fast_automaton_reader() - .cipher - .encrypt(Nonce::from_slice(&nonce), serialized.as_ref()) - { - Ok(ciphertext) => { - let mut encrypted = Vec::from_iter(nonce); - encrypted.extend(ciphertext); - - serializer.serialize_str(&encode(&encrypted)) - } - Err(err) => Err(ser::Error::custom(err.to_string())), - } - } - Err(err) => Err(ser::Error::custom(err.to_string())), - } - } -} - -impl<'de> serde::Deserialize<'de> for FastAutomaton { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - match String::deserialize(deserializer) { - Ok(decoded) => match decode(decoded) { - Ok(encrypted) => { - let nonce = &encrypted[0..12]; - let payload = encrypted[12..].to_vec(); - let cipher_result = get_fast_automaton_reader() - .cipher - .decrypt(Nonce::from_slice(nonce), payload.as_ref()); - - match cipher_result { - Ok(cipher_result) => { - let decrypted = decompress_data(&cipher_result); - - let automaton: Result< - SerializedAutomaton, - ciborium::de::Error, - > = ciborium::from_reader(&decrypted[..]); - match automaton { - Ok(automaton) => { - let mut temp_automaton = FastAutomaton::new_empty(); - temp_automaton.spanning_set = automaton.1; - let tokenizer = Tokenizer::new(&temp_automaton); - - match tokenizer.from_embedding( - &automaton - .0 - .into_iter() - .map(AutomatonToken::from_fair_token) - .collect::>(), - ) { - Ok(res) => Ok(res), - Err(err) => Err(de::Error::custom(err.to_string())), - } - } - Err(err) => Err(de::Error::custom(err.to_string())), - } - } - Err(err) => Err(de::Error::custom(err.to_string())), - } - } - Err(err) => Err(de::Error::custom(err.to_string())), - }, - Err(err) => Err(err), - } - } -} - -fn compress_data(data: &[u8]) -> Vec { - let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); - encoder.write_all(data).expect("Failed to write data"); - encoder.finish().expect("Failed to finish compression") -} - -fn decompress_data(data: &[u8]) -> Vec { - let mut decoder = ZlibDecoder::new(data); - let mut decompressed_data = Vec::new(); - decoder - .read_to_end(&mut decompressed_data) - .expect("Failed to read data"); - decompressed_data -} - -#[cfg(test)] -mod tests { - use crate::regex::RegularExpression; - - use super::*; - - #[test] - fn test_serialization() -> Result<(), String> { - assert_serialization("..."); - assert_serialization(".*abc"); - assert_serialization(".*"); - assert_serialization(".*abcdef.*dsqd"); - assert_serialization( - "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,2}", - ); - assert_serialization("(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])"); - - Ok(()) - } - - fn assert_serialization(regex: &str) { - let regex = RegularExpression::new(regex).unwrap(); - println!("{regex}"); - - let automaton = regex.to_automaton().unwrap(); - - let serialized = serde_json::to_string(&automaton).unwrap(); - println!("{serialized}"); - - let unserialized: FastAutomaton = serde_json::from_str(&serialized).unwrap(); - - let unserialized = unserialized.determinize().unwrap(); - let automaton = automaton.determinize().unwrap(); - - assert!(automaton.subtraction(&unserialized).unwrap().is_empty()); - assert!(unserialized.subtraction(&automaton).unwrap().is_empty()); - } - - #[test] - fn test_serialization_case_1() -> Result<(), String> { - let automaton1 = RegularExpression::new(".*") - .unwrap() - .to_automaton() - .unwrap(); - let automaton2 = RegularExpression::new("\\d+") - .unwrap() - .to_automaton() - .unwrap() - .determinize() - .unwrap(); - - let subtraction = automaton1.subtraction(&automaton2).unwrap(); - - let serialized = serde_json::to_string(&subtraction).unwrap(); - println!("{serialized}"); - - let unserialized: FastAutomaton = serde_json::from_str(&serialized).unwrap(); - - let unserialized = unserialized.determinize().unwrap(); - let automaton = subtraction.determinize().unwrap(); - - assert!(automaton.subtraction(&unserialized).unwrap().is_empty()); - assert!(unserialized.subtraction(&automaton).unwrap().is_empty()); - - Ok(()) - } -} diff --git a/src/fast_automaton/spanning_set/mod.rs b/src/fast_automaton/spanning_set/mod.rs index 2aa2780..d6629a5 100644 --- a/src/fast_automaton/spanning_set/mod.rs +++ b/src/fast_automaton/spanning_set/mod.rs @@ -1,28 +1,57 @@ use std::slice::Iter; -use ahash::AHashSet; -use regex_charclass::{char::Char, irange::RangeSet}; -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; +use ahash::AHashMap; +use regex_charclass::irange::RangeSet; -/// Contains a set of [`RangeSet`] that span all the transition of a [`crate::FastAutomaton`]. -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +use super::{from_scalar, scalar}; +use crate::CharRange; + +/// Converts merged, ascending scalar segments (see [`scalar`](super::scalar)) +/// back into a [`CharRange`]. The segments are disjoint and non-adjacent, so +/// the flat bound list is already the canonical representation the set +/// operations produce. +fn segments_to_range(segments: &[(u32, u32)]) -> CharRange { + let mut bounds = Vec::with_capacity(segments.len() * 2); + for &(start, end) in segments { + bounds.push(from_scalar(start).expect("segment bounds are valid scalars")); + bounds.push(from_scalar(end).expect("segment bounds are valid scalars")); + } + RangeSet(bounds) +} + +/// A set of [`CharRange`] that spans all transitions of a [`crate::FastAutomaton`]. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct SpanningSet(Vec>, RangeSet); +pub struct SpanningSet(Vec, CharRange); impl SpanningSet { + /// Creates a spanning set from explicit disjoint `ranges` plus the `rest` + /// range covering every character they don't. The caller is responsible for + /// these invariants; prefer [`compute_spanning_set`](Self::compute_spanning_set), + /// which derives a minimal, well-formed set from arbitrary ranges. + pub fn new(ranges: Vec, rest: CharRange) -> Self { + SpanningSet(ranges, rest) + } + + /// Creates the spanning set of an automaton with no transitions: no + /// explicit ranges, with the rest covering all characters. pub fn new_empty() -> Self { - SpanningSet(vec![], RangeSet::total()) + SpanningSet(vec![], CharRange::total()) } + /// Creates the spanning set with a single range covering all characters and + /// an empty rest. pub fn new_total() -> Self { - SpanningSet(vec![RangeSet::total()], RangeSet::empty()) + SpanningSet(vec![CharRange::total()], CharRange::empty()) } + /// Returns `true` if this is the empty spanning set (no explicit ranges; + /// see [`new_empty`](Self::new_empty)). pub fn is_empty(&self) -> bool { self.0.is_empty() && self.1.is_total() } + /// Returns `true` if this is the total spanning set (one all-covering + /// range; see [`new_total`](Self::new_total)). pub fn is_total(&self) -> bool { self.0.len() == 1 && self.0[0].is_total() && self.1.is_empty() } @@ -35,32 +64,33 @@ impl SpanningSet { } } - pub(crate) fn get_spanning_ranges_with_rest(&self) -> Vec> { - if self.1.is_empty() { - self.0.clone() - } else { - let mut elements = vec![self.1.clone()]; - elements.extend(self.0.clone()); - elements - } + pub(crate) fn spanning_ranges_with_rest(&self) -> impl Iterator { + std::iter::once(&self.1) + .filter(|rest| !rest.is_empty()) + .chain(self.0.iter()) } - pub fn get_spanning_ranges(&self) -> Iter> { + /// Returns an iterator over the explicit (non-rest) ranges in the spanning set. + pub fn spanning_ranges(&self) -> Iter<'_, CharRange> { self.0.iter() } - pub fn get_number_of_spanning_ranges(&self) -> usize { + /// Returns the number of explicit (non-rest) ranges in the spanning set. + pub fn number_of_spanning_ranges(&self) -> usize { self.0.len() } - pub fn get_spanning_range(&self, i: usize) -> Option<&RangeSet> { + /// Returns the explicit range at index `i`, or `None` if out of bounds. + pub fn spanning_range(&self, i: usize) -> Option<&CharRange> { self.0.get(i) } - pub fn get_rest(&self) -> &RangeSet { + /// Returns the "rest" range covering all characters not in any explicit range. + pub fn rest(&self) -> &CharRange { &self.1 } + /// Compute a new minimal spanning set by merging the provided spanning set. pub fn merge(&self, other: &Self) -> Self { let mut ranges = Vec::with_capacity(self.0.len() + other.0.len()); ranges.extend_from_slice(&self.0); @@ -69,8 +99,170 @@ impl SpanningSet { Self::compute_spanning_set(&ranges) } - pub fn compute_spanning_set(ranges: &[RangeSet]) -> Self { - let mut spanning_ranges: Vec> = ranges.to_vec(); + /// Compute a new minimal spanning set for the provided ranges. + /// + /// The bases are the *atoms* of the inputs: the classes of characters + /// covered by exactly the same subset of input ranges. They are computed + /// with one endpoint sweep over the surrogate-free scalar space, + /// O(E log E) in the total number of input intervals. + pub fn compute_spanning_set(ranges: &[CharRange]) -> Self { + let mut inputs: Vec<&CharRange> = ranges.iter().filter(|r| !r.is_empty()).collect(); + inputs.sort_unstable(); + inputs.dedup(); + + // The two dominant real-world calls: an empty automaton, and + // `add_transition_from_range` extending the set by a single range. + match inputs.as_slice() { + [] => return SpanningSet(vec![], CharRange::total()), + [input] => return SpanningSet(vec![(*input).clone()], input.complement()), + _ => {} + } + + // Interval [lo, hi] becomes the half-open [scalar(lo), scalar(hi)+1). + let interval_count: usize = inputs.iter().map(|input| input.0.len() / 2).sum(); + let mut events: Vec = Vec::with_capacity(interval_count * 2); + for (input_index, input) in inputs.iter().enumerate() { + for pair in input.0.chunks_exact(2) { + events.push(Event { + position: scalar(pair[0]), + input_index: input_index as u32, + is_start: true, + }); + events.push(Event { + position: scalar(pair[1]) + 1, + input_index: input_index as u32, + is_start: false, + }); + } + } + events.sort_unstable_by_key(|event| event.position); + + let (mut spanning_ranges, covered) = if inputs.len() <= 64 { + Self::sweep_small(&events) + } else { + Self::sweep_wide(inputs.len(), &events) + }; + spanning_ranges.sort_unstable(); + + SpanningSet(spanning_ranges, segments_to_range(&covered).complement()) + } + + /// The sweep for at most 64 inputs (virtually every call): the signature + /// fits one `u64`, so tagged segments are grouped into atoms with a plain + /// sort instead of a hash map. + fn sweep_small(events: &[Event]) -> (Vec, Vec<(u32, u32)>) { + let mut active = 0u64; + let mut tagged: Vec<(u64, u32, u32)> = Vec::with_capacity(events.len()); + let mut covered: Vec<(u32, u32)> = Vec::new(); + + let mut i = 0; + let mut previous_position = 0u32; + while i < events.len() { + let position = events[i].position; + if active != 0 && previous_position < position { + let segment = (previous_position, position - 1); + push_merging_adjacent(&mut covered, segment); + tagged.push((active, segment.0, segment.1)); + } + while i < events.len() && events[i].position == position { + active ^= 1 << events[i].input_index; + i += 1; + } + previous_position = position; + } + + // Group by signature; the sweep emitted segments in ascending + // position order, which the (signature, position) sort preserves + // within each group, so adjacent segments merge as in `sweep_wide`. + tagged.sort_unstable(); + let mut atoms: Vec = Vec::new(); + let mut i = 0; + while i < tagged.len() { + let signature = tagged[i].0; + let mut segments: Vec<(u32, u32)> = Vec::new(); + while i < tagged.len() && tagged[i].0 == signature { + push_merging_adjacent(&mut segments, (tagged[i].1, tagged[i].2)); + i += 1; + } + atoms.push(segments_to_range(&segments)); + } + (atoms, covered) + } + + /// The sweep for more than 64 inputs: the signature is a bit vector and + /// atoms are grouped through a hash map. + fn sweep_wide(input_count: usize, events: &[Event]) -> (Vec, Vec<(u32, u32)>) { + let mut active = vec![0u64; input_count.div_ceil(64)]; + let mut active_count = 0usize; + let mut atoms: AHashMap, Vec<(u32, u32)>> = AHashMap::new(); + let mut covered: Vec<(u32, u32)> = Vec::new(); + + let mut i = 0; + let mut previous_position = 0u32; + while i < events.len() { + let position = events[i].position; + if active_count > 0 && previous_position < position { + let segment = (previous_position, position - 1); + push_merging_adjacent(&mut covered, segment); + if let Some(segments) = atoms.get_mut(&active) { + push_merging_adjacent(segments, segment); + } else { + atoms.insert(active.clone(), vec![segment]); + } + } + while i < events.len() && events[i].position == position { + let event = &events[i]; + let (word, bit) = ((event.input_index / 64) as usize, event.input_index % 64); + if event.is_start { + active[word] |= 1 << bit; + active_count += 1; + } else { + active[word] &= !(1 << bit); + active_count -= 1; + } + i += 1; + } + previous_position = position; + } + + ( + atoms + .values() + .map(|segments| segments_to_range(segments)) + .collect(), + covered, + ) + } +} + +/// A boundary of one input interval during the sweep: the covering-input +/// signature is constant between consecutive event positions. +struct Event { + position: u32, + input_index: u32, + is_start: bool, +} + +/// Appends `segment` to an ascending segment list, extending the last entry +/// instead when they touch. +#[inline] +fn push_merging_adjacent(segments: &mut Vec<(u32, u32)>, segment: (u32, u32)) { + match segments.last_mut() { + Some(last) if last.1 + 1 == segment.0 => last.1 = segment.1, + _ => segments.push(segment), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ahash::AHashSet; + use proptest::prelude::*; + + /// The pairwise intersection-splitting fixpoint `compute_spanning_set` + /// replaced, kept as the oracle the sweep is fuzzed against. + fn compute_spanning_set_reference(ranges: &[CharRange]) -> SpanningSet { + let mut spanning_ranges: Vec = ranges.to_vec(); spanning_ranges.sort_unstable(); spanning_ranges.dedup(); @@ -87,29 +279,112 @@ impl SpanningSet { let other_set = spanning_ranges.swap_remove(index); let intersection_set = set.intersection(&other_set); new_spanning_ranges.insert(intersection_set); - let subtraction_set = set.difference(&other_set); - if !subtraction_set.is_empty() { - new_spanning_ranges.insert(subtraction_set); + let difference_set = set.difference(&other_set); + if !difference_set.is_empty() { + new_spanning_ranges.insert(difference_set); } - let subtraction_set = other_set.difference(&set); - if !subtraction_set.is_empty() { - new_spanning_ranges.insert(subtraction_set); + let difference_set = other_set.difference(&set); + if !difference_set.is_empty() { + new_spanning_ranges.insert(difference_set); } changed = true; } else if !set.is_empty() { new_spanning_ranges.insert(set); } } - spanning_ranges = new_spanning_ranges.iter().cloned().collect(); + spanning_ranges = new_spanning_ranges.drain().collect(); } spanning_ranges.sort_unstable(); - let mut total = RangeSet::empty(); + let mut total = CharRange::empty(); for base in &spanning_ranges { total = total.union(base); } SpanningSet(spanning_ranges, total.complement()) } + + /// A random `CharRange` in canonical representation (built by unioning + /// single intervals, which merges overlaps and adjacency) — the form + /// every real caller passes in. + fn arb_char_range() -> impl Strategy { + proptest::collection::vec((0u32..=0x10F7FF, 0u32..=0x10F7FF), 1..4).prop_map(|pairs| { + let mut range = CharRange::empty(); + for (a, b) in pairs { + let (low, high) = if a <= b { (a, b) } else { (b, a) }; + let interval = CharRange::new_from_range( + from_scalar(low).unwrap()..=from_scalar(high).unwrap(), + ); + range = range.union(&interval); + } + range + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn sweep_matches_the_fixpoint_reference( + inputs in proptest::collection::vec(arb_char_range(), 0..6) + ) { + let expected = compute_spanning_set_reference(&inputs); + let actual = SpanningSet::compute_spanning_set(&inputs); + prop_assert_eq!(expected, actual); + } + } + + // More than 64 inputs exercises the wide (bit-vector signature) sweep; + // the wide ranges force overlaps across many inputs at once. + #[test] + fn sweep_wide_matches_the_reference() { + let single = |low: u32, high: u32| { + CharRange::new_from_range(from_scalar(low).unwrap()..=from_scalar(high).unwrap()) + }; + + let mut inputs = Vec::new(); + for i in 0..70u32 { + inputs.push(single(i * 10, i * 10 + 5)); + } + inputs.push(single(3, 400)); + inputs.push(single(250, 699)); + + assert_eq!( + compute_spanning_set_reference(&inputs), + SpanningSet::compute_spanning_set(&inputs) + ); + } + + // Inputs whose intervals touch the extremes and the surrogate hole. + #[test] + fn sweep_handles_boundary_ranges() { + use regex_charclass::char::Char; + + let cases: Vec> = vec![ + vec![], + vec![CharRange::empty()], + vec![CharRange::total()], + vec![CharRange::total(), CharRange::total()], + vec![ + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{D7FF}')), + CharRange::new_from_range(Char::new('\u{E000}')..=Char::new('\u{10FFFF}')), + ], + vec![ + CharRange::new_from_range(Char::new('\u{D000}')..=Char::new('\u{F000}')), + CharRange::new_from_range(Char::new('\u{E000}')..=Char::new('\u{E000}')), + ], + vec![ + CharRange::new_from_range(Char::new('\u{10FFFF}')..=Char::new('\u{10FFFF}')), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{0}')), + ], + ]; + for inputs in cases { + assert_eq!( + compute_spanning_set_reference(&inputs), + SpanningSet::compute_spanning_set(&inputs), + "inputs: {inputs:?}" + ); + } + } } diff --git a/src/lib.rs b/src/lib.rs index 91493c7..702001c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,402 +1,1250 @@ +//! RegexSolver treats regular expressions as the **sets of strings they +//! match**, so you can intersect, subtract, compare, complement, repeat, and +//! enumerate them — and convert the result back into a regex pattern. +//! +//! # Quick start +//! +//! [`Term`] is the main entry point: it wraps either a [`RegularExpression`] +//! or a [`FastAutomaton`] and picks the cheaper representation for each +//! operation. +//! +//! ``` +//! use regexsolver::Term; +//! +//! let a: Term = "(ab|xy){2}".parse()?; +//! let b: Term = ".*xy".parse()?; +//! +//! // Which strings match BOTH patterns? Get the answer back as a regex: +//! let both = a.intersection([&b])?; +//! assert_eq!(both.to_pattern()?, "(ab|xy)xy"); +//! +//! // Matching is anchored (whole-string): +//! assert!(both.matches("abxy")?); +//! # Ok::<(), regexsolver::error::EngineError>(()) +//! ``` +//! +//! # Semantics +//! +//! RegexSolver implements **pure regular languages**, which differs from a +//! typical regex engine in two ways: matching is always **anchored** (a pattern +//! describes whole strings, so `abc` matches only `"abc"`), and `.` matches any +//! character including line feed. Constructs that a regular language can't +//! represent — backreferences, look-around, inline flags, and anchors/word +//! boundaries in non-redundant positions — return an [`EngineError`] rather +//! than being applied incorrectly. See the crate README for the full list. +//! +//! # Bounding execution +//! +//! Automaton operations can blow up on adversarial input, so a thread-local +//! [`ExecutionProfile`] can cap runtime and +//! state count and control implicit determinization; hitting a limit returns a +//! specific [`EngineError`] instead of hanging. +//! +//! # Modules +//! +//! Most users only need [`Term`]. The lower-level building blocks live in +//! [`regex`] (the parsed-pattern AST), [`fast_automaton`] (finite automata), +//! [`execution_profile`] (resource limits), [`cardinality`], and [`error`]. + +#![warn(missing_docs)] + use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, + borrow::{Borrow, Cow}, + collections::{HashMap, HashSet, VecDeque}, + fmt::Display, hash::BuildHasherDefault, + ops::{Bound, RangeBounds}, + str::FromStr, }; use cardinality::Cardinality; use error::EngineError; -use execution_profile::ThreadLocalParams; -use fast_automaton::FastAutomaton; -use nohash_hasher::NoHashHasher; +use fast_automaton::{FastAutomaton, GenerationOptions}; +#[cfg(feature = "parallel")] +use rayon::prelude::*; use regex::RegularExpression; use regex_charclass::{char::Char, irange::RangeSet}; -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; +use crate::execution_profile::ExecutionProfile; + +/// Cardinality of a language ([`Cardinality`]): a finite count, a count too +/// large for `u32`, or infinite. pub mod cardinality; +/// The [`EngineError`] type returned by fallible operations. pub mod error; +/// Resource limits: the thread-local [`ExecutionProfile`] governing timeouts, +/// state caps, and implicit determinization. pub mod execution_profile; +/// Finite automata: [`FastAutomaton`] and its building blocks (conditions, +/// spanning sets). pub mod fast_automaton; +/// The parsed-pattern AST: [`RegularExpression`]. pub mod regex; -pub mod tokenizer; -type IntMap = HashMap>>; -type IntSet = HashSet>>; -type Range = RangeSet; +/// Re-export of [`regex-charclass`](https://docs.rs/regex-charclass), the +/// crate behind [`CharRange`]: everything needed to build transition labels +/// by hand (`Char`, range sets) without adding a separately version-matched +/// dependency. +pub use regex_charclass; + +/// A no-op [`Hasher`](std::hash::Hasher) for integer keys that are already +/// well distributed, such as state ids: the key's value is used as the hash +/// directly. Only the integer key types it is implemented for can be hashed +/// with it; anything else does not compile. +#[derive(Clone, Copy, Debug, Default)] +pub struct NoHashHasher(u64, std::marker::PhantomData); + +macro_rules! impl_no_hash_hasher { + ($($int:ty => $write:ident),* $(,)?) => { + $( + impl std::hash::Hasher for NoHashHasher<$int> { + #[inline] + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, _: &[u8]) { + unreachable!("NoHashHasher hashes integer keys through their value"); + } + + #[inline] + fn $write(&mut self, n: $int) { + self.0 = n as u64; + } + } + )* + }; +} +impl_no_hash_hasher!(u32 => write_u32, u64 => write_u64, usize => write_usize); + +/// A hash map keyed by integer state ids using a no-op hasher. Internal. +pub(crate) type IntMap = HashMap>>; +/// A hash set of integer state ids using a no-op hasher (the hasher is fast +/// because state ids are already well-distributed small integers). Returned by +/// [`FastAutomaton::accept_states`] and related inspection methods. +pub type IntSet = HashSet>>; +/// A set of character ranges (the transition-label alphabet type), re-exported +/// from [`regex-charclass`](https://docs.rs/regex-charclass). +pub type CharRange = RangeSet; /// Represents a term that can be either a regular expression or a finite automaton. This term can be manipulated with a wide range of operations. /// -/// To put constraint and limitation on the execution of operations please refer to [`execution_profile::ExecutionProfile`]. -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +/// # Examples +/// ```rust +/// use regexsolver::Term; +/// use regexsolver::error::EngineError; +/// use regexsolver::fast_automaton::PathOrder; +/// +/// // Create terms from regex +/// let t1 = Term::from_pattern("abc.*")?; +/// let t2 = Term::from_pattern(".*xyz")?; +/// +/// // Concatenate +/// let concat = t1.concat(&[t2])?; +/// assert_eq!(concat.to_pattern()?, "abc.*xyz"); +/// +/// // Union +/// let union = t1.union(&[Term::from_pattern("fgh")?])?; +/// assert_eq!(union.to_pattern()?, "(abc.*|fgh)"); +/// +/// // Intersection +/// let inter = Term::from_pattern("(ab|xy){2}")? +/// .intersection(&[Term::from_pattern(".*xy")?])?; +/// assert_eq!(inter.to_pattern()?, "(ab|xy)xy"); +/// +/// // Difference +/// let diff = Term::from_pattern("a*")? +/// .difference(&Term::from_pattern("")?)?; +/// assert_eq!(diff.to_pattern()?, "a+"); +/// +/// // Repetition +/// let rep = Term::from_pattern("abc")? +/// .repeat(2..=4)?; +/// assert_eq!(rep.to_pattern()?, "(abc){2,4}"); +/// +/// // Analyze +/// assert_eq!(rep.length(), (Some(6), Some(12))); +/// assert!(!rep.is_empty()?); +/// +/// // Generate examples +/// let samples = Term::from_pattern("(x|y){1,3}")? +/// .generate_strings(5, 0, PathOrder::Interleave)?; +/// println!("Some matches: {:?}", samples); +/// +/// // Equivalence & subset +/// let a = Term::from_pattern("a+")?; +/// let b = Term::from_pattern("a*")?; +/// assert!(!a.equivalent(&b)?); +/// assert!(a.subset(&b)?); +/// # Ok::<(), EngineError>(()) +/// ``` +/// +/// To put constraint and limitation on the execution of operations please refer to [`ExecutionProfile`]. +/// +/// # Tracing +/// +/// The core operations on [`Term`], [`FastAutomaton`], and [`RegularExpression`] +/// are instrumented with [`tracing`](https://docs.rs/tracing) spans (mostly at +/// `debug` level). Install a [`tracing-subscriber`](https://docs.rs/tracing-subscriber) +/// (or any other `tracing` subscriber) in your application to observe them; if +/// no subscriber is installed, instrumentation has negligible overhead and +/// produces no output. +/// +/// # Equality +/// +/// `PartialEq`/`Eq` (`==`) compare the **underlying representation**, not the +/// language. Two terms that match exactly the same strings can compare +/// unequal (for example, an automaton and an equivalent regular expression, or +/// two differently-written regexes for the same language). To compare +/// *languages*, use [`equivalent`](Self::equivalent); for `self ⊆ other`, use +/// [`subset`](Self::subset). #[derive(Clone, PartialEq, Eq, Debug)] -#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))] +#[must_use = "terms are immutable; operations return a new term"] pub enum Term { - #[cfg_attr(feature = "serde", serde(rename = "regex"))] + /// The term is backed by a parsed regular-expression AST. RegularExpression(RegularExpression), - #[cfg_attr(feature = "serde", serde(rename = "fair"))] + /// The term is backed by a finite automaton. Automaton(FastAutomaton), } +/// The default term is the empty language (matches nothing), the identity for +/// [`union`](Term::union). See [`new_empty`](Term::new_empty). +impl Default for Term { + fn default() -> Self { + Term::new_empty() + } +} + +impl Display for Term { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Term::RegularExpression(regular_expression) => write!(f, "{regular_expression}"), + Term::Automaton(fast_automaton) => write!(f, "{fast_automaton}"), + } + } +} + +/// Parses a pattern into a [`Term`], so patterns can be built with +/// [`str::parse`]. +/// +/// # Examples +/// +/// ``` +/// use regexsolver::Term; +/// +/// let term: Term = ".*abc.*".parse().unwrap(); +/// ``` +impl FromStr for Term { + type Err = EngineError; + + fn from_str(pattern: &str) -> Result { + Term::from_pattern(pattern) + } +} + +impl From for Term { + fn from(regex: RegularExpression) -> Self { + Term::RegularExpression(regex) + } +} + +impl From for Term { + fn from(automaton: FastAutomaton) -> Self { + Term::Automaton(automaton) + } +} + impl Term { - /// Create a term based on the given pattern. + /// `Term` operations manage the underlying representation themselves, so + /// the determinizations they perform are by definition explicit: + /// they run with the profile's `implicit_determinization` setting + /// re-enabled (that knob targets direct [`FastAutomaton`] usage). The + /// rest of the profile is preserved. + fn run_with_implicit_determinization(f: impl FnOnce() -> R) -> R { + ExecutionProfile::get() + .with_implicit_determinization(true) + .apply(f) + } + + /// Creates a term that matches the empty language. + pub fn new_empty() -> Self { + Term::RegularExpression(RegularExpression::new_empty()) + } + + /// Creates a term that matches all possible strings. + pub fn new_total() -> Self { + Term::RegularExpression(RegularExpression::new_total()) + } + + /// Creates a term that only matches the empty string `""`. + pub fn new_empty_string() -> Self { + Term::RegularExpression(RegularExpression::new_empty_string()) + } + + /// Parses and simplifies the provided pattern and returns a new [`Term`] holding the resulting [`RegularExpression`]. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term = Term::from_regex(".*abc.*").unwrap(); + /// let term = Term::from_pattern(".*abc.*").unwrap(); /// ``` - pub fn from_regex(regex: &str) -> Result { - Ok(Term::RegularExpression(RegularExpression::new(regex)?)) + pub fn from_pattern(pattern: &str) -> Result { + Ok(Term::RegularExpression(RegularExpression::new(pattern)?)) + } + + /// Creates a new `Term` holding the provided [`RegularExpression`]. + pub fn from_regex(regex: RegularExpression) -> Self { + Term::RegularExpression(regex) } - /// Compute the union of the given collection of terms. - /// Returns the resulting term. + /// Creates a new `Term` holding the provided [`FastAutomaton`]. + pub fn from_automaton(automaton: FastAutomaton) -> Self { + Term::Automaton(automaton) + } + + /// Computes the concatenation of the given terms. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term1 = Term::from_regex("abc").unwrap(); - /// let term2 = Term::from_regex("de").unwrap(); - /// let term3 = Term::from_regex("fghi").unwrap(); + /// let term1 = Term::from_pattern("abc").unwrap(); + /// let term2 = Term::from_pattern("d.").unwrap(); + /// let term3 = Term::from_pattern(".*").unwrap(); /// - /// let union = term1.union(&[term2, term3]).unwrap(); + /// let concat = term1.concat([&term2, &term3]).unwrap(); /// - /// if let Term::RegularExpression(regex) = union { - /// assert_eq!("(abc|de|fghi)", regex.to_string()); - /// } + /// assert_eq!("abcd.+", concat.to_pattern().unwrap()); /// ``` - pub fn union(&self, terms: &[Term]) -> Result { - Self::check_number_of_terms(terms)?; - + #[tracing::instrument(level = "debug", skip_all)] + pub fn concat( + &self, + terms: impl IntoIterator>, + ) -> Result { let mut return_regex = RegularExpression::new_empty(); let mut return_automaton = FastAutomaton::new_empty(); + let mut has_automaton = false; match self { Term::RegularExpression(regular_expression) => { - return_regex = regular_expression.clone(); + return_regex = regular_expression.clone() } Term::Automaton(fast_automaton) => { + has_automaton = true; return_automaton = fast_automaton.clone(); } } - for operand in terms { - match operand { - Term::RegularExpression(regex) => { - return_regex = return_regex.union(regex); - if return_regex.is_total() { - return Ok(Term::RegularExpression(RegularExpression::new_total())); + for term in terms { + let term = term.borrow(); + if has_automaton { + return_automaton = return_automaton.concat(term.to_automaton()?.as_ref())?; + } else { + match term { + Term::RegularExpression(regular_expression) => { + return_regex = return_regex.concat(regular_expression, true); } - } - Term::Automaton(automaton) => { - return_automaton = return_automaton.union(automaton)?; - if return_automaton.is_total() { - return Ok(Term::RegularExpression(RegularExpression::new_total())); + Term::Automaton(fast_automaton) => { + has_automaton = true; + return_automaton = return_regex.to_automaton()?.concat(fast_automaton)?; } } } } - if return_automaton.is_empty() { + if !has_automaton { Ok(Term::RegularExpression(return_regex)) } else { - if !return_regex.is_empty() { - return_automaton = return_automaton.union(&return_regex.to_automaton()?)?; - } - - if let Some(regex) = return_automaton.to_regex() { - Ok(Term::RegularExpression(regex)) - } else { - Ok(Term::Automaton(return_automaton)) - } + Ok(Term::Automaton(return_automaton)) } } - /// Compute the intersection of the given collection of terms. - /// Returns the resulting term. + /// Computes the union of the given terms. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term1 = Term::from_regex("(abc|de){2}").unwrap(); - /// let term2 = Term::from_regex("de.*").unwrap(); - /// let term3 = Term::from_regex(".*abc").unwrap(); + /// let term1 = Term::from_pattern("abc").unwrap(); + /// let term2 = Term::from_pattern("de").unwrap(); + /// let term3 = Term::from_pattern("fghi").unwrap(); /// - /// let intersection = term1.intersection(&[term2, term3]).unwrap(); + /// let union = term1.union([&term2, &term3]).unwrap(); /// - /// if let Term::RegularExpression(regex) = intersection { - /// assert_eq!("deabc", regex.to_string()); - /// } + /// assert_eq!("(abc|de|fghi)", union.to_pattern().unwrap()); /// ``` - pub fn intersection(&self, terms: &[Term]) -> Result { - Self::check_number_of_terms(terms)?; - let mut return_automaton = self.get_automaton()?; - for term in terms { - let automaton = term.get_automaton()?; - return_automaton = Cow::Owned(return_automaton.intersection(&automaton)?); - if return_automaton.is_empty() { - return Ok(Term::RegularExpression(RegularExpression::new_empty())); + #[tracing::instrument(level = "debug", skip_all)] + pub fn union( + &self, + terms: impl IntoIterator>, + ) -> Result { + let terms: Vec<_> = terms.into_iter().collect(); + let terms: Vec<&Term> = terms.iter().map(Borrow::borrow).collect(); + + let mut has_automaton = matches!(self, Term::Automaton(_)); + if !has_automaton { + for term in &terms { + if matches!(term, Term::Automaton(_)) { + has_automaton = true; + break; + } } } - if let Some(regex) = return_automaton.to_regex() { - Ok(Term::RegularExpression(regex)) + if has_automaton { + let parallel = cfg!(feature = "parallel") && terms.len() > 3; + + let automaton_list = self.get_automata(&terms, parallel)?; + + let automaton_list = automaton_list.iter().map(AsRef::as_ref).collect::>(); + + #[cfg(feature = "parallel")] + let return_automaton = if parallel { + FastAutomaton::union_all_par(automaton_list) + } else { + FastAutomaton::union_all(automaton_list) + }?; + #[cfg(not(feature = "parallel"))] + let return_automaton = FastAutomaton::union_all(automaton_list)?; + + Ok(Term::Automaton(return_automaton)) } else { - Ok(Term::Automaton(return_automaton.into_owned())) + let regexes_list = self.get_regexes(&terms)?; + + let regexes_list = regexes_list.iter().map(AsRef::as_ref).collect::>(); + + Ok(Term::RegularExpression(RegularExpression::union_all( + regexes_list, + ))) } } - /// Compute the subtraction/difference of the two given terms. - /// Returns the resulting term. + /// Computes the intersection of the given terms. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term1 = Term::from_regex("(abc|de)").unwrap(); - /// let term2 = Term::from_regex("de").unwrap(); + /// let term1 = Term::from_pattern("(abc|de){2}").unwrap(); + /// let term2 = Term::from_pattern("de.*").unwrap(); + /// let term3 = Term::from_pattern(".*abc").unwrap(); /// - /// let subtraction = term1.subtraction(&term2).unwrap(); + /// let intersection = term1.intersection([&term2, &term3]).unwrap(); /// - /// if let Term::RegularExpression(regex) = subtraction { - /// assert_eq!("abc", regex.to_string()); - /// } + /// assert_eq!("deabc", intersection.to_pattern().unwrap()); /// ``` - pub fn subtraction(&self, subtrahend: &Term) -> Result { - let minuend_automaton = self.get_automaton()?; - let subtrahend_automaton = subtrahend.get_automaton()?; - let subtrahend_automaton = - Self::determinize_subtrahend(&minuend_automaton, &subtrahend_automaton)?; - let return_automaton = minuend_automaton.subtraction(&subtrahend_automaton)?; + #[tracing::instrument(level = "debug", skip_all)] + pub fn intersection( + &self, + terms: impl IntoIterator>, + ) -> Result { + let terms: Vec<_> = terms.into_iter().collect(); + let terms: Vec<&Term> = terms.iter().map(Borrow::borrow).collect(); + + let parallel = cfg!(feature = "parallel") && terms.len() > 3; + + let automaton_list = self.get_automata(&terms, parallel)?; - if let Some(regex) = return_automaton.to_regex() { - Ok(Term::RegularExpression(regex)) + let automaton_list = automaton_list.iter().map(AsRef::as_ref).collect::>(); + + #[cfg(feature = "parallel")] + let return_automaton = if terms.len() > 3 { + FastAutomaton::intersection_all_par(automaton_list) } else { + FastAutomaton::intersection_all(automaton_list) + }?; + #[cfg(not(feature = "parallel"))] + let return_automaton = FastAutomaton::intersection_all(automaton_list)?; + + Ok(Term::Automaton(return_automaton)) + } + + /// Computes the difference between `self` and `other`. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// let term1 = Term::from_pattern("(abc|de)").unwrap(); + /// let term2 = Term::from_pattern("de").unwrap(); + /// + /// let difference = term1.difference(&term2).unwrap(); + /// + /// assert_eq!("abc", difference.to_pattern().unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic(), other_deterministic = other.is_deterministic()))] + pub fn difference(&self, other: &Term) -> Result { + Self::run_with_implicit_determinization(|| { + let minuend_automaton = self.to_automaton()?; + let subtrahend_automaton = other.to_automaton()?; + // `FastAutomaton::difference` determinizes the subtrahend itself. + let return_automaton = minuend_automaton.difference(&subtrahend_automaton)?; + Ok(Term::Automaton(return_automaton)) + }) + } + + /// Computes the complement of `self`. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// let term = Term::from_pattern("(abc|de)").unwrap(); + /// + /// let complement = term.complement().unwrap(); + /// + /// assert!(term.intersection(&[complement.clone()]).unwrap().is_empty().unwrap()); + /// assert!(term.union(&[complement]).unwrap().is_total().unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic()))] + pub fn complement(&self) -> Result { + Self::run_with_implicit_determinization(|| { + // `FastAutomaton::complement` determinizes `self` itself. + let mut automaton = self.to_automaton()?.into_owned(); + automaton.complement()?; + + Ok(Term::Automaton(automaton)) + }) + } + + /// Computes the repetition of the current term over the given range of + /// counts. + /// + /// An unbounded end (`n..`) means unlimited repetition; an unset start + /// (`..n` or `..=n`) means zero. Exclusive bounds are normalized to inclusive. + /// A range containing no count at all (`0..0`, `3..3`, `5..2`) yields the + /// empty language. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// let term = Term::from_pattern("abc").unwrap(); + /// + /// assert_eq!("(abc)+", term.repeat(1..).unwrap().to_pattern().unwrap()); + /// assert_eq!("(abc){3,5}", term.repeat(3..=5).unwrap().to_pattern().unwrap()); + /// assert_eq!("(abc){3,5}", term.repeat(3..6).unwrap().to_pattern().unwrap()); + /// assert_eq!("(abc){0,2}", term.repeat(..=2).unwrap().to_pattern().unwrap()); + /// assert!(term.repeat(0..0).unwrap().is_empty().unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic(), min = tracing::field::Empty, max = tracing::field::Empty))] + pub fn repeat(&self, range: impl RangeBounds) -> Result { + let mut min = match range.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n.saturating_add(1), + Bound::Unbounded => 0, + }; + let max_opt = match range.end_bound() { + Bound::Included(&n) => Some(n), + Bound::Excluded(&n) => Some(n.saturating_sub(1)), + Bound::Unbounded => None, + }; + if matches!(range.end_bound(), Bound::Excluded(&0)) { + min = min.max(1); + } + let span = tracing::Span::current(); + span.record("min", min); + span.record("max", tracing::field::debug(max_opt)); + match self { + Term::RegularExpression(regular_expression) => Ok(Term::RegularExpression( + regular_expression.repeat(min, max_opt), + )), + Term::Automaton(fast_automaton) => { + let repeat_automaton = fast_automaton.repeat(min, max_opt)?; + Ok(Term::Automaton(repeat_automaton)) + } } } - /// See [`Self::subtraction`]. - #[inline] - pub fn difference(&self, subtrahend: &Term) -> Result { - self.subtraction(subtrahend) + /// Generates up to `limit` distinct strings matched by the term under the + /// given [`GenerationOptions`], skipping the first `offset` strings. + /// + /// `options` combines two independent axes — how paths are scheduled and + /// how the strings within them are ordered — plus an optional charset + /// and length bounds. + /// [`PathOrder::Sweep`](fast_automaton::PathOrder::Sweep) walks the + /// language one path at a time, + /// [`Interleave`](fast_automaton::PathOrder::Interleave) spreads the + /// strings over the shapes the pattern allows, and + /// [`PathOrder::Shuffled`](fast_automaton::PathOrder::Shuffled) + /// additionally draws which same-length shapes come first by a seed + /// ([`GenerationOptions::with_seed`]); + /// [`CharacterOrder::Ascending`](fast_automaton::CharacterOrder::Ascending) + /// yields each path's smallest strings first, while + /// [`CharacterOrder::Shuffled`](fast_automaton::CharacterOrder::Shuffled) + /// draws them through a seeded permutation. Both axes shuffled is what + /// you want to derive test cases from a pattern: coverage of every shape, + /// with strings that look like real inputs, reproducible and pageable. An + /// axis can be passed on its own wherever options are expected, and so + /// can a `(PathOrder, CharacterOrder)` pair. + /// + /// Strings are only guaranteed to be distinct **within a single call**: + /// the offset fast-skips by counting paths, and in a non-deterministic + /// automaton the same string can be reached through several paths, so + /// calls with different offsets may repeat strings (or skip some). The + /// enumeration order also depends on the automaton's structure, so + /// offsets are only consistent across calls made on the same term with + /// the same options. + /// + /// [`GenerationOptions::with_min_length`] and + /// [`with_max_length`](GenerationOptions::with_max_length) confine the + /// enumeration to a band of string lengths — without a max, a deep + /// `offset` into a looping language (`.*`) pages into arbitrarily long + /// strings. Generation runs under the active + /// [`ExecutionProfile`]: its + /// timeout aborts with [`EngineError::OperationTimeOutError`]. + /// + /// For pagination without repetition or skipped strings, make the term deterministic once and generate + /// from it. To check if a term is deterministic use [`is_deterministic`](Self::is_deterministic). + /// To determinize run [`determinize`](Self::determinize). + /// + /// # Examples + /// + /// ``` + /// use regexsolver::{CharRange, Term, fast_automaton::{CharacterOrder, GenerationOptions, PathOrder}}; + /// use regexsolver::regex_charclass::char::Char; + /// + /// // Minimize once, then paginate with consistent offsets. + /// let term = Term::from_pattern("(abc|de){2}").unwrap().minimize().unwrap(); + /// + /// let batch = term.generate_strings(2, 0, PathOrder::Sweep).unwrap(); + /// assert_eq!(2, batch.len()); // ["dede", "deabc"] + /// + /// let batch = term.generate_strings(2, 2, PathOrder::Sweep).unwrap(); + /// assert_eq!(2, batch.len()); // ["abcde", "abcabc"] + /// + /// // The sweep works through one path at a time, so a limit spent on + /// // `.*abc.*` never leaves the strings starting with `abc`. + /// let term = Term::from_pattern(".*abc.*").unwrap().minimize().unwrap(); + /// + /// let batch = term.generate_strings(5, 0, PathOrder::Sweep).unwrap(); + /// assert!(batch.iter().all(|s| s.starts_with("abc"))); + /// + /// // Interleaving covers the pattern instead. + /// let batch = term.generate_strings(5, 0, PathOrder::Interleave).unwrap(); + /// assert!(batch.iter().any(|s| !s.starts_with("abc"))); + /// + /// // Shuffling both axes covers it with arbitrary-looking strings; the + /// // fixed seed keeps them reproducible. + /// let options = GenerationOptions::from((PathOrder::Shuffled, CharacterOrder::Shuffled)) + /// .with_seed(42); + /// let batch = term.generate_strings(5, 0, options.clone()).unwrap(); + /// assert_eq!(batch, term.generate_strings(5, 0, options).unwrap()); + /// + /// // A charset keeps generation to the characters you can use. + /// let printable = CharRange::new_from_range(Char::new(' ')..=Char::new('~')); + /// let options = GenerationOptions::from(PathOrder::Interleave).with_charset(printable); + /// + /// let batch = term.generate_strings(5, 0, options).unwrap(); + /// assert!(batch.iter().all(|s| s.chars().all(|c| c.is_ascii_graphic() || c == ' '))); + /// ``` + #[tracing::instrument(level = "debug", skip(self, options), fields(self_deterministic = self.is_deterministic(), limit = limit, offset = offset))] + pub fn generate_strings( + &self, + limit: usize, + offset: usize, + options: impl Into, + ) -> Result, EngineError> { + self.to_automaton()? + .generate_strings(limit, offset, options) } - /// Returns the Details of the given term. + /// Returns a lazy iterator over the strings matched by the term under the + /// given [`GenerationOptions`], fetched in batches behind the scenes so you + /// can stop early without choosing a limit up front. + /// + /// The underlying deterministic automaton is computed once at construction time, not on + /// every batch. Each item is a `Result`: a construction or generation error + /// (e.g. a timeout from the active [`ExecutionProfile`]) surfaces as an + /// `Err`, after which the iterator ends. /// - /// # Example: + /// # Examples /// /// ``` - /// use regexsolver::{Term, cardinality::Cardinality}; + /// use regexsolver::{Term, fast_automaton::GenerationOptions}; /// - /// let term = Term::from_regex("(abc|de)").unwrap(); + /// let term = Term::from_pattern("(abc|de){2}").unwrap().minimize().unwrap(); /// - /// let details = term.get_details().unwrap(); + /// // Take the first three matches lazily. + /// let first_three = term + /// .iter_strings(GenerationOptions::new()) + /// .take(3) + /// .collect::, _>>() + /// .unwrap(); + /// assert_eq!(3, first_three.len()); /// - /// assert_eq!(Some(Cardinality::Integer(2)), *details.get_cardinality()); - /// assert_eq!((Some(2), Some(3)), *details.get_length()); - /// assert!(!details.is_empty()); - /// assert!(!details.is_total()); + /// // Length bounds keep a lazy walk of an infinite language finite: + /// // without a max, this iterator never ends. + /// let term = Term::from_pattern("(ab)*").unwrap().minimize().unwrap(); + /// + /// let options = GenerationOptions::new().with_min_length(3).with_max_length(8); + /// let band = term + /// .iter_strings(options) + /// .collect::, _>>() + /// .unwrap(); + /// assert_eq!(vec!["abab", "ababab", "abababab"], band); /// ``` - pub fn get_details(&self) -> Result { - match self { - Term::RegularExpression(regex) => Ok(Details { - cardinality: Some(regex.get_cardinality()), - length: regex.get_length(), - empty: regex.is_empty(), - total: regex.is_total(), - }), - Term::Automaton(automaton) => Ok(Details { - cardinality: automaton.get_cardinality(), - length: automaton.get_length(), - empty: automaton.is_empty(), - total: automaton.is_total(), - }), + pub fn iter_strings(&self, options: impl Into) -> StringGenerator<'_> { + let options = options.into(); + match self.to_deterministic_automaton() { + Ok(automaton) => StringGenerator { + automaton: Some(automaton), + pending_error: None, + offset: 0, + options, + buffer: VecDeque::new(), + }, + Err(e) => StringGenerator { + automaton: None, + pending_error: Some(e), + offset: 0, + options, + buffer: VecDeque::new(), + }, } } - /// Generate strings matched by the given term. + /// Returns an equivalent term backed by a deterministic automaton. /// - /// # Example: + /// Already-deterministic terms are returned as-is. + /// + /// Determinization is always explicit, so it runs regardless of the + /// profile's [`implicit_determinization`](crate::execution_profile::ExecutionProfileBuilder::implicit_determinization) + /// setting. + /// + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term = Term::from_regex("(abc|de){2}").unwrap(); + /// let term = Term::from_pattern(".*abc").unwrap(); + /// assert!(!term.is_deterministic()); /// - /// let strings = term.generate_strings(3).unwrap(); + /// let dfa = term.determinize().unwrap(); + /// assert!(dfa.is_deterministic()); + /// assert!(term.equivalent(&dfa).unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic()))] + pub fn determinize(&self) -> Result { + let automaton = self.to_automaton()?; + let determinized = automaton.determinize()?.into_owned(); + Ok(Term::Automaton(determinized)) + } + + /// Returns an equivalent term backed by the minimal deterministic + /// automaton. + /// + /// # Examples /// - /// assert_eq!(3, strings.len()); // ex: ["deabc", "dede", "abcde"] /// ``` - pub fn generate_strings(&self, count: usize) -> Result, EngineError> { - Ok(self - .get_automaton()? - .generate_strings(count)? - .into_iter() - .collect()) + /// use regexsolver::Term; + /// + /// let term = Term::from_pattern(".*abc").unwrap(); + /// let minimal = term.minimize().unwrap(); + /// assert!(minimal.is_minimal()); + /// assert!(term.equivalent(&minimal).unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic(), self_minimal = self.is_minimal()))] + pub fn minimize(&self) -> Result { + Self::run_with_implicit_determinization(|| { + let mut automaton = self.to_automaton()?.into_owned(); + automaton.minimize()?; + Ok(Term::Automaton(automaton)) + }) } - /// Compute if the two given terms are equivalent. + /// Returns `true` if both terms accept the same language. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term1 = Term::from_regex("(abc|de)").unwrap(); - /// let term2 = Term::from_regex("(abc|de)*").unwrap(); + /// let term1 = Term::from_pattern("(abc|de)").unwrap(); + /// let term2 = Term::from_pattern("(abc|de)*").unwrap(); /// - /// assert!(!term1.are_equivalent(&term2).unwrap()); + /// assert!(!term1.equivalent(&term2).unwrap()); /// ``` - pub fn are_equivalent(&self, that: &Term) -> Result { - if self == that { + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic(), other_deterministic = other.is_deterministic()))] + pub fn equivalent(&self, other: &Term) -> Result { + if self == other { return Ok(true); } - let automaton_1 = self.get_automaton()?; - let automaton_2 = that.get_automaton()?; - automaton_1.is_equivalent_of(&automaton_2) + Self::run_with_implicit_determinization(|| { + let automaton_1 = self.to_automaton()?; + let automaton_2 = other.to_automaton()?; + automaton_1.equivalent(&automaton_2) + }) } - /// Compute if the first term is a subset of the second one. + /// Returns `true` if all strings matched by the current term are also matched by the given term. /// - /// # Example: + /// # Examples /// /// ``` /// use regexsolver::Term; /// - /// let term1 = Term::from_regex("de").unwrap(); - /// let term2 = Term::from_regex("(abc|de)").unwrap(); + /// let term1 = Term::from_pattern("de").unwrap(); + /// let term2 = Term::from_pattern("(abc|de)").unwrap(); /// - /// assert!(term1.is_subset_of(&term2).unwrap()); + /// assert!(term1.subset(&term2).unwrap()); /// ``` - pub fn is_subset_of(&self, that: &Term) -> Result { - if self == that { + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic(), other_deterministic = other.is_deterministic()))] + pub fn subset(&self, other: &Term) -> Result { + if self == other { return Ok(true); } - let automaton_1 = self.get_automaton()?; - let automaton_2 = that.get_automaton()?; - automaton_1.is_subset_of(&automaton_2) + Self::run_with_implicit_determinization(|| { + let automaton_1 = self.to_automaton()?; + let automaton_2 = other.to_automaton()?; + automaton_1.subset(&automaton_2) + }) + } + + /// Returns `true` if the term matches the given string. + /// + /// Matching is **anchored** (full-string), consistent with the rest of the + /// crate: the whole input must be accepted, not just a substring. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// let term = Term::from_pattern("abc.*").unwrap(); + /// + /// assert!(term.matches("abcdef").unwrap()); + /// assert!(!term.matches("xyzabc").unwrap()); + /// ``` + #[tracing::instrument(level = "debug", skip(self, input), fields(self_deterministic = self.is_deterministic(), input_len = input.len()))] + pub fn matches(&self, input: &str) -> Result { + Ok(self.to_automaton()?.is_match(input)) + } + + /// Returns `true` if the term matches the empty language (no strings at all). + /// + /// Note: the empty language is distinct from the language containing only + /// the empty string `""`. Use [`is_empty_string`](Self::is_empty_string) to + /// test for the latter. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// assert!(Term::new_empty().is_empty().unwrap()); + /// assert!(!Term::new_empty_string().is_empty().unwrap()); // matches "" + /// assert!(!Term::from_pattern("abc").unwrap().is_empty().unwrap()); + /// ``` + pub fn is_empty(&self) -> Result { + Ok(match self { + Term::RegularExpression(regex) => regex.is_empty(), + Term::Automaton(automaton) => automaton.is_empty(), + }) } - fn check_number_of_terms(terms: &[Term]) -> Result<(), EngineError> { - let number_of_terms = terms.len() + 1; - let max_number_of_terms = ThreadLocalParams::get_max_number_of_terms(); - if number_of_terms > max_number_of_terms { - Err(EngineError::TooMuchTerms( - max_number_of_terms, - number_of_terms, - )) + /// Returns `true` if the term matches all possible strings. + pub fn is_total(&self) -> Result { + if let Term::RegularExpression(regex) = self + && regex.is_total() + { + return Ok(true); + } + let automaton = self.to_automaton()?; + if automaton.is_total() { + Ok(true) + } else if automaton.is_deterministic() { + Ok(false) } else { - Ok(()) + Ok(automaton.determinize()?.is_total()) } } - fn determinize_subtrahend<'a>( - minuend: &FastAutomaton, - subtrahend: &'a FastAutomaton, - ) -> Result, EngineError> { - if subtrahend.is_determinitic() { - Ok(Cow::Borrowed(subtrahend)) - } else if !minuend.is_cyclic() && subtrahend.is_cyclic() { - Ok(Cow::Owned(minuend.intersection(subtrahend)?.determinize()?)) - } else { - Ok(Cow::Owned(subtrahend.determinize()?)) + /// Returns `true` if the term matches only the empty string `""`. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// assert!(Term::new_empty_string().is_empty_string().unwrap()); + /// assert!(!Term::new_empty().is_empty_string().unwrap()); + /// assert!(!Term::from_pattern("a*").unwrap().is_empty_string().unwrap()); + /// ``` + pub fn is_empty_string(&self) -> Result { + Ok(match self { + Term::RegularExpression(regex) => regex.is_empty_string(), + Term::Automaton(automaton) => automaton.is_empty_string(), + }) + } + + /// Returns `true` if the term is *already backed by* a deterministic + /// automaton. + /// + /// A deterministic automaton has one path per accepted string. + /// + /// To determinize a term call [`determinize`](Self::determinize). + #[must_use] + pub fn is_deterministic(&self) -> bool { + match self { + Term::RegularExpression(_) => false, + Term::Automaton(automaton) => automaton.is_deterministic(), } } - fn get_automaton(&self) -> Result, EngineError> { + /// Returns `true` if the term is *already backed by* the minimal + /// deterministic automaton. + /// + /// The minimal deterministic automaton of a given language is unique. + /// + /// To minimize a term call [`minimize`](Self::minimize). + #[must_use] + pub fn is_minimal(&self) -> bool { + match self { + Term::RegularExpression(_) => false, + Term::Automaton(automaton) => automaton.is_minimal(), + } + } + + /// Returns the minimum and maximum length of matched strings. + /// + /// `None` for the minimum means the language is empty (no strings are + /// matched). `None` for the maximum means the language is infinite + /// (unbounded match length). + #[must_use] + pub fn length(&self) -> (Option, Option) { + match self { + Term::RegularExpression(regex) => regex.length(), + Term::Automaton(automaton) => automaton.length(), + } + } + + /// Returns the cardinality of the term (the number of distinct matched strings). + /// + /// The exact count is represented as `u32`. If the exact count exceeds + /// `u32::MAX`, the result is `Cardinality::BigInteger` rather than a + /// truncated value. Infinite languages return `Cardinality::Infinite`. + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic()))] + pub fn cardinality(&self) -> Result, EngineError> { + Self::run_with_implicit_determinization(|| self.to_automaton()?.cardinality()) + } + + /// Returns `true` if the term matches a finite number of strings. + /// + /// A finite language is one with no unbounded repetition (`*`, `+`, ...). + /// Convenience over [`cardinality`](Self::cardinality) when only the + /// finite/infinite distinction matters. + /// + /// # Examples + /// + /// ``` + /// use regexsolver::Term; + /// + /// assert!(Term::from_pattern("(ab|c){2}").unwrap().is_finite().unwrap()); + /// assert!(!Term::from_pattern("a+").unwrap().is_finite().unwrap()); + /// ``` + pub fn is_finite(&self) -> Result { + Ok(!matches!(self.cardinality()?, Cardinality::Infinite)) + } + + /// Converts the term to a [`FastAutomaton`]. + /// + /// Returns a [`Cow`]: borrows the automaton when the term is already + /// automaton-backed, and allocates a new one when converting from a + /// [`RegularExpression`]. + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic()))] + pub fn to_automaton(&self) -> Result, EngineError> { Ok(match self { Term::RegularExpression(regex) => Cow::Owned(regex.to_automaton()?), Term::Automaton(automaton) => Cow::Borrowed(automaton), }) } -} -/// Represents details about a [Term]. -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Clone, PartialEq, Eq, Debug)] -#[cfg_attr(feature = "serde", serde(tag = "type", rename = "details"))] -pub struct Details { - cardinality: Option>, - length: (Option, Option), - empty: bool, - total: bool, -} + fn to_deterministic_automaton(&self) -> Result, EngineError> { + let automaton = self.to_automaton()?; + if automaton.is_deterministic() { + return Ok(automaton); + } + Ok(Cow::Owned(automaton.determinize()?.into_owned())) + } -impl Details { - /// Return the number of unique strings matched. - pub fn get_cardinality(&self) -> &Option> { - &self.cardinality + /// Converts the term to a [`RegularExpression`]. + /// + /// Returns a [`Cow`]: borrows the expression when the term is already + /// regex-backed, and allocates a new one when converting from a + /// [`FastAutomaton`] via state elimination. + #[tracing::instrument(level = "debug", skip_all, fields(self_deterministic = self.is_deterministic()))] + pub fn to_regex(&self) -> Result, EngineError> { + Ok(match self { + Term::RegularExpression(regex) => Cow::Borrowed(regex), + Term::Automaton(automaton) => Cow::Owned(automaton.to_regex()?), + }) + } + + /// Converts the term to a regular expression pattern. + pub fn to_pattern(&self) -> Result { + Ok(self.to_regex()?.to_string()) } - /// Return the minimum and the maximum length of matched strings. - pub fn get_length(&self) -> &(Option, Option) { - &self.length + fn get_automata<'a>( + &'a self, + terms: &[&'a Term], + parallel: bool, + ) -> Result>, EngineError> { + let mut automaton_list = Vec::with_capacity(terms.len() + 1); + automaton_list.push(self.to_automaton()?); + + #[cfg(feature = "parallel")] + let mut terms_automata = if parallel { + let execution_profile = ExecutionProfile::get(); + terms + .par_iter() + .map(|a| execution_profile.apply(|| a.to_automaton())) + .collect::, _>>() + } else { + terms + .iter() + .map(|a| a.to_automaton()) + .collect::, _>>() + }?; + #[cfg(not(feature = "parallel"))] + let mut terms_automata = { + let _ = parallel; + terms + .iter() + .map(|a| a.to_automaton()) + .collect::, EngineError>>()? + }; + automaton_list.append(&mut terms_automata); + + Ok(automaton_list) } - /// Return `true` if it does not match any string. - pub fn is_empty(&self) -> bool { - self.empty + fn get_regexes<'a>( + &'a self, + terms: &[&'a Term], + ) -> Result>, EngineError> { + let mut regex_list = Vec::with_capacity(terms.len() + 1); + regex_list.push(self.to_regex()?); + for term in terms { + regex_list.push(term.to_regex()?); + } + Ok(regex_list) } +} + +/// Lazy iterator over the strings matched by a [`Term`], created by +/// [`Term::iter_strings`]. +/// +/// The underlying automaton is computed once at construction. Yields +/// `Result`: errors (from construction or generation) +/// are surfaced as `Err` items, after which the iterator ends. +#[derive(Debug)] +pub struct StringGenerator<'a> { + automaton: Option>, + pending_error: Option, + offset: usize, + options: GenerationOptions, + buffer: VecDeque, +} + +// Every terminal state (language exhausted, or error yielded) drops the +// automaton, after which `next` returns `None` forever. +impl std::iter::FusedIterator for StringGenerator<'_> {} + +impl Iterator for StringGenerator<'_> { + type Item = Result; - /// Return `true` if it match all possible strings. - pub fn is_total(&self) -> bool { - self.total + fn next(&mut self) -> Option { + const BATCH: usize = 32; + + if let Some(s) = self.buffer.pop_front() { + return Some(Ok(s)); + } + if let Some(e) = self.pending_error.take() { + return Some(Err(e)); + } + let automaton = self.automaton.as_ref()?; + match automaton.generate(BATCH, self.offset, &self.options) { + Ok(batch) => { + if batch.len() < BATCH { + self.automaton = None; + } + self.offset += batch.len(); + self.buffer.extend(batch); + self.buffer.pop_front().map(Ok) + } + Err(e) => { + self.automaton = None; + Some(Err(e)) + } + } } } #[cfg(test)] mod tests { + use crate::fast_automaton::GenerationOptions; use crate::regex::RegularExpression; use super::*; + // A range containing no count at all (`0..0`, `3..3`, `5..2`) is the + // empty language, while a range containing exactly the count 0 (`0..=0`, + // `0..1`) is the empty-string language. + #[test] + #[allow(clippy::reversed_empty_ranges)] // deliberately empty ranges are the point + fn repeat_empty_ranges_yield_the_empty_language() { + let regex_term = Term::from_pattern("abc").unwrap(); + let automaton_term = regex_term.determinize().unwrap(); + assert!(matches!(automaton_term, Term::Automaton(..))); + + for term in [regex_term, automaton_term] { + // Ranges containing no count at all: the empty language. + assert!(term.repeat(0..0).unwrap().is_empty().unwrap()); + assert!(term.repeat(3..3).unwrap().is_empty().unwrap()); + assert!(term.repeat(5..2).unwrap().is_empty().unwrap()); + + // Ranges containing exactly the count 0: the empty-string language. + assert!(term.repeat(0..=0).unwrap().is_empty_string().unwrap()); + assert!(term.repeat(0..1).unwrap().is_empty_string().unwrap()); + } + } + + // Pins the intentional `Display` behavior: regex-backed terms render + // their pattern; automaton-backed terms render Graphviz DOT. Use + // `to_pattern` to obtain a parseable pattern for either kind. #[test] - fn test_details() -> Result<(), String> { - let regex1 = Term::from_regex("a").unwrap(); - let regex2 = Term::from_regex("b").unwrap(); + fn display_is_pattern_for_regexes_and_dot_for_automata() { + let regex_term = Term::from_pattern("(abc){2}").unwrap(); + assert_eq!("(abc){2}", regex_term.to_string()); - let details = regex1.intersection(&vec![regex2]); - assert!(details.is_ok()); + let automaton_term = regex_term.determinize().unwrap(); + assert!(matches!(automaton_term, Term::Automaton(..))); + assert!(automaton_term.to_string().starts_with("digraph")); + let reparsed: Term = automaton_term.to_pattern().unwrap().parse().unwrap(); + assert!(reparsed.equivalent(&automaton_term).unwrap()); + } + + // `to_pattern` (state elimination) can grow super-polynomially, so it + // must honor the execution deadline and fail with a timeout rather than + // run unbudgeted. + #[test] + fn to_pattern_honors_the_execution_deadline() { + let term = Term::from_pattern(".*abc.*def.*") + .unwrap() + .determinize() + .unwrap(); + + crate::execution_profile::ExecutionProfileBuilder::new() + .execution_timeout(0) + .build() + .run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + term.to_pattern().unwrap_err() + ); + }); + + // Without the 0ms deadline the very same conversion succeeds. + assert!(term.to_pattern().is_ok()); + } + + #[test] + fn test_complement() -> Result<(), String> { + let term = Term::from_pattern("(abc|de)").unwrap(); + + let complement = term.complement().unwrap(); + + assert!( + term.intersection([&complement]) + .unwrap() + .is_empty() + .unwrap() + ); + + println!("term: {}", term.to_automaton().unwrap().to_dot()); + + if let Term::Automaton(complement) = &complement { + println!("complement: {}", complement.to_dot()); + } + + let union = term.union(&[complement]).unwrap(); + if let Term::Automaton(union) = &union { + println!("{}", union.to_dot()); + let union = union.determinize().unwrap(); + println!("{}", union.to_dot()); + } + + assert!(union.is_total().unwrap()); + + Ok(()) + } + + #[test] + fn union_of_regex_with_complement_pattern_is_total() { + for pattern in ["(abc|de)", "a", "x*", "[0-9]{2,4}"] { + let term = Term::from_pattern(pattern).unwrap(); + let complement_pattern = term.complement().unwrap().to_pattern().unwrap(); + let complement = Term::from_pattern(&complement_pattern).unwrap(); + assert!(matches!(complement, Term::RegularExpression(..))); + + let union = term.union([&complement]).unwrap(); + assert!(matches!(union, Term::RegularExpression(..))); + assert!(union.is_total().unwrap(), "not total for {pattern}"); + assert_eq!( + ".*", + union.minimize().unwrap().to_pattern().unwrap(), + "wrong minimized pattern for {pattern}" + ); + } + } + + #[test] + fn test_intersection() -> Result<(), String> { + let regex1 = Term::from_pattern("a").unwrap(); + let regex2 = Term::from_pattern("b").unwrap(); + + let intersection = regex1.intersection(&[regex2]).unwrap(); + assert!(intersection.is_empty().unwrap()); + assert_eq!("[]", intersection.to_pattern().unwrap()); Ok(()) } #[test] - fn test_subtraction_1() -> Result<(), String> { - let regex1 = Term::from_regex("a*").unwrap(); - let regex2 = Term::from_regex("").unwrap(); + fn test_difference_1() -> Result<(), String> { + let regex1 = Term::from_pattern("a*").unwrap(); + let regex2 = Term::from_pattern("").unwrap(); - let result = regex1.subtraction(®ex2); + let result = regex1.difference(®ex2); assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!( - Term::RegularExpression(RegularExpression::new("a+").unwrap()), - result - ); + let result = result.unwrap().to_pattern().unwrap(); + assert_eq!("a+", result); Ok(()) } #[test] - fn test_subtraction_2() -> Result<(), String> { - let regex1 = Term::from_regex("x*").unwrap(); - let regex2 = Term::from_regex("(xxx)*").unwrap(); + fn test_difference_2() -> Result<(), String> { + let regex1 = Term::from_pattern("x*").unwrap(); + let regex2 = Term::from_pattern("(xxx)*").unwrap(); - let result = regex1.subtraction(®ex2); + let result = regex1.difference(®ex2); assert!(result.is_ok()); - let result = result.unwrap(); + let result = result.unwrap().to_regex().unwrap().into_owned(); assert_eq!( - Term::RegularExpression(RegularExpression::new("(xxx)*(x|xx)").unwrap()), - result + Term::RegularExpression(RegularExpression::new("x(x{3})*x?").unwrap()), + Term::RegularExpression(result) ); Ok(()) @@ -404,41 +1252,235 @@ mod tests { #[test] fn test_intersection_1() -> Result<(), String> { - let regex1 = Term::from_regex("a*").unwrap(); - let regex2 = Term::from_regex("b*").unwrap(); + let regex1 = Term::from_pattern("a*").unwrap(); + let regex2 = Term::from_pattern("b*").unwrap(); - let result = regex1.intersection(&vec![regex2]); + let result = regex1.intersection(&[regex2]); assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(Term::from_regex("").unwrap(), result); + let result = result.unwrap().to_pattern().unwrap(); + assert_eq!("", result); Ok(()) } #[test] fn test_intersection_2() -> Result<(), String> { - let regex1 = Term::from_regex("x*").unwrap(); - let regex2 = Term::from_regex("(xxx)*").unwrap(); + let regex1 = Term::from_pattern("x*").unwrap(); + let regex2 = Term::from_pattern("(xxx)*").unwrap(); - let result = regex1.intersection(&vec![regex2]); + let result = regex1.intersection(&[regex2]); assert!(result.is_ok()); - let result = result.unwrap(); + let result = result.unwrap().to_pattern().unwrap(); + assert_eq!("(x{3})*", result); + + Ok(()) + } + + #[test] + fn test_default_is_empty_language() { + assert!(Term::default().is_empty().unwrap()); + assert_eq!(Term::default(), Term::new_empty()); + } + + #[test] + fn test_iter_strings_exhaustive_matches_generate_strings() { + // A finite, deterministic term: lazy iteration must yield exactly the + // same multiset as a single large `generate_strings` call, with no + // duplicates or omissions across batch boundaries. + let term = Term::from_pattern("[A-Za-z0-9]") + .unwrap() + .minimize() + .unwrap(); + + let eager = term + .generate_strings(1000, 0, GenerationOptions::new()) + .unwrap(); + let lazy = term + .iter_strings(GenerationOptions::new()) + .collect::, _>>() + .unwrap(); + + assert_eq!(eager.len(), lazy.len()); + assert_eq!(eager, lazy); + assert_eq!(62, lazy.len()); + } + + #[test] + fn test_is_finite() { + assert!( + Term::from_pattern("(ab|c){2}") + .unwrap() + .is_finite() + .unwrap() + ); + assert!(!Term::from_pattern("a+").unwrap().is_finite().unwrap()); + } + + #[test] + fn test_matches_is_anchored() { + let term = Term::from_pattern("abc.*").unwrap(); + assert!(term.matches("abc").unwrap()); + assert!(term.matches("abcdef").unwrap()); + // Anchored: a prefix/suffix match is not enough. + assert!(!term.matches("xyzabc").unwrap()); + + let exact = Term::from_pattern("abc").unwrap(); + assert!(exact.matches("abc").unwrap()); + assert!(!exact.matches("abcd").unwrap()); + + // Works on an automaton-backed term too. + let automaton_backed = exact.intersection([&term]).unwrap(); + assert!(matches!(automaton_backed, Term::Automaton(_))); + assert!(automaton_backed.matches("abc").unwrap()); + assert!(!automaton_backed.matches("abcd").unwrap()); + + // The empty language matches nothing; the empty string matches only "". + assert!(!Term::new_empty().matches("").unwrap()); + assert!(Term::new_empty_string().matches("").unwrap()); + assert!(!Term::new_empty_string().matches("a").unwrap()); + } + + #[test] + fn test_from_str_and_from_conversions() { + // `FromStr` agrees with `from_pattern`. + let parsed: Term = "abc".parse().unwrap(); + assert_eq!(parsed, Term::from_pattern("abc").unwrap()); + + // Invalid patterns surface as parse errors (backreferences are not regular). + assert!(r"(a)\1".parse::().is_err()); + + // `From` / `From` match the explicit constructors. + let regex = RegularExpression::new("abc").unwrap(); + let from_into: Term = regex.clone().into(); + assert_eq!(from_into, Term::from_regex(regex)); + + let automaton = Term::from_pattern("abc") + .unwrap() + .to_automaton() + .unwrap() + .into_owned(); + let from_into: Term = automaton.clone().into(); + assert_eq!(from_into, Term::from_automaton(automaton)); + } + + #[test] + fn test_is_deterministic_and_determinize() { + // A pattern-backed term is never reported deterministic (NFA form). + let regex_term = Term::from_pattern("(abc|de){2}").unwrap(); + assert!(!regex_term.is_deterministic()); + + // `determinize` produces a deterministic, language-equivalent term. + let dfa = regex_term.determinize().unwrap(); + assert!(dfa.is_deterministic()); + assert!(regex_term.equivalent(&dfa).unwrap()); + + // Determinizing an already-deterministic term keeps it deterministic + // and equivalent. + let dfa2 = dfa.determinize().unwrap(); + assert!(dfa2.is_deterministic()); + assert!(dfa.equivalent(&dfa2).unwrap()); + } + + #[test] + fn test_is_minimal_and_minimize() { + // A pattern-backed term is never reported minimal. + let regex_term = Term::from_pattern("(abc|de){2}").unwrap(); + assert!(!regex_term.is_minimal()); + + // `minimize` produces a minimal, language-equivalent term. + let minimal = regex_term.minimize().unwrap(); + assert!(minimal.is_minimal()); + assert!(minimal.is_deterministic()); // minimal implies deterministic + assert!(regex_term.equivalent(&minimal).unwrap()); + } + + #[test] + fn test_eq_is_structural_not_language() { + // Same language, different representation: structurally unequal, but + // language-equivalent. `==` must not be mistaken for `equivalent`. + let regex_term = Term::from_pattern("(a|b)*").unwrap(); + let automaton_term = Term::from_automaton(regex_term.to_automaton().unwrap().into_owned()); + + assert_ne!(regex_term, automaton_term); + assert!(regex_term.equivalent(&automaton_term).unwrap()); + } + + #[test] + fn test_repeat_range_edges() { + let term = Term::from_pattern("abc").unwrap(); + + // Unbounded / unset bounds. + assert_eq!("(abc)*", term.repeat(..).unwrap().to_pattern().unwrap()); + assert_eq!("(abc){2,}", term.repeat(2..).unwrap().to_pattern().unwrap()); assert_eq!( - Term::RegularExpression(RegularExpression::new("(x{3})*").unwrap()), - result + "(abc){0,2}", + term.repeat(..3).unwrap().to_pattern().unwrap() ); - Ok(()) + // Zero repetitions is the empty string. + assert!(term.repeat(0..=0).unwrap().is_empty_string().unwrap()); + + // A range whose normalized max < min denotes no valid repetition count, + // so the simplifier reduces it to the empty language (matches nothing). + // (Bounds from variables: a literal reversed range trips a lint.) + let (min, max) = (5u32, 3u32); + assert!(term.repeat(min..max).unwrap().is_empty().unwrap()); } #[test] - fn test__() -> Result<(), String> { - let term = Term::from_regex("(abc|de){2}").unwrap(); + fn test_iter_strings_is_lazy_on_infinite_language() { + // Must not hang on an infinite language: take a finite prefix. + let term = Term::from_pattern("a+").unwrap(); + let first = term + .iter_strings(GenerationOptions::new()) + .take(5) + .collect::, _>>() + .unwrap(); + assert_eq!(5, first.len()); + } - let strings = term.generate_strings(3).unwrap(); + #[test] + fn test_iter_strings_propagates_error_then_ends() { + use crate::execution_profile::ExecutionProfileBuilder; - println!("strings={:?}", strings); + // A tight state budget makes the underlying `to_automaton` fail; the + // iterator must surface that error once and then terminate. + let term = Term::from_pattern("abcdef").unwrap(); + let profile = ExecutionProfileBuilder::new() + .max_number_of_states(1) + .build(); - Ok(()) + profile.run(|| { + let mut it = term.iter_strings(GenerationOptions::new()); + assert!(matches!( + it.next(), + Some(Err(EngineError::AutomatonHasTooManyStates)) + )); + assert!(it.next().is_none()); + }); + } + + #[test] + fn test_variadic_ops_with_no_operands_equal_self() { + let term = Term::from_pattern("abc").unwrap(); + + assert!( + term.concat(std::iter::empty::<&Term>()) + .unwrap() + .equivalent(&term) + .unwrap() + ); + assert!( + term.union(std::iter::empty::<&Term>()) + .unwrap() + .equivalent(&term) + .unwrap() + ); + assert!( + term.intersection(std::iter::empty::<&Term>()) + .unwrap() + .equivalent(&term) + .unwrap() + ); } } diff --git a/src/regex/analyze/affixes.rs b/src/regex/analyze/affixes.rs index 4213e3f..cb07b00 100644 --- a/src/regex/analyze/affixes.rs +++ b/src/regex/analyze/affixes.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use super::*; impl RegularExpression { - pub fn get_common_affixes( + pub(crate) fn get_common_affixes( &self, other: &RegularExpression, ) -> ( @@ -21,7 +21,7 @@ impl RegularExpression { (common_prefix, (self_regex, other_regex), common_suffix) } - pub fn get_common_affix( + pub(crate) fn get_common_affix( &self, other: &RegularExpression, is_prefix: bool, @@ -46,27 +46,27 @@ impl RegularExpression { let other_regex; match (self, other) { - (RegularExpression::Concat(_), _) => { + (RegularExpression::Concat(..), _) => { (common_affix, (self_regex, other_regex)) = Self::opaffix_concat_and_other(self, other, is_prefix); } - (_, RegularExpression::Concat(_)) => { + (_, RegularExpression::Concat(..)) => { (common_affix, (other_regex, self_regex)) = Self::opaffix_concat_and_other(other, self, is_prefix); } - (RegularExpression::Character(_), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Character(..), RegularExpression::Repetition(..)) => { (common_affix, (self_regex, other_regex)) = Self::opaffix_character_and_repetition(self, other); } - (RegularExpression::Repetition(_, _, _), RegularExpression::Character(_)) => { + (RegularExpression::Repetition(..), RegularExpression::Character(..)) => { (common_affix, (other_regex, self_regex)) = Self::opaffix_character_and_repetition(other, self); } - (RegularExpression::Repetition(_, _, _), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Repetition(..), RegularExpression::Repetition(..)) => { (common_affix, (self_regex, other_regex)) = Self::opaffix_repetition_and_repetition(self, other); } - (RegularExpression::Alternation(_), RegularExpression::Alternation(_)) => { + (RegularExpression::Alternation(..), RegularExpression::Alternation(..)) => { (common_affix, (self_regex, other_regex)) = Self::opaffix_alternation_and_alternation(self, other); } @@ -86,11 +86,17 @@ impl RegularExpression { (RegularExpression, RegularExpression), ) { if let ( - RegularExpression::Character(_), + RegularExpression::Character(..), RegularExpression::Repetition(that_regex, that_min, that_max_opt), ) = (this_character, that_repetition) { - if this_character == &**that_regex && *that_min == 1 { + // The `max != 0` guard keeps a directly-constructed invalid + // repetition (`r{1,0}`) from underflowing; such trees are + // rejected by `to_automaton`, the simplifier just must not panic. + if this_character == &**that_regex + && *that_min == 1 + && that_max_opt.is_none_or(|that_max| that_max >= 1) + { let new_max = that_max_opt.as_ref().map(|that_max| that_max - 1); ( Some(this_character.clone()), @@ -119,7 +125,14 @@ impl RegularExpression { RegularExpression::Repetition(that_regex, that_min, that_max_opt), ) = (this_repetition, that_repetition) { - if this_regex == that_regex { + // As in `opaffix_character_and_repetition`: directly-constructed + // degenerate bounds (`r{5,2}`) must not reach the affix + // arithmetic below, whose `max - prefix_max` subtractions rely on + // `max >= min` on both sides. Such trees are rejected by + // `to_automaton`; the simplifier just must not panic. + let degenerate_bounds = this_max_opt.is_some_and(|this_max| this_max < *this_min) + || that_max_opt.is_some_and(|that_max| that_max < *that_min); + if this_regex == that_regex && !degenerate_bounds { let prefix_min = *cmp::min(this_min, that_min); let prefix_max_opt; if this_min == that_min { @@ -285,6 +298,7 @@ mod tests { assert_regex_affix(true, "(ab|cd)x", "(ab|cd)y", "(ab|cd)", "x", "y"); assert_regex_affix(true, "a+", "a+b", "a+", "", "b"); + assert_regex_affix(true, "(ab|cd)", "(ab|cd)", "(ab|cd)", "", ""); Ok(()) } diff --git a/src/regex/analyze/mod.rs b/src/regex/analyze/mod.rs index ae08148..bc17051 100644 --- a/src/regex/analyze/mod.rs +++ b/src/regex/analyze/mod.rs @@ -1,12 +1,12 @@ -use self::cardinality::Cardinality; - use super::*; mod affixes; mod number_of_states; impl RegularExpression { - pub fn get_length(&self) -> (Option, Option) { + /// Returns the minimum and maximum length of possible matched strings. + #[must_use] + pub fn length(&self) -> (Option, Option) { match self { RegularExpression::Character(range) => { if range.is_empty() { @@ -15,11 +15,24 @@ impl RegularExpression { (Some(1), Some(1)) } RegularExpression::Repetition(regex, min, max_opt) => { - let (min_length, max_length_opt) = regex.get_length(); + if let Some(max) = max_opt { + if max < min { + // No valid repetition count: the empty language, + // consistently with `repeat` and `to_automaton`. + return (None, None); + } + if *max == 0 { + // r⁰ = {""} regardless of the inner expression — + // including an unbounded one, which the general path + // below would report as having no maximum length. + return (Some(0), Some(0)); + } + } + let (min_length, max_length_opt) = regex.length(); if let Some(min_length) = min_length { - let new_min_length = min * min_length; + let new_min_length = min.saturating_mul(min_length); let new_max_length = if let Some(max_length) = max_length_opt { - max_opt.as_ref().map(|max| max * max_length) + max_opt.as_ref().map(|max| max.saturating_mul(max_length)) } else { None }; @@ -31,18 +44,18 @@ impl RegularExpression { } } RegularExpression::Concat(concat_vec) => { - let mut new_min_length = 0; - let mut new_max_length = Some(0); + let mut new_min_length: u32 = 0; + let mut new_max_length: Option = Some(0); for concat_element in concat_vec { - let (min_length, max_length_opt) = concat_element.get_length(); + let (min_length, max_length_opt) = concat_element.length(); if let Some(min_length) = min_length { - new_min_length += min_length; + new_min_length = new_min_length.saturating_add(min_length); if let Some(new_max) = new_max_length { if let Some(max_length) = max_length_opt { - new_max_length = Some(new_max + max_length); + new_max_length = Some(new_max.saturating_add(max_length)); } else { new_max_length = None; } @@ -60,11 +73,13 @@ impl RegularExpression { } let mut new_min_length = u32::MAX; let mut new_max_length = Some(0); + let mut any_non_empty = false; for alternation_element in alternation_vec { - let (min_length, max_length_opt) = alternation_element.get_length(); + let (min_length, max_length_opt) = alternation_element.length(); if let Some(min_length) = min_length { + any_non_empty = true; new_min_length = cmp::min(new_min_length, min_length); if let Some(new_max) = new_max_length { @@ -74,79 +89,15 @@ impl RegularExpression { new_max_length = None; } } - } else { - return (None, None); } } - (Some(new_min_length), new_max_length) - } - } - } - - pub fn get_cardinality(&self) -> Cardinality { - if self.is_empty() { - return Cardinality::Integer(0); - } else if self.is_total() { - return Cardinality::Infinite; - } - match self { - RegularExpression::Character(range) => Cardinality::Integer(range.get_cardinality()), - RegularExpression::Repetition(regular_expression, min, max_opt) => { - if let Some(max) = max_opt { - let regex_cardinality = regular_expression.get_cardinality(); - if let Cardinality::Integer(cardinality) = regex_cardinality { - let mut cardinality_temp: u32 = 0; - for i in *min..*max + 1 { - if let Some(pow) = cardinality.checked_pow(i) { - if let Some(add) = cardinality_temp.checked_add(pow) { - cardinality_temp = add; - } else { - return Cardinality::BigInteger; - } - } else { - return Cardinality::BigInteger; - } - } - Cardinality::Integer(cardinality_temp) - } else { - regex_cardinality - } - } else { - Cardinality::Infinite - } - } - RegularExpression::Concat(concat) => { - let mut cardinality: u32 = 1; - for concat_element in concat { - let element_cardinality = concat_element.get_cardinality(); - if let Cardinality::Integer(element_cardinality) = element_cardinality { - if let Some(mult) = cardinality.checked_mul(element_cardinality) { - cardinality = mult; - } else { - return Cardinality::BigInteger; - } - } else { - return element_cardinality; - } - } - Cardinality::Integer(cardinality) - } - RegularExpression::Alternation(alternation) => { - let mut cardinality: u32 = 0; - for alternation_element in alternation { - let element_cardinality = alternation_element.get_cardinality(); - if let Cardinality::Integer(element_cardinality) = element_cardinality { - if let Some(add) = cardinality.checked_add(element_cardinality) { - cardinality = add; - } else { - return Cardinality::BigInteger; - } - } else { - return element_cardinality; - } + if !any_non_empty { + // Every branch was the empty language ⇒ empty language. + return (None, None); } - Cardinality::Integer(cardinality) + + (Some(new_min_length), new_max_length) } } } @@ -156,6 +107,22 @@ impl RegularExpression { mod tests { use super::*; + // `r{0,0}` over an unbounded inner expression has length exactly {""} + // (`(Some(0), Some(0))`), and degenerate hand-built bounds (`a{5,2}`, the + // empty language) report `(None, None)` per the ∅ convention. + #[test] + fn length_of_zero_and_degenerate_repetitions() { + let a_star = RegularExpression::new("a*").unwrap(); + let zero = RegularExpression::Repetition(Box::new(a_star), 0, Some(0)); + assert_eq!((Some(0), Some(0)), zero.length()); + assert_eq!(zero.to_automaton().unwrap().length(), zero.length()); + + let a = RegularExpression::new("a").unwrap(); + let degenerate = RegularExpression::Repetition(Box::new(a), 5, Some(2)); + assert_eq!((None, None), degenerate.length()); + assert_eq!(RegularExpression::new_empty().length(), degenerate.length()); + } + #[test] fn test_length() -> Result<(), String> { assert_length(".{1,1000}"); @@ -169,16 +136,18 @@ mod tests { assert_length("(at?)"); assert_length("(ot){3,4}"); assert_length("(ot?d){1,4}"); - assert_length("((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}"); + assert_length( + "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}", + ); assert_eq!( - FastAutomaton::new_empty().get_length(), - RegularExpression::new_empty().get_length() + FastAutomaton::new_empty().length(), + RegularExpression::new_empty().length() ); assert_eq!( - FastAutomaton::new_total().get_length(), - RegularExpression::new_total().get_length() + FastAutomaton::new_total().length(), + RegularExpression::new_total().length() ); Ok(()) } @@ -187,51 +156,12 @@ mod tests { println!("{}", regex); let regex = RegularExpression::new(regex).unwrap(); - let (min, max_opt) = regex.get_length(); + let (min, max_opt) = regex.length(); let automaton = regex.to_automaton().unwrap(); - //automaton.to_dot(); - let (min_automaton_opt, max_automaton_opt) = automaton.get_length(); + let (min_automaton_opt, max_automaton_opt) = automaton.length(); assert_eq!((min_automaton_opt, max_automaton_opt), (min, max_opt)); } - - #[test] - fn test_cardinality() -> Result<(), String> { - assert_cardinality(".{1,1000}"); - assert_cardinality("toto"); - assert_cardinality("."); - assert_cardinality(".{2,3}"); - assert_cardinality("q(ab|ca|abc)x"); - assert_cardinality("q(ab|ca|ab|abc)x"); - assert_cardinality(".*"); - assert_cardinality(".?"); - assert_cardinality("a*(aad|ads|a)abc.*def.*ghi"); - assert_cardinality("(at?)"); - assert_cardinality("(ot){3,4}"); - assert_cardinality("(t){1,3}"); - assert_cardinality("(ot?d){1,4}"); - assert_cardinality("((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}"); - Ok(()) - } - - fn assert_cardinality(regex: &str) { - println!("{}", regex); - let regex = RegularExpression::new(regex).unwrap(); - - let cardinality = regex.get_cardinality(); - - let mut automaton = regex.to_automaton().unwrap(); - - if !automaton.is_cyclic() { - automaton = automaton.determinize().unwrap(); - } - - //automaton.to_dot(); - - let expected = automaton.get_cardinality().unwrap(); - - assert_eq!(expected, cardinality); - } } diff --git a/src/regex/analyze/number_of_states.rs b/src/regex/analyze/number_of_states.rs index 90c1897..ccb9ea9 100644 --- a/src/regex/analyze/number_of_states.rs +++ b/src/regex/analyze/number_of_states.rs @@ -9,7 +9,7 @@ struct AbstractStateMetadata { } impl AbstractStateMetadata { - pub fn new(has_incoming_edges: bool, has_outgoing_edges: bool) -> Self { + pub(crate) fn new(has_incoming_edges: bool, has_outgoing_edges: bool) -> Self { AbstractStateMetadata { has_incoming_edges, has_outgoing_edges, @@ -17,39 +17,72 @@ impl AbstractStateMetadata { } } -#[derive(Debug)] +#[derive(Clone, Debug)] struct AbstractNFAMetadata { start: AbstractStateMetadata, accepted: Vec, number_of_states: usize, + /// Whether the language contains the empty string, i.e. the start state + /// is accepting. Exact for the constructions modeled here. + accepts_empty_string: bool, + /// Whether some accept state may sit one transition away from the start + /// state. Over-approximate (may be `true` when none does), never + /// under-approximate: `alternate` withholds a merge discount on it, so + /// erring towards `true` keeps the estimate an upper bound. + accept_adjacent_to_start: bool, } impl AbstractNFAMetadata { - pub fn new() -> Self { + pub(crate) fn new() -> Self { AbstractNFAMetadata { start: AbstractStateMetadata::new(false, true), accepted: vec![AbstractStateMetadata::new(true, false)], number_of_states: 2, + accepts_empty_string: false, + accept_adjacent_to_start: true, } } - pub fn new_empty_string() -> Self { + pub(crate) fn new_empty_string() -> Self { AbstractNFAMetadata { start: AbstractStateMetadata::new(false, false), accepted: vec![AbstractStateMetadata::new(false, false)], number_of_states: 1, + accepts_empty_string: true, + accept_adjacent_to_start: false, } } - pub fn new_empty() -> Self { + pub(crate) fn new_empty() -> Self { AbstractNFAMetadata { start: AbstractStateMetadata::new(false, false), accepted: vec![], number_of_states: 1, + accepts_empty_string: false, + accept_adjacent_to_start: false, } } - pub fn concat(&self, nfa: &AbstractNFAMetadata) -> Self { + /// [`accept_adjacent_to_start`](Self::accept_adjacent_to_start) of the + /// concatenation `self · nfa`: the boundary attaches `nfa`'s structure to + /// `self`'s accept states, so an accept can only end up next to the start + /// through an accepting start on one side of the boundary. + fn concat_accept_adjacency(&self, nfa: &AbstractNFAMetadata) -> bool { + (nfa.accepts_empty_string && self.accept_adjacent_to_start) + || (self.accepts_empty_string && nfa.accept_adjacent_to_start) + } + + pub(crate) fn concat(&self, nfa: &AbstractNFAMetadata) -> Self { + let is_empty_string = |m: &AbstractNFAMetadata| { + m.number_of_states == 1 && !m.accepted.is_empty() && !m.start.has_outgoing_edges + }; + if is_empty_string(nfa) { + return self.clone(); + } + if is_empty_string(self) { + return nfa.clone(); + } + let start_state_and_accept_states_not_mergeable = nfa.start.has_incoming_edges && self.accepted.iter().any(|s| s.has_outgoing_edges); @@ -57,18 +90,57 @@ impl AbstractNFAMetadata { AbstractNFAMetadata { start: self.start.clone(), accepted: nfa.accepted.clone(), - number_of_states: self.number_of_states + nfa.number_of_states, + number_of_states: self.number_of_states.saturating_add(nfa.number_of_states), + accepts_empty_string: self.accepts_empty_string && nfa.accepts_empty_string, + accept_adjacent_to_start: self.concat_accept_adjacency(nfa), } } else { AbstractNFAMetadata { start: self.start.clone(), accepted: nfa.accepted.clone(), - number_of_states: self.number_of_states + nfa.number_of_states - 1, + number_of_states: self.number_of_states.saturating_add(nfa.number_of_states) - 1, + accepts_empty_string: self.accepts_empty_string && nfa.accepts_empty_string, + accept_adjacent_to_start: self.concat_accept_adjacency(nfa), } } } - pub fn repeat(&self, min: u32, max_opt: &Option) -> Self { + pub(crate) fn repeat(&self, min: u32, max_opt: &Option) -> Self { + // r⁰ = {""} (the empty-string automaton, a single state). + if max_opt == &Some(0) { + return Self::new_empty_string(); + } + + if self.accepted.is_empty() { + return if min == 0 { + Self::new_empty_string() + } else { + self.clone() + }; + } + if max_opt.is_none() && min >= 1 { + let appended_copy_cost = if self.start.has_incoming_edges + && self.accepted.iter().any(|s| s.has_outgoing_edges) + { + self.number_of_states + } else { + self.number_of_states - 1 + }; + let mandatory = AbstractNFAMetadata { + start: self.start.clone(), + accepted: self.accepted.clone(), + number_of_states: self + .number_of_states + .saturating_add((min as usize - 1).saturating_mul(appended_copy_cost)), + accepts_empty_string: self.accepts_empty_string, + // An accept of rᵐⁱⁿ can neighbour the start only when it is + // one copy deep, or when copies collapse over "" ∈ r. + accept_adjacent_to_start: self.accept_adjacent_to_start + && (min == 1 || self.accepts_empty_string), + }; + return mandatory.concat(&self.repeat(0, &None)); + } + let start_state_not_mergeable = self.start.has_incoming_edges; let accepted_not_mergeable = self.accepted.iter().any(|s| s.has_outgoing_edges); let start_state_or_accept_states_not_mergeable = @@ -77,41 +149,72 @@ impl AbstractNFAMetadata { let mut return_start = self.start.clone(); let mut return_accepted = self.accepted.clone(); + if min == 0 && start_state_not_mergeable { + return_start.has_incoming_edges = false; + } + if max_opt.is_none() { for accepted in return_accepted.iter_mut() { accepted.has_outgoing_edges = true; } } - if min == 0 && !start_state_or_accept_states_not_mergeable { + if min == 0 + && !start_state_or_accept_states_not_mergeable + && max_opt.is_none() + && self.accepted.len() == 1 + { return_start.has_incoming_edges = true; return_accepted.push(return_start.clone()); - if max_opt.is_none() { - let return_number_of_states = if !start_state_or_accept_states_not_mergeable { - self.number_of_states - 1 - } else { - self.number_of_states - }; - return AbstractNFAMetadata { - start: return_start, - accepted: return_accepted, - number_of_states: return_number_of_states, - }; - } + + return AbstractNFAMetadata { + start: return_start, + accepted: return_accepted, + number_of_states: (self.number_of_states - 1).max(1), + accepts_empty_string: true, + accept_adjacent_to_start: self.accept_adjacent_to_start, + }; } if min == 0 { return_accepted.push(return_start.clone()); } + if let Some(max) = max_opt + && *max > cmp::max(min, 1) + { + return_accepted.push(AbstractStateMetadata::new(true, true)); + } + let return_number_of_states = if let Some(max) = max_opt { - let mult = if start_state_not_mergeable && (accepted_not_mergeable || min == 0) { + // Mirror `repeat_mut`: rᵐⁱⁿ mandatory copies built by merging + // concatenation, then `max - max(min,1)` optional tail copies. A + // tail copy whose start has incoming edges is concatenated without + // merging (a fresh start state, so +`number_of_states`); otherwise + // it merges (+`number_of_states - 1`). + let max = *max as usize; + let merge_cost = if start_state_not_mergeable && accepted_not_mergeable { + self.number_of_states + } else { + self.number_of_states - 1 + }; + let tail_cost = if start_state_not_mergeable { self.number_of_states } else { self.number_of_states - 1 }; - *max as usize * mult + 1 + if min == 0 { + let base = self + .number_of_states + .saturating_add(if start_state_not_mergeable { 1 } else { 0 }); + base.saturating_add(max.saturating_sub(1).saturating_mul(tail_cost)) + } else { + let mandatory = self + .number_of_states + .saturating_add((min as usize - 1).saturating_mul(merge_cost)); + mandatory.saturating_add(max.saturating_sub(min as usize).saturating_mul(tail_cost)) + } } else { let mult = if start_state_not_mergeable { self.number_of_states @@ -119,17 +222,24 @@ impl AbstractNFAMetadata { self.number_of_states - 1 }; - cmp::max(min, 1) as usize * mult + 1 + (cmp::max(min, 1) as usize) + .saturating_mul(mult) + .saturating_add(1) }; AbstractNFAMetadata { start: return_start, accepted: return_accepted, number_of_states: return_number_of_states, + accepts_empty_string: min == 0 || self.accepts_empty_string, + // An accept can neighbour the start only when it is one copy deep + // (min <= 1), or when copies collapse over "" ∈ r. + accept_adjacent_to_start: self.accept_adjacent_to_start + && (min <= 1 || self.accepts_empty_string), } } - pub fn alternate(&mut self, nfa: &AbstractNFAMetadata) -> Self { + pub(crate) fn alternate(&mut self, nfa: &AbstractNFAMetadata) -> Self { let self_start_state_not_mergeable = self.start.has_incoming_edges; let self_accepted_not_mergeable = self.accepted.iter().any(|s| s.has_outgoing_edges); @@ -139,36 +249,77 @@ impl AbstractNFAMetadata { let return_start = AbstractStateMetadata::new(false, true); let mut return_accepted = vec![]; - let mut return_number_of_states = self.number_of_states + nfa.number_of_states; + let mut return_number_of_states = + self.number_of_states.saturating_add(nfa.number_of_states); if !self_start_state_not_mergeable && !nfa_start_state_not_mergeable { return_number_of_states -= 1; + } else if self_start_state_not_mergeable && nfa_start_state_not_mergeable { + return_number_of_states = return_number_of_states.saturating_add(1); } - if !self_accepted_not_mergeable && !nfa_accepted_not_mergeable { + // A looping start (incoming edges) makes the union materialize + // the start's direct successors before accept states are merged, + // and an accept among those successors never merges (e.g. `a*a`, + // whose accept hangs directly off the looping start). Withhold the + // saving when an accept may sit there: an upper bound may + // overshoot, but never undershoot. + let nfa_accept_beside_looping_start = + nfa_start_state_not_mergeable && nfa.accept_adjacent_to_start; + + if !self_accepted_not_mergeable + && !nfa_accepted_not_mergeable + && !nfa_accept_beside_looping_start + && !self.accepted.is_empty() + && !nfa.accepted.is_empty() + && self.number_of_states > 1 + && nfa.number_of_states > 1 + { return_number_of_states -= 1; return_accepted.push(AbstractStateMetadata::new(true, false)); } else { - return_accepted.extend(self.accepted.clone()); - return_accepted.extend(nfa.accepted.clone()); + let mut extend = |operand: &AbstractNFAMetadata| { + if operand.number_of_states == 1 + && !operand.accepted.is_empty() + && !operand.start.has_incoming_edges + { + return_accepted.push(AbstractStateMetadata::new(false, true)); + } else { + return_accepted.extend(operand.accepted.clone()); + } + }; + extend(self); + extend(nfa); } AbstractNFAMetadata { start: return_start, accepted: return_accepted, - number_of_states: return_number_of_states, + accepts_empty_string: self.accepts_empty_string || nfa.accepts_empty_string, + // The union's entry state carries both operands' start edges. + accept_adjacent_to_start: self.accept_adjacent_to_start || nfa.accept_adjacent_to_start, + // Both merge discounts can apply to two single-state {""} + // operands (e.g. `a{0,0}|b{0,0}`); clamp so the count never + // reaches zero (see `repeat`). + number_of_states: return_number_of_states.max(1), } } } impl RegularExpression { - pub fn get_number_of_states_in_nfa(&self) -> usize { + pub(crate) fn get_number_of_states_in_nfa(&self) -> usize { self.evaluate_number_of_states_in_nfa().number_of_states } fn evaluate_number_of_states_in_nfa(&self) -> AbstractNFAMetadata { match self { - RegularExpression::Character(_) => AbstractNFAMetadata::new(), + RegularExpression::Character(range) => { + if range.is_empty() { + AbstractNFAMetadata::new_empty() + } else { + AbstractNFAMetadata::new() + } + } RegularExpression::Repetition(regex, min, max_opt) => regex .evaluate_number_of_states_in_nfa() .repeat(*min, max_opt), @@ -203,8 +354,7 @@ mod tests { #[test] fn test_number_of_states_in_nfa() -> Result<(), String> { - //TODO: - //assert_number_of_states_in_nfa("(ab|c)+"); + assert_number_of_states_in_nfa("(ab|c)+"); assert_number_of_states_in_nfa("A+"); assert_number_of_states_in_nfa("B*"); assert_number_of_states_in_nfa("([ab]*a)"); @@ -222,6 +372,24 @@ mod tests { assert_number_of_states_in_nfa("(b*a){5,26}"); assert_number_of_states_in_nfa("(ba*){5,26}"); + // Unbounded with min >= 1 over a self-looping start (r{min,} = rᵐⁱⁿ·r*). + assert_number_of_states_in_nfa("(b*a){1,}"); + assert_number_of_states_in_nfa("(b*a){2,}"); + assert_number_of_states_in_nfa("(b*a){5,}"); + assert_number_of_states_in_nfa("(a*b){1,}"); + assert_number_of_states_in_nfa("(a*b){3,}"); + + // Operands whose start has incoming edges AND whose accept states have + // outgoing edges: the `min == 0` construction allocates a fresh start + // with no incoming edges, and the metadata must say so — keeping the + // flag set overcounted the following concatenation by one state. + assert_number_of_states_in_nfa("(a*ba*){2,}"); + assert_number_of_states_in_nfa("(a*ba*){5,}"); + assert_number_of_states_in_nfa("(a?b?){2,}"); + assert_number_of_states_in_nfa("a+(b*a)?"); + assert_number_of_states_in_nfa("a+(b*a){0,2}"); + assert_number_of_states_in_nfa("(b*a){0,3}"); + assert_number_of_states_in_nfa(""); assert_number_of_states_in_nfa("toto"); assert_number_of_states_in_nfa("A+B*"); @@ -261,20 +429,192 @@ mod tests { assert_number_of_states_in_nfa("q(ab|ca|ab|abc)x"); assert_number_of_states_in_nfa("a*(aad|ads|a)abc.*def.*ghi"); - assert_number_of_states_in_nfa("((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}"); + assert_number_of_states_in_nfa( + "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}", + ); Ok(()) } + // The unbounded `r{min,}` estimate is a closed form (not a loop over + // `min`), so a short pattern with a huge minimum like `a{4294967295,}` + // saturates the estimate and the state budget rejects it immediately, + // rather than burning CPU inside the estimator. + #[test] + fn huge_unbounded_repetition_is_rejected_quickly() { + use crate::error::EngineError; + use crate::execution_profile::ExecutionProfileBuilder; + + ExecutionProfileBuilder::new() + .max_number_of_states(100) + .build() + .run(|| { + let regex = RegularExpression::new("a{4294967295,}").unwrap(); + assert_eq!( + EngineError::AutomatonHasTooManyStates, + regex.to_automaton().unwrap_err() + ); + }); + } + + // The estimator's arithmetic saturates, so nested huge (but parseable) + // bounds do not overflow; the budget then rejects the pattern. + #[test] + fn nested_huge_bounds_saturate_instead_of_overflowing() { + use crate::error::EngineError; + use crate::execution_profile::ExecutionProfileBuilder; + + ExecutionProfileBuilder::new() + .max_number_of_states(100) + .build() + .run(|| { + let regex = + RegularExpression::new("((a{4294967295}){4294967295}){4294967295}").unwrap(); + assert_eq!( + EngineError::AutomatonHasTooManyStates, + regex.to_automaton().unwrap_err() + ); + + // A `?`-wrapped saturated repetition whose operand start has + // incoming edges takes the `min == 0` finite-max arm, whose + // `+ 1` for the fresh start state must saturate too (it used + // to be an unchecked add that overflowed in debug builds). + let regex = + RegularExpression::new("((((a*b){4294967295}){4294967295}){4294967295})?") + .unwrap(); + assert_eq!( + EngineError::AutomatonHasTooManyStates, + regex.to_automaton().unwrap_err() + ); + }); + } + + // Directly-constructed (unsimplified) repetitions over {""} + // sub-expressions — shapes the string parser simplifies away, but any + // user of the public enum can build — must not drive the abstract state + // count to zero, which would underflow the merge discounts. + #[test] + fn degenerate_repetitions_do_not_underflow() { + use std::collections::VecDeque; + + let atom = RegularExpression::new("a").unwrap(); + // a{0,0} denotes {""} without being the canonical empty-string form. + let empty_string = RegularExpression::Repetition(Box::new(atom), 0, Some(0)); + let star_of_alternation = RegularExpression::Repetition( + Box::new(RegularExpression::Alternation(vec![ + empty_string.clone(), + empty_string.clone(), + ])), + 0, + None, + ); + let star_of_concat = RegularExpression::Repetition( + Box::new(RegularExpression::Concat(VecDeque::from([ + empty_string.clone(), + RegularExpression::Repetition(Box::new(empty_string), 0, None), + ]))), + 0, + None, + ); + + for regex in [star_of_alternation, star_of_concat] { + let estimate = regex.get_number_of_states_in_nfa(); + assert!(estimate >= 1, "state estimate of {regex} must be >= 1"); + let automaton = regex.to_automaton().unwrap(); + assert!(automaton.number_of_states() >= 1); + } + } + fn assert_number_of_states_in_nfa(regex: &str) { println!("{}", regex); let regex = RegularExpression::new(regex).unwrap(); - //regex.to_automaton().unwrap().to_dot(); - let number_of_states_in_nfa = regex.get_number_of_states_in_nfa(); let automaton = regex.to_automaton().unwrap(); - assert_eq!(automaton.get_number_of_states(), number_of_states_in_nfa); + assert_eq!(automaton.number_of_states(), number_of_states_in_nfa); + } + + mod prop { + use super::*; + use crate::CharRange; + use crate::execution_profile::ExecutionProfileBuilder; + use proptest::prelude::*; + use regex_charclass::char::Char; + + fn letter(c: char) -> CharRange { + let c = Char::new(c); + CharRange::new_from_range(c..=c) + } + + /// Regular-expression trees built directly over the enum — covering + /// shapes the string parser simplifies away — with occasional huge + /// repetition bounds to exercise the saturating arithmetic. + fn arb_regex_tree() -> impl Strategy { + let leaf = prop_oneof![ + 4 => Just(letter('a')), + 4 => Just(letter('b')), + 1 => Just(CharRange::empty()), + 1 => Just(CharRange::total()), + ] + .prop_map(RegularExpression::Character); + leaf.prop_recursive(4, 32, 3, |inner| { + let bound = prop_oneof![ + 8 => 0u32..=3, + 1 => u32::MAX - 2..=u32::MAX, + ]; + prop_oneof![ + 3 => (inner.clone(), bound.clone(), bound, any::()).prop_map( + |(r, min, extra, has_max)| { + // max is min + extra, so the bounds are always valid + let max = if has_max { + Some(min.saturating_add(extra)) + } else { + None + }; + RegularExpression::Repetition(Box::new(r), min, max) + } + ), + 2 => proptest::collection::vec(inner.clone(), 1..=3) + .prop_map(|v| RegularExpression::Concat(v.into())), + 2 => proptest::collection::vec(inner, 1..=3) + .prop_map(RegularExpression::Alternation), + ] + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + /// The estimate must never *under*-estimate: the state budget + /// rejects a pattern when the estimate exceeds it, so an + /// under-estimate would let an oversized construction through + /// (the denial-of-service direction). It must also never panic — + /// huge (but parseable) bounds have to saturate, not overflow. + /// + /// Exactness (`==`) intentionally is not asserted here: the + /// estimate is exact for the deterministic corpus above but only + /// an upper bound in general (e.g. `(ab|c)+`). + #[test] + fn estimate_is_a_sound_upper_bound(regex in arb_regex_tree()) { + let estimate = regex.get_number_of_states_in_nfa(); + prop_assert!(estimate >= 1, "state estimate of {} must be >= 1", regex); + + let automaton = ExecutionProfileBuilder::new() + .max_number_of_states(4096) + .execution_timeout(2000) + .build() + .run(|| regex.to_automaton()); + if let Ok(automaton) = automaton { + prop_assert!( + automaton.number_of_states() <= estimate, + "the estimate under-estimated {}: estimate {} < actual {}", + regex, + estimate, + automaton.number_of_states() + ); + } + } + } } } diff --git a/src/regex/builder.rs b/src/regex/builder.rs index e8a354f..12b2dda 100644 --- a/src/regex/builder.rs +++ b/src/regex/builder.rs @@ -1,68 +1,126 @@ -use ::regex::Regex; -use lazy_static::lazy_static; use regex_charclass::irange::range::AnyRange; use regex_syntax::ParserBuilder; use super::*; -lazy_static! { - static ref RE_FLAG_DETECTION: Regex = - Regex::new(r"\(\?[imsx]*-?[imsx]*\)").expect("Can not compile flag detection regex."); -} - impl RegularExpression { - pub fn new(regex: &str) -> Result { - if regex.is_empty() { + /// Parses and simplifies the provided pattern and returns the resulting [`RegularExpression`]. + pub fn new(pattern: &str) -> Result { + Self::parse(pattern, true) + } + + /// Parses the provided pattern and returns the resulting [`RegularExpression`]. If `simplify` is `true`, the expression is simplified during parsing. + #[tracing::instrument(level = "debug", skip(pattern), fields(pattern_len = pattern.len()))] + pub fn parse(pattern: &str, simplify: bool) -> Result { + if pattern.is_empty() { return Ok(RegularExpression::new_empty_string()); } - if regex == "[]" { + if pattern == "[]" { return Ok(RegularExpression::new_empty()); } + // Inline flags such as `(?i)` change matching semantics the engine + // cannot represent (it operates uniformly over character ranges). + // Silently ignoring them would diverge from every mainstream engine + // (e.g. `(?i)abc` would not match `ABC`), so they are rejected instead. + Self::reject_inline_flags(pattern)?; match ParserBuilder::new() .dot_matches_new_line(true) .build() - .parse(&Self::remove_flags(regex)) + .parse(pattern) { - Ok(hir) => Self::convert_to_regex(&hir), + // The whole pattern is matched against the full input (anchored), + // so a leading start-of-text anchor and a trailing end-of-text + // anchor are accepted as redundant no-ops; anchors elsewhere and + // word boundaries are rejected (see `convert_look`). + Ok(hir) => Self::convert_to_regex(&hir, simplify, true, true), Err(err) => Err(EngineError::RegexSyntaxError(err.to_string())), } } - fn remove_flags(regex: &str) -> String { - RE_FLAG_DETECTION.replace_all(regex, "").to_string() + /// Rejects patterns that set inline flags, either as a `(?flags)` directive + /// or a `(?flags:...)` group. Parsing to the AST (rather than scanning the + /// string) means flag-lookalikes inside character classes (e.g. `[a(?i)]`) + /// are correctly *not* treated as flags. + fn reject_inline_flags(pattern: &str) -> Result<(), EngineError> { + use regex_syntax::ast::parse::Parser; + + let ast = Parser::new() + .parse(pattern) + .map_err(|err| EngineError::RegexSyntaxError(err.to_string()))?; + if Self::ast_sets_flags(&ast) { + return Err(EngineError::UnsupportedRegexFeature( + "inline flags such as (?i), (?m), (?s) and (?x) are not supported".to_string(), + )); + } + Ok(()) + } + + /// Returns `true` if the AST sets any inline flag. Non-capturing groups + /// without flags (`(?:...)`) are allowed. + fn ast_sets_flags(ast: ®ex_syntax::ast::Ast) -> bool { + use regex_syntax::ast::{Ast, GroupKind}; + + match ast { + Ast::Flags(_) => true, + Ast::Group(group) => { + matches!(&group.kind, GroupKind::NonCapturing(flags) if !flags.items.is_empty()) + || Self::ast_sets_flags(&group.ast) + } + Ast::Repetition(repetition) => Self::ast_sets_flags(&repetition.ast), + Ast::Alternation(alternation) => alternation.asts.iter().any(Self::ast_sets_flags), + Ast::Concat(concat) => concat.asts.iter().any(Self::ast_sets_flags), + _ => false, + } } + /// Creates a regular expression that matches all possible strings. pub fn new_total() -> Self { RegularExpression::Repetition( - Box::new(RegularExpression::Character(Range::total())), + Box::new(RegularExpression::Character(CharRange::total())), 0, None, ) } + /// Creates a regular expression that matches the empty language. pub fn new_empty() -> Self { - RegularExpression::Character(Range::empty()) + RegularExpression::Character(CharRange::empty()) } + /// Creates a regular expression that matches only the empty string `""`. pub fn new_empty_string() -> Self { RegularExpression::Concat(VecDeque::new()) } - fn convert_to_regex(hir: &Hir) -> Result { + /// Converts a parsed HIR node into a [`RegularExpression`]. + /// + /// `at_start`/`at_end` track whether this node sits at the very start/end + /// of the overall match (nothing can be consumed before/after it). They + /// govern which anchors are accepted as redundant no-ops; see + /// [`convert_look`](Self::convert_look). + fn convert_to_regex( + hir: &Hir, + simplify: bool, + at_start: bool, + at_end: bool, + ) -> Result { match hir.kind() { HirKind::Empty => Ok(RegularExpression::new_empty_string()), HirKind::Literal(literal) => { - let mut regex_concat = RegularExpression::new_empty_string(); - if let Ok(string) = String::from_utf8(literal.0.clone().into_vec()) { + if let Ok(string) = std::str::from_utf8(&literal.0) { + let mut elements = VecDeque::new(); for char in string.chars() { - regex_concat = regex_concat.concat( - &RegularExpression::Character(Range::new_from_range( + RegularExpression::push_concat_element( + &mut elements, + RegularExpression::Character(CharRange::new_from_range( Char::new(char)..=Char::new(char), )), - true, ); } - Ok(regex_concat) + Ok(match elements.len() { + 1 => elements.pop_front().expect("len() == 1"), + _ => RegularExpression::Concat(elements), + }) } else { Err(EngineError::InvalidCharacterInRegex) } @@ -77,51 +135,122 @@ impl RegularExpression { Ok(RegularExpression::Character(range)) } }, - HirKind::Look(_) => Ok(RegularExpression::new_empty_string()), + HirKind::Look(look) => Self::convert_look(look, at_start, at_end), HirKind::Repetition(repetition) => { let (min, max) = (repetition.min, repetition.max); - Self::convert_to_regex(&repetition.sub).map(|v| v.repeat(min, max)) + // The body can repeat, so it is not at the overall boundary in + // general; an anchor inside it is rejected. + let regex = Self::convert_to_regex(&repetition.sub, simplify, false, false)?; + Ok(if simplify { + regex.repeat(min, max) + } else { + RegularExpression::Repetition(Box::new(regex), min, max) + }) + } + // A capture group does not consume input, so it inherits the + // surrounding boundary context unchanged. + HirKind::Capture(capture) => { + Self::convert_to_regex(&capture.sub, simplify, at_start, at_end) } - HirKind::Capture(capture) => Self::convert_to_regex(&capture.sub), HirKind::Concat(concat) => { - let mut concat_regex = - RegularExpression::Concat(VecDeque::with_capacity(concat.len())); - for c in concat { - let concat_value = Self::convert_to_regex(c)?; - concat_regex = concat_regex.concat(&concat_value, true); + let len = concat.len(); + let mut concat_regex = RegularExpression::Concat(VecDeque::with_capacity(len)); + for (i, c) in concat.iter().enumerate() { + // Only the first element can be at the overall start, and + // only the last at the overall end. + let child_at_start = at_start && i == 0; + let child_at_end = at_end && i + 1 == len; + let concat_value = + Self::convert_to_regex(c, simplify, child_at_start, child_at_end)?; + if simplify { + concat_regex = concat_regex.concat(&concat_value, true); + } else if let RegularExpression::Concat(mut values) = concat_regex { + values.push_back(concat_value); + concat_regex = RegularExpression::Concat(values); + } } Ok(concat_regex) } HirKind::Alternation(alternation) => { - let mut alternation_regex = - RegularExpression::Alternation(Vec::with_capacity(alternation.len())); + let mut branches = Vec::with_capacity(alternation.len()); for a in alternation { - let alternation_value = Self::convert_to_regex(a)?; - alternation_regex = alternation_regex.union(&alternation_value); + // Each branch occupies the alternation's own position, so + // it inherits the boundary context unchanged. + branches.push(Self::convert_to_regex(a, simplify, at_start, at_end)?); + } + if simplify { + // Folds the branches through the incremental accumulator + // (folding via pairwise `union` re-cloned the accumulated + // alternation per branch, quadratic in the branch count) + // and honors the execution deadline per branch, so a + // pathological alternation cannot burn CPU outside the + // profile. + RegularExpression::union_all_bounded(branches.iter()) + } else { + Ok(RegularExpression::Alternation(branches)) + } + } + } + } + + /// Interprets a look-around assertion under the engine's full-string + /// (anchored) matching model. + /// + /// A start-of-text anchor (`^`, `\A`) at the very start of the match and an + /// end-of-text anchor (`$`, `\z`) at the very end are redundant, so they + /// are accepted as the empty string. Anchors anywhere else would constrain + /// matching in a way the engine cannot represent (e.g. `ab$cd`), and word + /// boundaries (`\b`, `\B`) never can, so both are rejected rather than + /// silently changing the language. + fn convert_look(look: &Look, at_start: bool, at_end: bool) -> Result { + match look { + Look::Start | Look::StartLF | Look::StartCRLF => { + if at_start { + Ok(RegularExpression::new_empty_string()) + } else { + Err(EngineError::UnsupportedRegexFeature( + "a start-of-text anchor (^ or \\A) is only supported at the start of the \ + pattern; matching is implicitly anchored to the full string" + .to_string(), + )) } - Ok(alternation_regex) } + Look::End | Look::EndLF | Look::EndCRLF => { + if at_end { + Ok(RegularExpression::new_empty_string()) + } else { + Err(EngineError::UnsupportedRegexFeature( + "an end-of-text anchor ($ or \\z) is only supported at the end of the \ + pattern; matching is implicitly anchored to the full string" + .to_string(), + )) + } + } + _ => Err(EngineError::UnsupportedRegexFeature( + "word boundaries (\\b, \\B) and look-around assertions are not supported" + .to_string(), + )), } } - fn to_range_unicode(class_unicode: &ClassUnicode) -> Range { + fn to_range_unicode(class_unicode: &ClassUnicode) -> CharRange { let mut new_range = Vec::with_capacity(class_unicode.ranges().len()); for range in class_unicode.ranges() { new_range.push(AnyRange::from( Char::new(range.start())..=Char::new(range.end()), )); } - Range::new_from_ranges(&new_range) + CharRange::new_from_ranges(&new_range) } - fn to_range_bytes(class_bytes: &ClassBytes) -> Range { + fn to_range_bytes(class_bytes: &ClassBytes) -> CharRange { let mut new_range = Vec::with_capacity(class_bytes.ranges().len()); for range in class_bytes.ranges() { new_range.push(AnyRange::from( Char::new(range.start() as char)..=Char::new(range.end() as char), )); } - Range::new_from_ranges(&new_range) + CharRange::new_from_ranges(&new_range) } } @@ -129,6 +258,167 @@ impl RegularExpression { mod tests { use crate::regex::RegularExpression; + // Folding an alternation with many shared-affix branches is quadratic in + // the branch count; the parse must honor the execution deadline instead + // of burning CPU outside the profile. + #[test] + fn parse_honors_the_execution_deadline() { + use crate::error::EngineError; + use crate::execution_profile::ExecutionProfileBuilder; + + let branches: Vec = (0..20000).map(|i| format!("x{i:05}y")).collect(); + let pattern = format!("({})", branches.join("|")); + + ExecutionProfileBuilder::new() + .execution_timeout(10) + .build() + .run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + RegularExpression::new(&pattern).unwrap_err() + ); + }); + } + + // Inline flags are rejected (the engine cannot honor them); non-capturing + // groups `(?:...)` are still accepted, and flag-lookalikes inside a + // character class are not mistaken for flags. + #[test] + fn inline_flags_are_rejected() { + use crate::error::EngineError; + + for pattern in ["(?i)a", "a(?m-s)b", "a(?-s)b", "(?i:abc)", "(?x) a b c"] { + assert!( + matches!( + RegularExpression::new(pattern), + Err(EngineError::UnsupportedRegexFeature(_)) + ), + "pattern {pattern:?} with inline flags should be rejected" + ); + } + + // Non-capturing groups without flags are fine. + assert!(RegularExpression::new("(?:ab|c)d").is_ok()); + // `(?i)` inside a character class is a set of literal members, not a + // flag directive, so it must not be rejected. + assert!(RegularExpression::new("[a(?i)]").is_ok()); + } + + // Anchors are accepted only where they are redundant under full-string + // matching (leading `^`, trailing `$`); elsewhere they and word + // boundaries are rejected rather than silently changing the language. + #[test] + fn anchors_and_boundaries() { + use crate::error::EngineError; + + // Redundant anchors are no-ops: `^abc$` == `abc`. + let anchored = RegularExpression::new("^abc$").unwrap(); + let plain = RegularExpression::new("abc").unwrap(); + assert!( + anchored + .to_automaton() + .unwrap() + .equivalent(&plain.to_automaton().unwrap()) + .unwrap() + ); + assert!(RegularExpression::new("^abc").is_ok()); + assert!(RegularExpression::new("abc$").is_ok()); + // Alternation branches carry the boundary context. + assert!(RegularExpression::new("^a|b$").is_ok()); + + // Mid-pattern anchors and word boundaries are rejected. + for pattern in ["ab$cd", "a^b", r"a\bc", r"a\Bc", r"a(^b)c"] { + assert!( + matches!( + RegularExpression::new(pattern), + Err(EngineError::UnsupportedRegexFeature(_)) + ), + "pattern {pattern:?} should be rejected" + ); + } + } + + // A hand-built tree nested past `MAX_NESTING_DEPTH` is rejected at the + // conversion boundary instead of overflowing the stack. + #[test] + fn to_automaton_rejects_too_deeply_nested() { + use crate::error::EngineError; + use regex_charclass::char::Char; + + let mut regex = RegularExpression::Character(crate::CharRange::new_from_range( + Char::new('a')..=Char::new('a'), + )); + for _ in 0..(RegularExpression::MAX_NESTING_DEPTH + 10) { + regex = RegularExpression::Repetition(Box::new(regex), 1, Some(1)); + } + assert!(matches!( + regex.to_automaton(), + Err(EngineError::RegexTooDeeplyNested(_)) + )); + + // A shallow tree still converts fine. + assert!( + RegularExpression::new("(a(b(c)))") + .unwrap() + .to_automaton() + .is_ok() + ); + } + + // The variants are freely constructible (open enum); invalid bounds are + // rejected at the conversion boundary instead. + #[test] + fn to_automaton_rejects_invalid_repetition_bounds() { + use crate::error::EngineError; + + let a = RegularExpression::new("a").unwrap(); + let invalid = RegularExpression::Repetition(Box::new(a.clone()), 5, Some(2)); + assert_eq!( + invalid.to_automaton().unwrap_err(), + EngineError::InvalidRepetitionBounds(5, 2) + ); + + // Nested invalid repetitions are caught by the recursion. + let nested = RegularExpression::Concat([a.clone(), invalid].into()); + assert_eq!( + nested.to_automaton().unwrap_err(), + EngineError::InvalidRepetitionBounds(5, 2) + ); + + // The simplifying combinators must not panic on invalid trees either + // (e.g. the affix factoring of `r{1,0}` must not underflow). + let degenerate = RegularExpression::Repetition(Box::new(a.clone()), 1, Some(0)); + let _ = a.union(°enerate); + let _ = a.concat(°enerate, true); + } + + // Singleton Alternation/Concat wrappers print transparently, so a + // quantifier applied to one must be parenthesized by looking through the + // wrapper: `((.a))*` must print as `(.a)*`, not `.a*` (a different + // language). + #[test] + fn display_parenthesizes_through_singleton_wrappers() { + use regex_charclass::char::Char; + + let dot = RegularExpression::Character(crate::CharRange::total()); + let a = RegularExpression::Character(crate::CharRange::new_from_range( + Char::new('a')..=Char::new('a'), + )); + let wrapped = + RegularExpression::Alternation(vec![RegularExpression::Concat([dot, a].into())]); + let star = RegularExpression::Repetition(Box::new(wrapped), 0, None); + assert_eq!(star.to_string(), "(.a)*"); + + // The printed pattern must denote the same language as the tree. + let reparsed = RegularExpression::parse(&star.to_string(), false).unwrap(); + assert!( + star.to_automaton() + .unwrap() + .equivalent(&reparsed.to_automaton().unwrap()) + .unwrap() + ); + } + #[test] fn test_parse() -> Result<(), String> { assert_parse("abc+"); @@ -246,24 +536,27 @@ mod tests { let regex_parsed = RegularExpression::new(".").unwrap(); let automaton = regex_parsed.to_automaton().unwrap(); - assert!(automaton.match_string("a")); - assert!(automaton.match_string("\t")); - assert!(automaton.match_string("\n")); - assert!(automaton.match_string("\r")); - - let regex_parsed = RegularExpression::new("(?i)a").unwrap(); - let automaton = regex_parsed.to_automaton().unwrap(); + assert!(automaton.is_match("a")); + assert!(automaton.is_match("\t")); + assert!(automaton.is_match("\n")); + assert!(automaton.is_match("\r")); - assert!(automaton.match_string("a")); - assert!(!automaton.match_string("A")); + // Inline flags are rejected rather than silently stripped. + assert!(RegularExpression::new("(?i)a").is_err()); + assert!(RegularExpression::new("a(?i)a(?-s).").is_err()); - let regex_parsed = RegularExpression::new("a(?i)a(?-s).").unwrap(); - let automaton = regex_parsed.to_automaton().unwrap(); + assert!(RegularExpression::new("\\1").is_err()); - assert!(automaton.match_string("aa\n")); - assert!(!automaton.match_string("aAb")); + let two_chars = RegularExpression::new("..") + .unwrap() + .to_automaton() + .unwrap(); + assert!(two_chars.is_match("aé")); + assert!(two_chars.is_match("éa")); + assert!(two_chars.is_match("éé")); + assert!(!two_chars.is_match("é")); + assert!(!two_chars.is_match("aéa")); - assert!(RegularExpression::new("\\1").is_err()); Ok(()) } diff --git a/src/regex/mod.rs b/src/regex/mod.rs index 176612f..9181b5b 100644 --- a/src/regex/mod.rs +++ b/src/regex/mod.rs @@ -1,9 +1,8 @@ use std::{cmp, collections::VecDeque, fmt::Display}; -use crate::Range; -use execution_profile::ThreadLocalParams; +use crate::execution_profile::ExecutionProfile; use regex_charclass::CharacterClass; -use regex_syntax::hir::{Class, ClassBytes, ClassUnicode, Hir, HirKind}; +use regex_syntax::hir::{Class, ClassBytes, ClassUnicode, Hir, HirKind, Look}; use self::fast_automaton::FastAutomaton; @@ -12,85 +11,150 @@ use super::*; mod analyze; mod builder; mod operation; -#[cfg(feature = "serde")] -mod serializer; -/// Represent a regular expression. +/// Represents a regular expression. +/// +/// The variants are public and freely constructible and matchable. Values +/// can also be built with the parser ([`new`](Self::new) / +/// [`parse`](Self::parse)) or the simplifying combinators +/// ([`concat`](Self::concat), [`union`](Self::union), +/// [`repeat`](Self::repeat)). A directly-constructed repetition whose +/// maximum is below its minimum denotes no valid language and is rejected +/// with [`EngineError::InvalidRepetitionBounds`] when converted by +/// [`to_automaton`](Self::to_automaton). +/// +/// ``` +/// use regexsolver::regex::RegularExpression; +/// +/// let regex = RegularExpression::new("a{2,3}").unwrap(); +/// if let RegularExpression::Repetition(inner, min, max) = ®ex { +/// assert_eq!((*min, *max), (2, Some(3))); +/// assert_eq!(inner.to_string(), "a"); +/// } +/// ``` #[derive(Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +#[must_use = "regular expressions are immutable; operations return a new expression"] pub enum RegularExpression { - Character(Range), + /// A single character drawn from the given range; an empty range denotes + /// the empty language `[]`. + Character(CharRange), + /// `r{min,max}`; `None` means unbounded. Expected invariant: `max >= min` + /// when bounded (checked by [`to_automaton`](Self::to_automaton)). Repetition(Box, u32, Option), + /// The concatenation of the parts in order; no parts denotes the empty + /// string `""`. Concat(VecDeque), + /// The union of the parts; no parts denotes the empty language `[]`. Alternation(Vec), } impl Display for RegularExpression { + /// Streams the pattern via an explicit work stack instead of recursion, + /// so printing a pathologically deep hand-built tree cannot overflow the + /// call stack. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let str = match self { - RegularExpression::Character(range) => { - if range.is_empty() { - return write!(f, "[]"); - } - range.to_regex() - } - RegularExpression::Repetition(regular_expression, min, max_opt) => { - let regex_part = regular_expression.to_string(); - let multiplicator_part; - if *min == 0 && max_opt.is_none() { - multiplicator_part = String::from("*"); - } else if *min == 1 && max_opt.is_none() { - multiplicator_part = String::from("+"); - } else if *min == 0 && max_opt.is_some() && max_opt.unwrap() == 1 { - multiplicator_part = String::from("?"); - } else if let Some(max) = max_opt { - if max == min { - multiplicator_part = format!("{{{}}}", max); + enum Frame<'a> { + Node(&'a RegularExpression), + Literal(&'static str), + Quantifier(u32, Option), + } + + // Frames pop in reverse push order, so children are pushed + // right-to-left and trailing literals before them. + let mut stack = vec![Frame::Node(self)]; + while let Some(frame) = stack.pop() { + match frame { + Frame::Literal(literal) => f.write_str(literal)?, + Frame::Quantifier(min, max_opt) => { + if min == 0 && max_opt.is_none() { + write!(f, "*")?; + } else if min == 1 && max_opt.is_none() { + write!(f, "+")?; + } else if min == 0 && max_opt == Some(1) { + write!(f, "?")?; + } else if let Some(max) = max_opt { + if max == min { + write!(f, "{{{max}}}")?; + } else { + write!(f, "{{{min},{max}}}")?; + } } else { - multiplicator_part = format!("{{{},{}}}", min, max); + write!(f, "{{{min},}}")?; } - } else { - multiplicator_part = format!("{{{},}}", min); } - match **regular_expression { - RegularExpression::Repetition(_, _, _) => { - format!("({}){}", regex_part, multiplicator_part) + Frame::Node(RegularExpression::Character(range)) => { + if range.is_empty() { + write!(f, "[]")?; + } else { + write!(f, "{}", range.to_regex())?; } - RegularExpression::Concat(_) => { - format!("({}){}", regex_part, multiplicator_part) + } + Frame::Node(RegularExpression::Repetition(regular_expression, min, max_opt)) => { + stack.push(Frame::Quantifier(*min, *max_opt)); + if RegularExpression::quantifier_needs_parens(regular_expression) { + stack.push(Frame::Literal(")")); + stack.push(Frame::Node(regular_expression)); + stack.push(Frame::Literal("(")); + } else { + stack.push(Frame::Node(regular_expression)); } - _ => format!("{}{}", regex_part, multiplicator_part), } - } - RegularExpression::Concat(concat) => { - let mut sb = String::new(); - for regex in concat.iter() { - sb.push_str(regex.to_string().as_str()); + Frame::Node(RegularExpression::Concat(concat)) => { + stack.extend(concat.iter().rev().map(Frame::Node)); } - sb - } - RegularExpression::Alternation(alternation) => { - if alternation.is_empty() { - return write!(f, "[]"); - } - let mut sb = String::new(); - for i in 0..alternation.len() { - sb.push_str(alternation[i].to_string().as_str()); - if i != alternation.len() - 1 { - sb.push('|'); + Frame::Node(RegularExpression::Alternation(alternation)) => { + match alternation.as_slice() { + [] => write!(f, "[]")?, + [single] => stack.push(Frame::Node(single)), + parts => { + stack.push(Frame::Literal(")")); + for (i, regex) in parts.iter().enumerate().rev() { + stack.push(Frame::Node(regex)); + if i != 0 { + stack.push(Frame::Literal("|")); + } + } + stack.push(Frame::Literal("(")); + } } } - if alternation.len() == 1 { - sb - } else { - format!("({})", sb) - } } - }; - write!(f, "{}", str) + } + Ok(()) } } impl RegularExpression { + /// Whether applying a quantifier to the printed form of `r` requires + /// wrapping it in a group. Singleton `Concat`/`Alternation` wrappers + /// print transparently, so the decision must look through them instead + /// of matching on the direct child's variant — iteratively, since a + /// hand-built tree can chain such wrappers arbitrarily deep. + fn quantifier_needs_parens(mut r: &RegularExpression) -> bool { + loop { + match r { + // Prints as a single char or a [class]: one token. + RegularExpression::Character(..) => return false, + RegularExpression::Repetition(..) => return true, + RegularExpression::Concat(parts) => match parts.len() { + 1 => r = &parts[0], + // Covers both the empty concatenation (which prints as "" + // and needs the explicit group; `()*` is valid but a bare + // `*` is not) and real multi-part concatenations. + _ => return true, + }, + RegularExpression::Alternation(parts) => match parts.len() { + // The empty alternation prints as "[]": one token. + 0 => return false, + 1 => r = &parts[0], + // Multi-part alternations print self-parenthesized. + _ => return false, + }, + } + } + } + + /// Returns `true` if the regular expression matches the empty language. pub fn is_empty(&self) -> bool { match self { RegularExpression::Alternation(alternation) => alternation.is_empty(), @@ -99,6 +163,7 @@ impl RegularExpression { } } + /// Returns `true` if the regular expression matches only the empty string `""`. pub fn is_empty_string(&self) -> bool { match self { RegularExpression::Concat(concat) => concat.is_empty(), @@ -106,6 +171,7 @@ impl RegularExpression { } } + /// Returns `true` if the regular expression matches all possible strings. pub fn is_total(&self) -> bool { match self { RegularExpression::Repetition(regular_expression, min, max_opt) => { @@ -122,33 +188,180 @@ impl RegularExpression { } } + /// The deepest a directly-constructed expression tree may nest before + /// [`to_automaton`](Self::to_automaton) refuses to convert it. Parsed + /// patterns never approach this (`regex-syntax` caps parse nesting far + /// lower); it only bounds the recursion so a pathologically deep hand-built + /// tree returns an error instead of overflowing the stack. + pub const MAX_NESTING_DEPTH: usize = 1000; + + /// Returns an error if the tree nests deeper than [`MAX_NESTING_DEPTH`](Self::MAX_NESTING_DEPTH). + /// + /// Uses an explicit stack (not recursion) so measuring a deep tree cannot + /// itself overflow, and bails as soon as the limit is exceeded. + fn assert_depth_within_limit(&self) -> Result<(), EngineError> { + let mut stack = vec![(self, 1usize)]; + while let Some((node, depth)) = stack.pop() { + if depth > Self::MAX_NESTING_DEPTH { + return Err(EngineError::RegexTooDeeplyNested(Self::MAX_NESTING_DEPTH)); + } + match node { + RegularExpression::Character(_) => {} + RegularExpression::Repetition(inner, _, _) => stack.push((inner, depth + 1)), + RegularExpression::Concat(parts) => { + stack.extend(parts.iter().map(|p| (p, depth + 1))); + } + RegularExpression::Alternation(parts) => { + stack.extend(parts.iter().map(|p| (p, depth + 1))); + } + } + } + Ok(()) + } + + /// Converts the regular expression to an equivalent [`FastAutomaton`]. + #[tracing::instrument(level = "trace", skip_all)] pub fn to_automaton(&self) -> Result { - if self.get_number_of_states_in_nfa() >= ThreadLocalParams::get_max_number_of_states() { - return Err(EngineError::AutomatonHasTooManyStates); + // Both whole-tree checks run once here: a subtree can never nest + // deeper than the tree it came from, and the execution profile's + // thread-locals don't change mid-conversion. + self.assert_depth_within_limit()?; + self.to_automaton_inner(&ExecutionProfile::get()) + } + + fn to_automaton_inner( + &self, + execution_profile: &ExecutionProfile, + ) -> Result { + // The per-node state estimate is load-bearing (a subtree like the + // inner of `big{0,0}` can exceed the budget even when the root's + // estimate doesn't), but is only worth its O(subtree) walk when a + // state limit is actually configured. + if execution_profile.limits_number_of_states() { + execution_profile.assert_max_number_of_states(self.get_number_of_states_in_nfa())?; } + match self { - RegularExpression::Character(range) => FastAutomaton::make_from_range(range), + RegularExpression::Character(range) => Ok(FastAutomaton::new_from_range(range)), RegularExpression::Repetition(regular_expression, min, max_opt) => { - let mut automaton = regular_expression.to_automaton()?; - automaton.repeat(*min, *max_opt)?; + // The variants are freely constructible; invalid bounds are + // rejected at this boundary instead. + if let Some(max) = max_opt + && max < min + { + return Err(EngineError::InvalidRepetitionBounds(*min, *max)); + } + let mut automaton = regular_expression.to_automaton_inner(execution_profile)?; + automaton.repeat_mut(*min, *max_opt)?; Ok(automaton) } RegularExpression::Concat(concat) => { let mut concats = Vec::with_capacity(concat.len()); for c in concat.iter() { - concats.push(c.to_automaton()?); + concats.push(c.to_automaton_inner(execution_profile)?); } - FastAutomaton::concatenate(concats) + FastAutomaton::concat_all(&concats) } RegularExpression::Alternation(alternation) => { - let mut concats = Vec::with_capacity(alternation.len()); + let mut alternates = Vec::with_capacity(alternation.len()); for c in alternation.iter() { - concats.push(c.to_automaton()?); + alternates.push(c.to_automaton_inner(execution_profile)?); } - FastAutomaton::alternation(concats) + FastAutomaton::union_all(&alternates) } } } + + /// Returns a heuristic score for the readability of the pattern. + pub fn evaluate_complexity(&self) -> f64 { + let (score, depth, _) = self.eval_inner(); + score + Self::depth_penalty(depth) + } + + /// Returns: (score, max_depth, contains_repetition) + fn eval_inner(&self) -> (f64, usize, bool) { + match self { + RegularExpression::Character(range) => { + let len = range.to_regex().len() as f64; + // small, capped cost for raw length + let base = 1.0 + 0.05 * len.min(40.0); + (base, 1, false) + } + + RegularExpression::Repetition(inner, min, max_opt) => { + let (inner_score, inner_depth, inner_has_rep) = inner.eval_inner(); + + // multipliers tuned for readability impact + let mut m = match max_opt { + None => 1.6, + Some(max) if max > min => 1.3, + Some(max) if max == min && *min > 1 => 1.1, + _ => 1.0, + }; + + // nested quantifiers like (...+)+ are harder + if inner_has_rep { + m *= 1.5; + } + + (inner_score * m, inner_depth + 1, true) + } + + RegularExpression::Concat(items) => { + let mut sum = 0.0; + let mut max_depth = 0usize; + let mut has_rep = false; + + for (i, it) in items.iter().enumerate() { + let (s, d, h) = it.eval_inner(); + sum += s; + if i > 0 { + // tiny discount: linear sequences are relatively easy to read + sum *= 0.98; + } + if d > max_depth { + max_depth = d; + } + has_rep |= h; + } + + (sum, max_depth + 1, has_rep) + } + + RegularExpression::Alternation(branches) => { + if branches.is_empty() { + return (0.0, 1, false); + } + let mut sum = 0.0; + let mut max_depth = 0usize; + let mut has_rep = false; + + for b in branches { + let (s, d, h) = b.eval_inner(); + sum += s; + if d > max_depth { + max_depth = d; + } + has_rep |= h; + } + + // branching cost: more alternatives = harder to scan + let k = branches.len() as f64; + let multiplier = 1.0 + 0.15 * (k - 1.0); + + (sum * multiplier, max_depth + 1, has_rep) + } + } + } + + fn depth_penalty(depth: usize) -> f64 { + // no penalty up to depth 2, then quadratic growth + if depth <= 2 { + 0.0 + } else { + ((depth - 2) as f64).powi(2) * 0.8 + } + } } #[cfg(test)] @@ -181,4 +394,56 @@ mod tests { assert!(automaton.is_total()); Ok(()) } + + /// Drops a deep chain-shaped tree level by level. `Box`'s drop glue + /// recurses (see [`RegularExpression::MAX_NESTING_DEPTH`]), so the deep + /// trees below must not be dropped whole. + fn drop_chain_iteratively(mut regex: RegularExpression) { + loop { + regex = match regex { + RegularExpression::Repetition(inner, _, _) => *inner, + RegularExpression::Concat(mut parts) if parts.len() == 1 => { + parts.pop_front().expect("len() == 1") + } + RegularExpression::Alternation(mut parts) if parts.len() == 1 => { + parts.pop().expect("len() == 1") + } + _ => return, + }; + } + } + + // Display streams via an explicit work stack: a hand-built tree far + // deeper than any recursive formatter could survive must still print. + #[test] + fn display_does_not_recurse_on_deep_trees() { + const DEPTH: usize = 100_000; + + // `((...(a*)*...)*)*`: every level goes through the parenthesization + // decision and the quantifier path. + let mut regex = RegularExpression::new("a").unwrap(); + for _ in 0..DEPTH { + regex = RegularExpression::Repetition(Box::new(regex), 0, None); + } + let printed = regex.to_string(); + assert_eq!(3 * DEPTH - 1, printed.len()); + assert!(printed.starts_with("(((")); + assert!(printed.ends_with(")*)*)*")); + drop_chain_iteratively(regex); + + // A deep chain of singleton wrappers prints transparently, and the + // quantifier's parenthesization must look through all of them + // without recursing. + let mut regex = RegularExpression::new("ab").unwrap(); + for i in 0..DEPTH { + regex = if i % 2 == 0 { + RegularExpression::Concat(VecDeque::from([regex])) + } else { + RegularExpression::Alternation(vec![regex]) + }; + } + let regex = RegularExpression::Repetition(Box::new(regex), 0, None); + assert_eq!("(ab)*", regex.to_string()); + drop_chain_iteratively(regex); + } } diff --git a/src/regex/operation/concat.rs b/src/regex/operation/concat.rs index 6907d9b..b127bea 100644 --- a/src/regex/operation/concat.rs +++ b/src/regex/operation/concat.rs @@ -1,6 +1,22 @@ use super::*; impl RegularExpression { + /// Returns a regular expression that is the concatenation of all expressions in `regexes`. + #[tracing::instrument(level = "trace", skip_all)] + pub fn concat_all<'a, I: IntoIterator>( + regexes: I, + ) -> RegularExpression { + let mut result = RegularExpression::new_empty_string(); + + for other in regexes { + result = result.concat(other, true); + } + + result + } + + /// Returns a new regular expression representing the concatenation of `self` and `other`; `append_back` determines their order. + #[tracing::instrument(level = "trace", skip(self, other), fields(append_back = append_back))] pub fn concat(&self, other: &RegularExpression, append_back: bool) -> RegularExpression { if self.is_empty() || other.is_empty() { return RegularExpression::new_empty(); @@ -10,35 +26,19 @@ impl RegularExpression { return self.clone(); } - match (self, other) { - (RegularExpression::Concat(_), RegularExpression::Concat(_)) => { - if append_back { - Self::opconcat_concat_and_concat(self, other) - } else { - Self::opconcat_concat_and_concat(other, self) - } - } - (RegularExpression::Concat(_), _) => { - if append_back { - Self::opconcat_concat_and_other(self, other) - } else { - Self::opconcat_other_and_concat(other, self) - } - } - (_, RegularExpression::Concat(_)) => { - if append_back { - Self::opconcat_other_and_concat(self, other) - } else { - Self::opconcat_concat_and_other(other, self) - } - } - (_, _) => { - if append_back { - Self::opconcat_other_and_other(self, other) - } else { - Self::opconcat_other_and_other(other, self) - } + let (front, back) = if append_back { + (self, other) + } else { + (other, self) + }; + + match (front, back) { + (RegularExpression::Concat(..), RegularExpression::Concat(..)) => { + Self::opconcat_concat_and_concat(front, back) } + (RegularExpression::Concat(..), _) => Self::opconcat_concat_and_other(front, back), + (_, RegularExpression::Concat(..)) => Self::opconcat_other_and_concat(front, back), + (_, _) => Self::opconcat_other_and_other(front, back), } } @@ -69,12 +69,14 @@ impl RegularExpression { return merged; } - let mut vec = that_elements.clone(); - let that_index = 0; - - if let Some(merged) = Self::opconcat_can_be_merged(this, &that_elements[that_index]) { - vec[that_index] = merged; + // Clone the surviving elements only: the boundary element is + // either replaced by the merge or kept alongside `this`. + let mut vec: VecDeque; + if let Some(merged) = Self::opconcat_can_be_merged(this, &that_elements[0]) { + vec = that_elements.iter().skip(1).cloned().collect(); + vec.push_front(merged); } else { + vec = that_elements.clone(); vec.push_front(this.clone()); } @@ -101,12 +103,14 @@ impl RegularExpression { return merged; } - let mut vec = this_elements.clone(); + // Clone the surviving elements only (see opconcat_other_and_concat). let this_index = this_elements.len() - 1; - + let mut vec: VecDeque; if let Some(merged) = Self::opconcat_can_be_merged(&this_elements[this_index], that) { - vec[this_index] = merged; + vec = this_elements.iter().take(this_index).cloned().collect(); + vec.push_back(merged); } else { + vec = this_elements.clone(); vec.push_back(that.clone()); } @@ -139,15 +143,17 @@ impl RegularExpression { return merged; } - let mut vec = this_elements.clone(); + // Clone the surviving elements only (see opconcat_other_and_concat). let (this_index, that_index) = (this_elements.len() - 1, 0); - + let mut vec: VecDeque; if let Some(merged) = Self::opconcat_can_be_merged(&this_elements[this_index], &that_elements[that_index]) { - vec[this_index] = merged; + vec = this_elements.iter().take(this_index).cloned().collect(); + vec.push_back(merged); vec.extend(that_elements.iter().skip(1).cloned()); } else { + vec = this_elements.clone(); vec.extend(that_elements.iter().cloned()); } @@ -161,6 +167,45 @@ impl RegularExpression { } } + /// Appends `that` to the fold accumulator `elements`, merging with the + /// back element when possible: the in-place equivalent of + /// `acc = acc.concat(&that, true)` for a char-by-char fold. + /// + /// `that` must be neither the empty language nor the empty string (both + /// are handled by `concat`'s degenerate checks, not here), and a merge + /// between the whole accumulated `Concat` and a single element is + /// structurally impossible, so only the back element needs checking. + pub(crate) fn push_concat_element( + elements: &mut VecDeque, + that: RegularExpression, + ) { + if let Some(back) = elements.back() + && let Some(merged) = Self::opconcat_can_be_merged(back, &that) + { + *elements.back_mut().expect("back() was Some") = merged; + return; + } + elements.push_back(that); + } + + /// Merges the bounds of two adjacent repetitions of the same expression, + /// `r{a,b}r{c,d}` → `r{a+c,b+d}`. Returns `None` ("cannot be merged", + /// falling back to plain concatenation) when an addition would overflow. + fn merge_repetition_bounds( + this_min: u32, + this_max_opt: &Option, + that_min: u32, + that_max_opt: &Option, + ) -> Option<(u32, Option)> { + let new_min = this_min.checked_add(that_min)?; + let new_max_opt = if let (Some(this_max), Some(that_max)) = (this_max_opt, that_max_opt) { + Some(this_max.checked_add(*that_max)?) + } else { + None + }; + Some((new_min, new_max_opt)) + } + fn opconcat_can_be_merged( this: &RegularExpression, that: &RegularExpression, @@ -171,24 +216,15 @@ impl RegularExpression { RegularExpression::Repetition(_, that_min, that_max_opt), ) = (this, that) { - let new_min = this_min + that_min; - let new_max_opt = - if let (Some(this_max), Some(that_max)) = (this_max_opt, that_max_opt) { - Some(this_max + that_max) - } else { - None - }; - Some(RegularExpression::Repetition( - this_regex.clone(), - new_min, - new_max_opt, - )) + let (new_min, new_max_opt) = Self::merge_repetition_bounds( + *this_min, + this_max_opt, + *that_min, + that_max_opt, + )?; + Some(this_regex.repeat(new_min, new_max_opt)) } else { - Some(RegularExpression::Repetition( - Box::new(this.clone()), - 2, - Some(2), - )) + Some(this.repeat(2, Some(2))) } } else if let ( RegularExpression::Repetition(this_regex, this_min, this_max_opt), @@ -196,53 +232,39 @@ impl RegularExpression { ) = (this, that) { if this_regex == that_regex { - let new_min = this_min + that_min; - let new_max_opt = - if let (Some(this_max), Some(that_max)) = (this_max_opt, that_max_opt) { - Some(this_max + that_max) - } else { - None - }; - Some(RegularExpression::Repetition( - this_regex.clone(), - new_min, - new_max_opt, - )) + let (new_min, new_max_opt) = Self::merge_repetition_bounds( + *this_min, + this_max_opt, + *that_min, + that_max_opt, + )?; + Some(this_regex.repeat(new_min, new_max_opt)) } else if let ( RegularExpression::Character(this_range), RegularExpression::Character(that_range), - ) = (*this_regex.clone(), *that_regex.clone()) + ) = (&**this_regex, &**that_regex) { - if this_range.contains_all(&that_range) && that_min == &0 && this_max_opt.is_none() - { - return Some(this.clone()); + if this_range.contains_all(that_range) && that_min == &0 && this_max_opt.is_none() { + Some(this.clone()) } else { - return None; + None } } else { - return None; + None } } else if let RegularExpression::Repetition(this_regex, this_min, this_max_opt) = this { if **this_regex == *that { - let new_min = this_min + 1; - let new_max_opt = this_max_opt.as_ref().map(|this_max| this_max + 1); - Some(RegularExpression::Repetition( - this_regex.clone(), - new_min, - new_max_opt, - )) + let (new_min, new_max_opt) = + Self::merge_repetition_bounds(*this_min, this_max_opt, 1, &Some(1))?; + Some(this_regex.repeat(new_min, new_max_opt)) } else { None } } else if let RegularExpression::Repetition(that_regex, that_min, that_max_opt) = that { if **that_regex == *this { - let new_min = that_min + 1; - let new_max_opt = that_max_opt.as_ref().map(|this_max| this_max + 1); - Some(RegularExpression::Repetition( - that_regex.clone(), - new_min, - new_max_opt, - )) + let (new_min, new_max_opt) = + Self::merge_repetition_bounds(*that_min, that_max_opt, 1, &Some(1))?; + Some(that_regex.repeat(new_min, new_max_opt)) } else { None } @@ -256,6 +278,19 @@ impl RegularExpression { mod tests { use super::*; + // Merging adjacent repetitions whose summed bounds would overflow must + // fall back to plain concatenation instead of overflowing. + #[test] + fn concat_merge_bound_overflow_falls_back_to_concat() { + let a = RegularExpression::new("a").unwrap(); + let big = RegularExpression::Repetition(Box::new(a), u32::MAX, None); + let result = big.concat(&big, true); + assert!(matches!( + &result, + RegularExpression::Concat(parts) if parts.len() == 2 + )); + } + #[test] fn test_concat() -> Result<(), String> { assert_concat("xxx", "x{3}"); diff --git a/src/regex/operation/mod.rs b/src/regex/operation/mod.rs index 2baa587..0147fc9 100644 --- a/src/regex/operation/mod.rs +++ b/src/regex/operation/mod.rs @@ -1,236 +1,6 @@ use super::*; mod concat; +mod repeat; mod simplify; mod union; - -impl RegularExpression { - pub fn repeat(&self, min: u32, max_opt: Option) -> RegularExpression { - if self.is_total() { - return RegularExpression::new_total(); - } else if self.is_empty() { - return RegularExpression::new_empty(); - } else if self.is_empty_string() { - return Self::new_empty_string(); - } else if let Some(max) = max_opt { - if max < min || max == 0 { - return RegularExpression::new_empty_string(); - } else if min == 1 && max == 1 { - return self.clone(); - } - } - - match self { - RegularExpression::Repetition(regular_expression, o_min, o_max_opt) => { - let new_max = if let (Some(max), Some(o_max)) = (max_opt, o_max_opt) { - Some(max * o_max) - } else { - None - }; - - let o_min = *o_min; - if let Some(o_max) = o_max_opt { - let o_max = *o_max; - if o_min <= 1 || max_opt.is_some() && max_opt.unwrap() == min { - RegularExpression::Repetition( - regular_expression.clone(), - min * o_min, - new_max, - ) - } else if o_min == o_max && o_min > 1 { - RegularExpression::Repetition(Box::new(self.clone()), min, max_opt) - } else { - let r = ((o_max as f64) - 1f64) / ((o_max as f64) - (o_min as f64)); - if r > cmp::max(2, min) as f64 { - return RegularExpression::Repetition( - Box::new(self.clone()), - min, - max_opt, - ); - } - - RegularExpression::Repetition( - regular_expression.clone(), - min * o_min, - new_max, - ) - } - } else if o_max_opt.is_none() - || max_opt.is_some() && (max_opt.unwrap() == min || max_opt.unwrap() == 1) - || o_max_opt.is_some() && o_max_opt.unwrap() == 1 - || max_opt.is_none() && o_min == 0 - { - RegularExpression::Repetition(regular_expression.clone(), min * o_min, new_max) - } else { - RegularExpression::Repetition(Box::new(self.clone()), min, max_opt) - } - } - _ => RegularExpression::Repetition(Box::new(self.clone()), min, max_opt), - } - } -} - -#[cfg(test)] -mod tests { - use regex_charclass::{char::Char, irange::RangeSet}; - - use crate::regex::RegularExpression; - - #[test] - fn test_parse_and_simplify() -> Result<(), String> { - assert_parse_and_simplify("(xxx)*", "(x{3})*"); - assert_parse_and_simplify("(x*){3}", "x*"); - assert_parse_and_simplify("(x+)?", "x*"); - assert_parse_and_simplify("(x?)+", "x*"); - assert_parse_and_simplify("(x{0,3})+", "x*"); - assert_parse_and_simplify("(x{2,3})+", "x{2,}"); - assert_parse_and_simplify("(x{7,9})+", "(x{7,9})+"); - assert_parse_and_simplify("(x+)*", "x*"); - assert_parse_and_simplify(".*abc", ".*abc"); - assert_parse_and_simplify(".*a(b|cd)", ".*a(b|cd)"); - assert_parse_and_simplify( - "a(bcfe|bcdg|mkv)*(abc){2,3}(abc){2}", - "a(bc(dg|fe)|mkv)*(abc){4,5}", - ); - assert_parse_and_simplify("((abc|fg)abc|(abc|fg)fg)", "(abc|fg){2}"); - assert_parse_and_simplify("(a{2}|a{3})", "a{2,3}"); - assert_parse_and_simplify("(a|b)", "[ab]"); - assert_parse_and_simplify("(ab|a|cd|b|ef)", "(b|ab?|cd|ef)"); - assert_parse_and_simplify("(ab|ab)", "ab"); - assert_parse_and_simplify("(ab)(ab)(ab)", "(ab){3}"); - assert_parse_and_simplify("aaaabbbbbccc", "a{4}b{5}c{3}"); - assert_parse_and_simplify("((ab))?(ab)(((ab)))((((ab)){3}))", "(ab){5,6}"); - assert_parse_and_simplify("(cd|ab)*(ab|cd)*", "(ab|cd)*"); - assert_parse_and_simplify(".*q(ab|ab|abc|ca)x", ".*q(abc?|ca)x"); - assert_parse_and_simplify("((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}", "(q|(a|ads|a{2}d)*abc.*def.*uif(x|ads|a{2}d)*abc.*oxs.*def(ads|ax|a{2}d)*abc.*def.*ksd){1,100}"); - Ok(()) - } - - fn assert_parse_and_simplify(regex: &str, regex_simplified: &str) { - let regex_parsed = RegularExpression::new(regex).unwrap(); - assert_eq!(regex_simplified, regex_parsed.to_string()); - } - - #[test] - fn test_repeat_simplify() -> Result<(), String> { - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 2, - Some(2), - 3, - Some(3), - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 2, - Some(2), - 2, - Some(4), - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 3, - Some(3), - 0, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 0, - Some(3), - 1, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 1, - Some(2), - 1, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 2, - Some(3), - 1, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 3, - Some(4), - 1, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 7, - Some(8), - 1, - None, - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 0, - None, - 3, - Some(3), - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 1, - None, - 0, - Some(1), - ); - - assert_repeat_simplify( - &RangeSet::new_from_range(Char::new('a')..=Char::new('a')), - 0, - Some(1), - 1, - None, - ); - - Ok(()) - } - - fn assert_repeat_simplify( - range: &RangeSet, - min1: u32, - max1: Option, - min2: u32, - max2: Option, - ) { - let repeat = RegularExpression::Repetition( - Box::new(RegularExpression::Repetition( - Box::new(RegularExpression::Character(range.clone())), - min1, - max1, - )), - min2, - max2, - ); - - let got = RegularExpression::new(&repeat.to_string()).unwrap(); - - println!("{} -> {}", repeat, got); - - let repeat = repeat.to_automaton().unwrap(); - - //repeat.to_dot(); - - let result = got.to_automaton().unwrap(); - - assert!(repeat.is_equivalent_of(&result).unwrap()); - } -} diff --git a/src/regex/operation/repeat.rs b/src/regex/operation/repeat.rs new file mode 100644 index 0000000..21855bf --- /dev/null +++ b/src/regex/operation/repeat.rs @@ -0,0 +1,377 @@ +use super::*; + +impl RegularExpression { + /// Computes the repetition of the expression between `min` and `max_opt` times; if `max_opt` is `None`, the repetition is unbounded. + /// + /// When `max_opt` is below `min` there is no valid repetition count and + /// the result is the empty language, consistently with + /// [`FastAutomaton::repeat`](crate::fast_automaton::FastAutomaton::repeat). + #[tracing::instrument(level = "trace", skip(self), fields(min = min, max_opt = tracing::field::debug(max_opt)))] + pub fn repeat(&self, min: u32, max_opt: Option) -> RegularExpression { + if let Some(max) = max_opt { + if max < min { + return RegularExpression::new_empty(); + } else if max == 0 { + return RegularExpression::new_empty_string(); + } + } + + if self.is_total() { + return RegularExpression::new_total(); + } else if self.is_empty() { + return if min == 0 { + Self::new_empty_string() + } else { + RegularExpression::new_empty() + }; + } else if self.is_empty_string() { + return Self::new_empty_string(); + } else if min == 1 && max_opt == Some(1) { + return self.clone(); + } + + match self { + RegularExpression::Repetition(regular_expression, i_min, i_max_opt) => { + // Only collapse (r{i_min,i_max}){min,max} into + // r{min·i_min,max·i_max} when the bounds are gap-free AND the + // multiplications don't overflow; the nested form is always a + // correct fallback. + if Self::can_simplify_nested_repetition(*i_min, *i_max_opt, min, max_opt) { + let new_min = min.checked_mul(*i_min); + let new_max = match (max_opt, i_max_opt) { + (Some(o_max), Some(i_max)) => o_max.checked_mul(*i_max).map(Some), + // 0·∞ = 0: an inner maximum of 0 pins the product at + // zero no matter how many copies the unbounded outer + // count allows — `(a{0,0})*` is `{""}`, not `a*`. + (None, Some(0)) => Some(Some(0)), + _ => Some(None), + }; + if let (Some(new_min), Some(new_max)) = (new_min, new_max) { + return RegularExpression::Repetition( + regular_expression.clone(), + new_min, + new_max, + ); + } + } + RegularExpression::Repetition(Box::new(self.clone()), min, max_opt) + } + _ => RegularExpression::Repetition(Box::new(self.clone()), min, max_opt), + } + } + + /// Evaluate if the repetition `(r{i_min,i_max_opt}){o_min,o_max_opt}` can be simplified to `r{i_min*o_min,i_max_opt*o_max_opt}`. + fn can_simplify_nested_repetition( + i_min: u32, + i_max_opt: Option, + o_min: u32, + o_max_opt: Option, + ) -> bool { + if let Some(o_max) = o_max_opt + && o_min == o_max + { + return true; + } + + if let Some(i_max) = i_max_opt { + // We check if there is any gap by resolving: + // o_min * i_max >= (o_min + 1) * i_min - 1 + // <=> o_min * (i_max - i_min) >= i_min - 1 + o_min.saturating_mul(i_max.saturating_sub(i_min)) >= i_min.saturating_sub(1) + } else if o_min > 0 { + true + } else { + i_min <= 1 + } + } +} + +#[cfg(test)] +mod tests { + + use regex_charclass::char::Char; + + use crate::{CharRange, regex::RegularExpression}; + + // Huge (but valid) bounds whose nested-repetition product would overflow + // must fall back to the nested form instead of overflowing. + #[test] + fn repeat_bound_overflow_keeps_nested_form() { + let a = RegularExpression::new("a").unwrap(); + let inner = a.repeat(2, Some(2)); // a{2} + let outer = inner.repeat(u32::MAX, Some(u32::MAX)); // 2·u32::MAX overflows + assert!(matches!( + &outer, + RegularExpression::Repetition(r, u32::MAX, Some(u32::MAX)) + if matches!(&**r, RegularExpression::Repetition(..)) + )); + } + + // The count guards must apply before the total/∅/ε receiver shortcuts, + // in the same order as `FastAutomaton::repeat_mut`: `.*{0,0}` is `{""}` + // (not `.*`) and `.*{5,2}` is ∅ (not `.*`). + #[test] + fn repeat_count_guards_precede_receiver_shortcuts() { + let total = RegularExpression::new_total(); + assert!(total.repeat(0, Some(0)).is_empty_string()); + assert!(total.repeat(5, Some(2)).is_empty()); + assert!(total.repeat(0, Some(3)).is_total()); + assert!(total.repeat(2, None).is_total()); + + let empty = RegularExpression::new_empty(); + assert!(empty.repeat(0, Some(0)).is_empty_string()); + assert!(empty.repeat(0, Some(5)).is_empty_string()); + assert!(empty.repeat(0, None).is_empty_string()); + assert!(empty.repeat(2, Some(5)).is_empty()); + assert!(empty.repeat(2, None).is_empty()); + + let empty_string = RegularExpression::new_empty_string(); + assert!(empty_string.repeat(0, Some(0)).is_empty_string()); + assert!(empty_string.repeat(5, Some(2)).is_empty()); + assert!(empty_string.repeat(3, Some(7)).is_empty_string()); + + // Each case must agree with the automaton construction. + for receiver in [&total, &empty, &empty_string] { + for (min, max_opt) in [ + (0, Some(0)), + (0, Some(3)), + (1, Some(1)), + (5, Some(2)), + (0, None), + (2, None), + ] { + let via_regex = receiver.repeat(min, max_opt).to_automaton().unwrap(); + let via_automaton = receiver + .to_automaton() + .unwrap() + .repeat(min, max_opt) + .unwrap(); + assert!( + via_regex.equivalent(&via_automaton).unwrap(), + "{receiver}{{{min},{max_opt:?}}}" + ); + } + } + + // `simplify` goes through `repeat` and must stay language-preserving + // on these edges. + let degenerate = + RegularExpression::Repetition(Box::new(RegularExpression::new_total()), 0, Some(0)); + assert!(degenerate.simplify().is_empty_string()); + } + + // The nested-repetition collapse must treat 0·∞ as 0: an unbounded + // repetition of an inner `r{0,0}` (the language {""}) is still {""}, + // not `r*`. Reachable through public `union` (merging the bounds of + // same-base repetitions) and `simplify` on hand-built trees. + #[test] + fn unbounded_repeat_of_zero_max_repetition_stays_empty_string() { + let a = RegularExpression::new("a").unwrap(); + let inner = RegularExpression::Repetition(Box::new(a), 0, Some(0)); // a{0,0} = {""} + + let star = inner.repeat(0, None); // ({""})* = {""} + let automaton = star.to_automaton().unwrap(); + assert!(automaton.is_match("")); + assert!(!automaton.is_match("a"), "(a{{0,0}})* must not contain 'a'"); + + // The union of two {""}-denoting repetitions over the same base is + // where the collapse used to produce `a*`. + let r1 = RegularExpression::Repetition(Box::new(inner.clone()), 0, None); + let r2 = RegularExpression::Repetition(Box::new(inner), 0, Some(0)); + let union = RegularExpression::union_all([&r1, &r2]); + let automaton = union.to_automaton().unwrap(); + assert!(automaton.is_match("")); + assert!( + !automaton.is_match("a"), + "{{\"\"}} ∪ {{\"\"}} must stay {{\"\"}}" + ); + } + + // r{min,max} with max < min has no valid repetition count: the language + // is empty, consistently with `FastAutomaton::repeat`. + #[test] + fn repeat_with_max_below_min_is_empty() { + let a = RegularExpression::new("a").unwrap(); + assert!(a.repeat(5, Some(2)).is_empty()); + + let automaton = a.to_automaton().unwrap().repeat(5, Some(2)).unwrap(); + assert!(automaton.is_empty()); + } + + #[test] + fn test_parse_and_simplify() -> Result<(), String> { + assert_parse_and_simplify("(xxx)*", "(x{3})*"); + assert_parse_and_simplify("(x*){3}", "x*"); + assert_parse_and_simplify("(x+)?", "x*"); + assert_parse_and_simplify("(x?)+", "x*"); + assert_parse_and_simplify("(x{0,3})+", "x*"); + assert_parse_and_simplify("(x{2,3})+", "x{2,}"); + assert_parse_and_simplify("(x{7,9})+", "(x{7,9})+"); + assert_parse_and_simplify("(x+)*", "x*"); + assert_parse_and_simplify(".*abc", ".*abc"); + assert_parse_and_simplify(".*a(b|cd)", ".*a(b|cd)"); + assert_parse_and_simplify( + "a(bcfe|bcdg|mkv)*(abc){2,3}(abc){2}", + "a(bc(dg|fe)|mkv)*(abc){4,5}", + ); + assert_parse_and_simplify("((abc|fg)abc|(abc|fg)fg)", "(abc|fg){2}"); + assert_parse_and_simplify("(a{2}|a{3})", "a{2,3}"); + assert_parse_and_simplify("(a|b)", "[ab]"); + assert_parse_and_simplify("(ab|a|cd|b|ef)", "(b|ab?|cd|ef)"); + assert_parse_and_simplify("(ab|ab)", "ab"); + assert_parse_and_simplify("(ab)(ab)(ab)", "(ab){3}"); + assert_parse_and_simplify("aaaabbbbbccc", "a{4}b{5}c{3}"); + assert_parse_and_simplify("((ab))?(ab)(((ab)))((((ab)){3}))", "(ab){5,6}"); + assert_parse_and_simplify("(cd|ab)*(ab|cd)*", "(ab|cd)*"); + assert_parse_and_simplify(".*q(ab|ab|abc|ca)x", ".*q(abc?|ca)x"); + assert_parse_and_simplify( + "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q){1,100}", + "(q|(a|ads|a{2}d)*abc.*def.*uif(x|ads|a{2}d)*abc.*oxs.*def(ads|ax|a{2}d)*abc.*def.*ksd){1,100}", + ); + + assert_parse_and_simplify("(a{2,4}){2,4}", "a{4,16}"); + Ok(()) + } + + fn assert_parse_and_simplify(regex: &str, regex_simplified: &str) { + let regex_parsed = RegularExpression::new(regex).unwrap(); + assert_eq!(regex_simplified, regex_parsed.to_string()); + } + + #[test] + fn test_repeat_simplify() -> Result<(), String> { + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 2, + Some(2), + 3, + Some(3), + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 2, + Some(2), + 2, + Some(4), + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 3, + Some(3), + 0, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 0, + Some(3), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 1, + Some(2), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 2, + Some(3), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 3, + Some(4), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 7, + Some(8), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 0, + None, + 3, + Some(3), + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 1, + None, + 0, + Some(1), + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 0, + Some(1), + 1, + None, + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 2, + Some(4), + 2, + Some(4), + ); + + assert_repeat_simplify( + &CharRange::new_from_range(Char::new('a')..=Char::new('a')), + 2, + Some(3), + 2, + Some(2), + ); + + Ok(()) + } + + fn assert_repeat_simplify( + range: &CharRange, + min1: u32, + max1: Option, + min2: u32, + max2: Option, + ) { + let repeat = RegularExpression::Repetition( + Box::new(RegularExpression::Repetition( + Box::new(RegularExpression::Character(range.clone())), + min1, + max1, + )), + min2, + max2, + ); + + let got = RegularExpression::new(&repeat.to_string()).unwrap(); + + println!("{} -> {}", repeat, got); + + let repeat = repeat.to_automaton().unwrap(); + + let result = got.to_automaton().unwrap(); + + assert!(repeat.equivalent(&result).unwrap()); + } +} diff --git a/src/regex/operation/simplify.rs b/src/regex/operation/simplify.rs index ae87087..1c0830c 100644 --- a/src/regex/operation/simplify.rs +++ b/src/regex/operation/simplify.rs @@ -1,32 +1,18 @@ use super::*; impl RegularExpression { + /// Returns a simplified version by eliminating redundant constructs and applying canonical reductions. + #[tracing::instrument(level = "trace", skip_all)] pub fn simplify(&self) -> Self { match self { - RegularExpression::Character(_) => self.clone(), + RegularExpression::Character(..) => self.clone(), RegularExpression::Repetition(regex, min, max_opt) => { - let regex = regex.simplify(); - match regex { - RegularExpression::Repetition( - simplified_regex, - simplified_min, - simplified_max_opt, - ) => { - let new_max = if let (Some(max), Some(simplified_max)) = - (max_opt, simplified_max_opt) - { - Some(max * simplified_max) - } else { - None - }; - RegularExpression::Repetition( - simplified_regex, - min * simplified_min, - new_max, - ) - } - _ => RegularExpression::Repetition(Box::new(regex), *min, *max_opt), - } + // Delegate to `repeat`, which guards the nested-repetition + // collapse with `can_simplify_nested_repetition`. Collapsing + // `(r{a,b}){c,d}` to `r{a*c,b*d}` unconditionally is unsound + // when the step lengths leave a gap (e.g. `(a{3,4}){1,2}` + // would wrongly widen to `a{3,8}`). + regex.simplify().repeat(*min, *max_opt) } RegularExpression::Concat(elements) => { let elements: VecDeque<_> = @@ -39,14 +25,9 @@ impl RegularExpression { regex } RegularExpression::Alternation(elements) => { - let elements: VecDeque<_> = - elements.iter().map(|element| element.simplify()).collect(); + let elements: Vec<_> = elements.iter().map(|element| element.simplify()).collect(); - let mut regex = RegularExpression::new_empty(); - elements - .iter() - .for_each(|element| regex = regex.union(element)); - regex + RegularExpression::union_all(elements.iter()) } } } diff --git a/src/regex/operation/union.rs b/src/regex/operation/union.rs index 8f5c1ae..689a9d3 100644 --- a/src/regex/operation/union.rs +++ b/src/regex/operation/union.rs @@ -3,74 +3,127 @@ use std::collections::BTreeSet; use super::*; impl RegularExpression { + /// Returns a regular expression matching the union of `self` and `other`. + #[tracing::instrument(level = "trace", skip_all)] pub fn union(&self, other: &RegularExpression) -> RegularExpression { + Self::union_all([self, other]) + } + + /// Returns a regular expression that is the union of all expressions in `regexes`. + /// + /// Folding through [`union`](Self::union) directly would clone the whole + /// accumulated alternation once per operand (quadratic in the number of + /// operands); the accumulator below applies the same element-merging + /// rules in place instead. + #[tracing::instrument(level = "trace", skip_all)] + pub fn union_all<'a, I: IntoIterator>( + regexes: I, + ) -> RegularExpression { + let mut accumulator = UnionAccumulator::new(); + + for other in regexes { + accumulator.push(other); + + if accumulator.is_total() { + break; + } + } + + accumulator.finish() + } + + /// Deadline-honoring variant of [`union_all`](Self::union_all) used by + /// the parser: folding an alternation branch into the accumulated result + /// can be expensive (e.g. common-affix extraction against a large + /// accumulated alternation), so the fold checks the execution profile + /// once per operand and aborts with `OperationTimeOutError` past the + /// deadline. + pub(crate) fn union_all_bounded<'a, I: IntoIterator>( + regexes: I, + ) -> Result { + let execution_profile = ExecutionProfile::get(); + let mut accumulator = UnionAccumulator::new(); + + for other in regexes { + execution_profile.assert_not_timed_out()?; + accumulator.push(other); + + if accumulator.is_total() { + break; + } + } + + Ok(accumulator.finish()) + } + + fn union_<'a>(&self, other: &'a RegularExpression) -> Cow<'a, RegularExpression> { if self.is_total() || other.is_total() { - return RegularExpression::new_total(); + return Cow::Owned(RegularExpression::new_total()); } else if self.is_empty() { - return other.clone(); + return Cow::Borrowed(other); } else if other.is_empty() || self == other { - return self.clone(); + return Cow::Owned(self.clone()); } else if other.is_empty_string() { - return self.clone().repeat(0, Some(1)); + return Cow::Owned(self.repeat(0, Some(1))); } else if self.is_empty_string() { - return other.clone().repeat(0, Some(1)); + return Cow::Owned(other.repeat(0, Some(1))); } - match (self, other) { + Cow::Owned(match (self, other) { ( RegularExpression::Character(self_range), RegularExpression::Character(other_range), ) => RegularExpression::Character(self_range.union(other_range)), - (RegularExpression::Character(_), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Character(..), RegularExpression::Repetition(..)) => { Self::opunion_character_and_repetition(self, other) } - (RegularExpression::Character(_), RegularExpression::Concat(_)) => { + (RegularExpression::Character(..), RegularExpression::Concat(..)) => { Self::opunion_character_and_concat(self, other) } - (RegularExpression::Character(_), RegularExpression::Alternation(_)) => { + (RegularExpression::Character(..), RegularExpression::Alternation(..)) => { Self::opunion_character_and_alternation(self, other) } - (RegularExpression::Repetition(_, _, _), RegularExpression::Character(_)) => { + (RegularExpression::Repetition(..), RegularExpression::Character(..)) => { Self::opunion_character_and_repetition(other, self) } - (RegularExpression::Repetition(_, _, _), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Repetition(..), RegularExpression::Repetition(..)) => { Self::opunion_repetition_and_repetition(self, other) } - (RegularExpression::Repetition(_, _, _), RegularExpression::Concat(_)) => { + (RegularExpression::Repetition(..), RegularExpression::Concat(..)) => { Self::opunion_concat_and_repetition(other, self) } - (RegularExpression::Repetition(_, _, _), RegularExpression::Alternation(_)) => { + (RegularExpression::Repetition(..), RegularExpression::Alternation(..)) => { Self::opunion_repetition_and_alternation(self, other) } - (RegularExpression::Concat(_), RegularExpression::Character(_)) => { + (RegularExpression::Concat(..), RegularExpression::Character(..)) => { Self::opunion_character_and_concat(other, self) } - (RegularExpression::Concat(_), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Concat(..), RegularExpression::Repetition(..)) => { Self::opunion_concat_and_repetition(self, other) } - (RegularExpression::Concat(_), RegularExpression::Concat(_)) => { + (RegularExpression::Concat(..), RegularExpression::Concat(..)) => { Self::opunion_common_affixes(self, other) } - (RegularExpression::Concat(_), RegularExpression::Alternation(_)) => { + (RegularExpression::Concat(..), RegularExpression::Alternation(..)) => { Self::opunion_concat_and_alternation(self, other) } - (RegularExpression::Alternation(_), RegularExpression::Character(_)) => { + (RegularExpression::Alternation(..), RegularExpression::Character(..)) => { Self::opunion_character_and_alternation(other, self) } - (RegularExpression::Alternation(_), RegularExpression::Repetition(_, _, _)) => { + (RegularExpression::Alternation(..), RegularExpression::Repetition(..)) => { Self::opunion_repetition_and_alternation(other, self) } - (RegularExpression::Alternation(_), RegularExpression::Concat(_)) => { + (RegularExpression::Alternation(..), RegularExpression::Concat(..)) => { Self::opunion_concat_and_alternation(other, self) } - (RegularExpression::Alternation(self_elements), RegularExpression::Alternation(_)) => { - let mut new_alternation = other.clone(); + (RegularExpression::Alternation(self_elements), RegularExpression::Alternation(..)) => { + let mut accumulator = UnionAccumulator::seeded_from(other); for self_element in self_elements { - new_alternation = new_alternation.union(self_element); + accumulator.push(self_element); } - new_alternation + accumulator.finish() } - } + }) } fn opunion_character_and_repetition( @@ -78,26 +131,22 @@ impl RegularExpression { that_repetition: &RegularExpression, ) -> RegularExpression { if let ( - RegularExpression::Character(_), + RegularExpression::Character(..), RegularExpression::Repetition(that_regex, that_min, that_max_opt), ) = (this_character, that_repetition) { - if this_character == &**that_regex && *that_min <= 2 { - RegularExpression::Repetition( - that_regex.clone(), - cmp::min(1, *that_min), - *that_max_opt, - ) + if this_character == &**that_regex + && *that_min <= 2 + && that_max_opt.is_none_or(|that_max| that_max >= 1) + { + that_regex.repeat(cmp::min(1, *that_min), *that_max_opt) } else { let mut alternate = vec![this_character.clone(), that_repetition.clone()]; alternate.sort_unstable(); RegularExpression::Alternation(alternate) } } else { - panic!( - "Not character and repetition {:?} {:?}", - this_character, that_repetition - ) + panic!("Not character and repetition {this_character:?} {that_repetition:?}") } } @@ -116,17 +165,17 @@ impl RegularExpression { if prefix.is_none() && suffix.is_none() { let mut alternate_elements = vec![self_regex, other_regex]; alternate_elements.sort_unstable(); - RegularExpression::Alternation(alternate_elements) + Cow::Owned(RegularExpression::Alternation(alternate_elements)) } else { - self_regex.union(&other_regex) + self_regex.union_(&other_regex) } } else { - RegularExpression::Repetition(Box::new(self_regex), 0, Some(1)) + Cow::Owned(self_regex.repeat(0, Some(1))) } } else if !other_regex.is_empty_string() { - RegularExpression::Repetition(Box::new(other_regex), 0, Some(1)) + Cow::Owned(other_regex.repeat(0, Some(1))) } else { - RegularExpression::new_empty_string() + Cow::Owned(RegularExpression::new_empty_string()) }; regex = regex.concat(®ex_from_alternate, true); @@ -153,10 +202,10 @@ impl RegularExpression { if let RegularExpression::Character(range) = element { set.insert(RegularExpression::Character(this_range.union(range))); had_character_union = true; - } else if matches!(element, RegularExpression::Repetition(_, _, _)) { + } else if matches!(element, RegularExpression::Repetition(..)) { let repetition = Self::opunion_character_and_repetition(this_character, element); - if matches!(repetition, RegularExpression::Repetition(_, _, _)) { + if matches!(repetition, RegularExpression::Repetition(..)) { set.insert(repetition); had_character_union = true; } else { @@ -179,7 +228,7 @@ impl RegularExpression { this_character: &RegularExpression, that_concat: &RegularExpression, ) -> RegularExpression { - if let (RegularExpression::Character(_), RegularExpression::Concat(that_elements)) = + if let (RegularExpression::Character(..), RegularExpression::Concat(that_elements)) = (this_character, that_concat) { if that_elements.len() == 1 && that_elements[0] == *this_character { @@ -197,16 +246,17 @@ impl RegularExpression { that_repetition: &RegularExpression, ) -> RegularExpression { if let ( - RegularExpression::Concat(_), + RegularExpression::Concat(..), RegularExpression::Repetition(that_regex, that_min, that_max_opt), ) = (this_concat, that_repetition) { - if this_concat == &**that_regex && *that_min <= 2 { - RegularExpression::Repetition( - that_regex.clone(), - cmp::min(1, *that_min), - *that_max_opt, - ) + // See `opunion_character_and_repetition`: the merge is only sound + // when the repetition admits at least one copy. + if this_concat == &**that_regex + && *that_min <= 2 + && that_max_opt.is_none_or(|that_max| that_max >= 1) + { + that_regex.repeat(cmp::min(1, *that_min), *that_max_opt) } else { Self::opunion_common_affixes(this_concat, that_repetition) } @@ -219,16 +269,16 @@ impl RegularExpression { this_concat: &RegularExpression, that_alternation: &RegularExpression, ) -> RegularExpression { - if let (RegularExpression::Concat(_), RegularExpression::Alternation(that_elements)) = + if let (RegularExpression::Concat(..), RegularExpression::Alternation(that_elements)) = (this_concat, that_alternation) { let mut set = BTreeSet::new(); let mut had_concat_union = false; for element in that_elements { - if matches!(element, RegularExpression::Repetition(_, _, _)) { + if matches!(element, RegularExpression::Repetition(..)) { let repetition = Self::opunion_concat_and_repetition(this_concat, element); - if matches!(repetition, RegularExpression::Repetition(_, _, _)) { + if matches!(repetition, RegularExpression::Repetition(..)) { set.insert(repetition); had_concat_union = true; } else { @@ -259,21 +309,29 @@ impl RegularExpression { if this_regex == that_regex { if let (Some(this_max), Some(that_max)) = (this_max_opt, that_max_opt) { if this_min <= that_max && that_min <= this_max - || this_max + 1 == *that_min - || that_max + 1 == *this_min + || this_max.saturating_add(1) == *that_min + || that_max.saturating_add(1) == *this_min { - return RegularExpression::Repetition( - this_regex.clone(), + return this_regex.repeat( cmp::min(*this_min, *that_min), Some(cmp::max(*this_max, *that_max)), ); } } else { - return RegularExpression::Repetition( - this_regex.clone(), - cmp::min(*this_min, *that_min), - None, - ); + // At least one side is unbounded. The union collapses to + // r{min(m1,m2),} only when the ranges overlap or are + // adjacent (i.e. the unbounded side starts no later than + // one past the bounded side's end). Otherwise there is a + // gap (e.g. a? ∪ a{3,} must NOT become a*). + let mergeable = match (this_max_opt, that_max_opt) { + (None, None) => true, + (Some(this_max), None) => *that_min <= this_max.saturating_add(1), + (None, Some(that_max)) => *this_min <= that_max.saturating_add(1), + (Some(_), Some(_)) => unreachable!("handled above"), + }; + if mergeable { + return this_regex.repeat(cmp::min(*this_min, *that_min), None); + } } } @@ -294,39 +352,40 @@ impl RegularExpression { RegularExpression::Alternation(that_elements), ) = (this_repetition, that_alternation) { - if that_alternation == &**this_regex && *this_min <= 2 { - RegularExpression::Repetition( - this_regex.clone(), - cmp::min(1, *this_min), - *this_max_opt, - ) + // See `opunion_character_and_repetition`: the merge is only sound + // when the repetition admits at least one copy. + if that_alternation == &**this_regex + && *this_min <= 2 + && this_max_opt.is_none_or(|this_max| this_max >= 1) + { + this_regex.repeat(cmp::min(1, *this_min), *this_max_opt) } else { let mut set = BTreeSet::new(); let mut had_repetition_union = false; for element in that_elements { - if matches!(element, RegularExpression::Repetition(_, _, _)) { + if matches!(element, RegularExpression::Repetition(..)) { let repetition = Self::opunion_repetition_and_repetition(this_repetition, element); - if matches!(repetition, RegularExpression::Repetition(_, _, _)) { + if matches!(repetition, RegularExpression::Repetition(..)) { set.insert(repetition); had_repetition_union = true; } else { set.insert(element.clone()); } - } else if matches!(element, RegularExpression::Character(_)) { + } else if matches!(element, RegularExpression::Character(..)) { let repetition = Self::opunion_character_and_repetition(element, this_repetition); - if matches!(repetition, RegularExpression::Repetition(_, _, _)) { + if matches!(repetition, RegularExpression::Repetition(..)) { set.insert(repetition); had_repetition_union = true; } else { set.insert(element.clone()); } - } else if matches!(element, RegularExpression::Concat(_)) { + } else if matches!(element, RegularExpression::Concat(..)) { let repetition = Self::opunion_concat_and_repetition(element, this_repetition); - if matches!(repetition, RegularExpression::Repetition(_, _, _)) { + if matches!(repetition, RegularExpression::Repetition(..)) { set.insert(repetition); had_repetition_union = true; } else { @@ -347,13 +406,417 @@ impl RegularExpression { } } +/// Incremental accumulator behind [`RegularExpression::union_all`] and the +/// alternation ∪ alternation arm of `union_`. +/// +/// While the accumulated result is not a multi-element alternation, operands +/// fold through the pairwise `union_` rules unchanged. Once it is one, the +/// exact per-element merge rules of the `opunion_*_and_alternation` helpers +/// are applied in place on a `BTreeSet` instead of rebuilding (and +/// re-cloning) the whole alternation for every operand, which made folding an +/// n-branch alternation quadratic in n. +enum UnionAccumulator { + Expression(RegularExpression), + Alternation(AlternationSet), +} + +/// The elements of an accumulated alternation, with counts of the element +/// kinds an incoming operand could merge with, so pushes that cannot merge +/// with anything (the common case: long alternations of distinct literals) +/// skip the merge scan entirely. +#[derive(Default)] +struct AlternationSet { + elements: BTreeSet, + characters: usize, + repetitions: usize, +} + +impl AlternationSet { + fn from_elements(elements: impl IntoIterator) -> Self { + let mut set = AlternationSet::default(); + for element in elements { + set.insert(element); + } + set + } + + fn insert(&mut self, element: RegularExpression) { + let is_character = matches!(element, RegularExpression::Character(..)); + let is_repetition = matches!(element, RegularExpression::Repetition(..)); + if self.elements.insert(element) { + self.characters += usize::from(is_character); + self.repetitions += usize::from(is_repetition); + } + } + + fn remove(&mut self, element: &RegularExpression) { + if self.elements.remove(element) { + match element { + RegularExpression::Character(..) => self.characters -= 1, + RegularExpression::Repetition(..) => self.repetitions -= 1, + _ => {} + } + } + } + + /// Applies the `(old, new)` element merges collected by a scan; when no + /// merge happened, the incoming operand joins the alternation as its own + /// element, mirroring the `had_*_union` bookkeeping of the + /// `opunion_*_and_alternation` helpers. + fn apply( + &mut self, + replacements: Vec<(RegularExpression, RegularExpression)>, + fallback: &RegularExpression, + ) { + if replacements.is_empty() { + self.insert(fallback.clone()); + } else { + for (old, _) in &replacements { + self.remove(old); + } + for (_, new) in replacements { + self.insert(new); + } + } + } +} + +impl UnionAccumulator { + fn new() -> Self { + Self::Expression(RegularExpression::new_empty()) + } + + /// Starts from an existing expression, exactly as if it had been pushed + /// onto a fresh accumulator. + fn seeded_from(regex: &RegularExpression) -> Self { + let mut accumulator = Self::new(); + accumulator.push(regex); + accumulator + } + + fn from_expression(regex: RegularExpression) -> Self { + match regex { + // `new_empty()` is the zero-element alternation and must stay an + // `Expression` so the `is_empty` arm of `union_` keeps firing; + // singleton alternations behave like their element under the + // pairwise rules and are left untouched too. + RegularExpression::Alternation(elements) if elements.len() > 1 => { + Self::Alternation(AlternationSet::from_elements(elements)) + } + regex => Self::Expression(regex), + } + } + + fn push(&mut self, other: &RegularExpression) { + match self { + Self::Expression(regex) => { + let result = regex.union_(other).into_owned(); + *self = Self::from_expression(result); + } + Self::Alternation(set) => { + if let RegularExpression::Alternation(other_elements) = other { + // The `union_` checks that apply to an alternation + // operand: ∅ contributes nothing, and neither does an + // operand equal to the accumulator. + if other_elements.is_empty() + || (other_elements.len() == set.elements.len() + && other_elements.iter().eq(set.elements.iter())) + { + return; + } + // Mirror the alternation ∪ alternation arm of `union_`: + // restart from `other` and re-push the accumulated + // elements (a collapse mid-way, e.g. to Σ*, then keeps + // folding the remaining elements through the pairwise + // rules, exactly like the original left-fold). + let previous = std::mem::take(set); + *self = Self::from_expression(other.clone()); + for element in previous.elements { + self.push(&element); + } + } else if let Some(collapsed) = Self::push_into_set(set, other) { + *self = Self::from_expression(collapsed); + } + } + } + } + + /// Unions a non-alternation operand into the accumulated elements in + /// place. Returns `Some(result)` when the union collapses to something + /// other than the updated alternation (Σ*, or the whole alternation under + /// a quantifier). + fn push_into_set( + set: &mut AlternationSet, + other: &RegularExpression, + ) -> Option { + // The checks at the top of `union_`, specialized to a multi-element + // alternation accumulator (which is structurally never ∅, {""}, nor + // total). + if other.is_total() { + return Some(RegularExpression::new_total()); + } + if other.is_empty() { + return None; + } + if other.is_empty_string() { + // L ∪ {""} = L?, exactly like the empty-string arm of `union_`. + return Some(Self::materialize(set).repeat(0, Some(1))); + } + + match other { + RegularExpression::Character(..) => { + Self::union_character_into_set(set, other); + None + } + RegularExpression::Repetition(base, min, max_opt) => { + // The whole accumulator is the repetition's base: + // `(a|b) ∪ (a|b){1,2}` collapses to `(a|b){1,2}`. Mirrors + // `opunion_repetition_and_alternation`, including the + // `max >= 1` guard that keeps `r{0,0}` (= {""}) from + // absorbing the alternation. + if *min <= 2 + && max_opt.is_none_or(|max| max >= 1) + && matches!(&**base, RegularExpression::Alternation(elements) + if elements.len() == set.elements.len() + && elements.iter().eq(set.elements.iter())) + { + return Some(base.repeat(cmp::min(1, *min), *max_opt)); + } + Self::union_repetition_into_set(set, other); + None + } + RegularExpression::Concat(..) => { + Self::union_concat_into_set(set, other); + None + } + RegularExpression::Alternation(..) => unreachable!("handled by push"), + } + } + + /// Mirrors `opunion_character_and_alternation`. + fn union_character_into_set(set: &mut AlternationSet, this_character: &RegularExpression) { + let RegularExpression::Character(this_range) = this_character else { + unreachable!("Not character") + }; + + if set.characters == 0 && set.repetitions == 0 { + set.insert(this_character.clone()); + return; + } + + let mut replacements = Vec::new(); + for element in &set.elements { + match element { + RegularExpression::Character(range) => { + replacements.push(( + element.clone(), + RegularExpression::Character(this_range.union(range)), + )); + } + RegularExpression::Repetition(..) => { + let repetition = RegularExpression::opunion_character_and_repetition( + this_character, + element, + ); + if matches!(repetition, RegularExpression::Repetition(..)) { + replacements.push((element.clone(), repetition)); + } + } + _ => {} + } + } + set.apply(replacements, this_character); + } + + /// Mirrors the element loop of `opunion_repetition_and_alternation`. + fn union_repetition_into_set(set: &mut AlternationSet, this_repetition: &RegularExpression) { + let mut replacements = Vec::new(); + for element in &set.elements { + let merged = match element { + RegularExpression::Repetition(..) => { + RegularExpression::opunion_repetition_and_repetition(this_repetition, element) + } + RegularExpression::Character(..) => { + RegularExpression::opunion_character_and_repetition(element, this_repetition) + } + RegularExpression::Concat(..) => { + RegularExpression::opunion_concat_and_repetition(element, this_repetition) + } + _ => continue, + }; + if matches!(merged, RegularExpression::Repetition(..)) { + replacements.push((element.clone(), merged)); + } + } + set.apply(replacements, this_repetition); + } + + /// Mirrors the element loop of `opunion_concat_and_alternation`. + fn union_concat_into_set(set: &mut AlternationSet, this_concat: &RegularExpression) { + if set.repetitions == 0 { + set.insert(this_concat.clone()); + return; + } + + let mut replacements = Vec::new(); + for element in &set.elements { + if matches!(element, RegularExpression::Repetition(..)) { + let merged = RegularExpression::opunion_concat_and_repetition(this_concat, element); + if matches!(merged, RegularExpression::Repetition(..)) { + replacements.push((element.clone(), merged)); + } + } + } + set.apply(replacements, this_concat); + } + + fn materialize(set: &AlternationSet) -> RegularExpression { + RegularExpression::Alternation(set.elements.iter().cloned().collect()) + } + + fn is_total(&self) -> bool { + matches!(self, Self::Expression(regex) if regex.is_total()) + } + + fn finish(self) -> RegularExpression { + match self { + Self::Expression(regex) => regex, + Self::Alternation(set) => { + RegularExpression::Alternation(set.elements.into_iter().collect()) + } + } + } +} + #[cfg(test)] mod tests { use super::*; + // With an unbounded side, merging two repetitions is only sound when the + // unbounded range starts no later than one past the bounded range's end; + // otherwise there is a gap (e.g. `a? ∪ a{3,}` must not become `a*`, since + // `a{2}` is in neither operand). + #[test] + fn union_does_not_merge_gapped_repetitions() { + let union = |x: &str, y: &str| { + RegularExpression::parse(x, false) + .unwrap() + .union(&RegularExpression::parse(y, false).unwrap()) + .to_string() + }; + + // Gapped: must stay alternations. + assert_eq!("(a?|a{3,})", union("a?", "a{3,}")); + assert_eq!("(a?|a{3,})", union("a{3,}", "a?")); + assert_eq!("(a{2}|a{5,})", union("a{2}", "a{5,}")); + + // Overlapping or adjacent: still merge. + assert_eq!("a*", union("a?", "a{2,}")); + assert_eq!("a{2,}", union("a{2}", "a{3,}")); + assert_eq!("a*", union("a*", "a{3,}")); + assert_eq!("a{3,}", union("a{3,}", "a{5,}")); + } + + // Merging `r ∪ r{0,0}` must keep both operands (= `r?`), not collapse to + // `r{0,0}` (= {""}) and silently drop the other operand. + #[test] + fn union_with_zero_repetition_keeps_both_operands() { + let equivalent = |result: &RegularExpression, expected: &str| { + let expected = RegularExpression::new(expected).unwrap(); + result + .to_automaton() + .unwrap() + .equivalent(&expected.to_automaton().unwrap()) + .unwrap() + }; + + // Character ∪ repetition. + let a = RegularExpression::new("a").unwrap(); + let a_zero = RegularExpression::Repetition(Box::new(a.clone()), 0, Some(0)); + assert!(equivalent(&a.union(&a_zero), "a?")); + assert!(equivalent(&a_zero.union(&a), "a?")); + + // Concat ∪ repetition. + let ab = RegularExpression::new("ab").unwrap(); + let ab_zero = RegularExpression::Repetition(Box::new(ab.clone()), 0, Some(0)); + assert!(equivalent(&ab.union(&ab_zero), "(ab)?")); + + // Alternation ∪ repetition of the whole alternation. + let alt = RegularExpression::new("(ab|cd)").unwrap(); + let alt_zero = RegularExpression::Repetition(Box::new(alt.clone()), 0, Some(0)); + assert!(equivalent(&alt.union(&alt_zero), "(ab|cd)?")); + + // Degenerate bounds (`a{2,0}` is the empty language) must not absorb + // the other operand either. The result cannot be converted (degenerate + // bounds are rejected by `to_automaton`), so check structurally that + // `a` is still there. + let a_degenerate = RegularExpression::Repetition(Box::new(a.clone()), 2, Some(0)); + let result = a.union(&a_degenerate); + if let RegularExpression::Alternation(elements) = &result { + assert!(elements.contains(&a), "`a` was dropped from {result}"); + } else { + panic!("expected an alternation, got {result}"); + } + } + + // Extracting common affixes from repetitions with degenerate hand-built + // bounds (`a{5,2}`, the empty language) must not underflow when the other + // side is unbounded with the same minimum. + #[test] + fn union_does_not_underflow_on_degenerate_repetition_bounds() { + let a = RegularExpression::new("a").unwrap(); + let unbounded = RegularExpression::Concat(VecDeque::from([RegularExpression::Repetition( + Box::new(a.clone()), + 5, + None, + )])); + let degenerate = RegularExpression::Repetition(Box::new(a), 5, Some(2)); + + // Must not panic, in either order, and must keep both operands + // (degenerate bounds are rejected by `to_automaton`, so the check is + // structural). + for result in [unbounded.union(°enerate), degenerate.union(&unbounded)] { + if let RegularExpression::Alternation(elements) = &result { + assert_eq!(2, elements.len(), "unexpected shape: {result}"); + } else { + panic!("expected an alternation, got {result}"); + } + } + } + + // `union_all` folds through the incremental accumulator; distinct + // branches must all be kept and stay queryable. + #[test] + fn union_all_keeps_distinct_branches() { + // Vary the first and last character so no common affix is extracted + // and the branches genuinely accumulate as alternation elements. + let letter = |i: usize| (b'a' + (i % 26) as u8) as char; + let branch = |i: usize| format!("{}{i:03}{}", letter(i), letter(i + 1)); + let branches: Vec<_> = (0..500) + .map(|i| RegularExpression::new(&branch(i)).unwrap()) + .collect(); + let union = RegularExpression::union_all(branches.iter()); + + if let RegularExpression::Alternation(elements) = &union { + assert_eq!(500, elements.len()); + } else { + panic!("expected an alternation, got {union}"); + } + let automaton = union.to_automaton().unwrap(); + assert!(automaton.is_match(&branch(0))); + assert!(automaton.is_match(&branch(42))); + assert!(automaton.is_match(&branch(499))); + assert!(!automaton.is_match("zzz")); + } + #[test] fn test_union() -> Result<(), String> { assert_union("(a+|a+b)", "a+b?"); + assert_union("(a+|a*)", "a*"); + assert_union("(a?|a{0,2})", "a{0,2}"); + assert_union("(a{2,4}|a{1,3})", "a{1,4}"); + assert_union("(a{1,2}|a{3,4})", "a{1,4}"); + assert_union("(a{3,4}|a{1,2})", "a{1,4}"); Ok(()) } diff --git a/src/regex/serializer.rs b/src/regex/serializer.rs deleted file mode 100644 index 83fd99f..0000000 --- a/src/regex/serializer.rs +++ /dev/null @@ -1,28 +0,0 @@ -use serde::{de, Deserializer, Serializer}; - -use super::*; - -impl serde::Serialize for RegularExpression { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> serde::Deserialize<'de> for RegularExpression { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let regex_string = match String::deserialize(deserializer) { - Ok(str) => str, - Err(err) => return Err(err), - }; - match RegularExpression::new(®ex_string) { - Ok(regex) => Ok(regex), - Err(err) => Err(de::Error::custom(err.to_string())), - } - } -} diff --git a/src/tokenizer/embed_automaton.rs b/src/tokenizer/embed_automaton.rs deleted file mode 100644 index 79697dd..0000000 --- a/src/tokenizer/embed_automaton.rs +++ /dev/null @@ -1,213 +0,0 @@ -use token::TokenError; - -use crate::{error::EngineError, fast_automaton::condition::Condition}; - -use self::token::range_token::RangeToken; - -use super::*; - -impl Tokenizer<'_> { - pub fn to_embedding(&self) -> Vec { - let mut vec = vec![]; - - let mut worklist = VecDeque::new(); - let mut seen = IntSet::default(); - - worklist.push_front(self.automaton.get_start_state()); - - while let Some(current_state) = worklist.pop_back() { - if !vec.is_empty() { - // separator - vec.push(AutomatonToken::SeparatorState) - } - seen.insert(current_state); - - // state - let embedded_state = - AutomatonToken::State(*self.state_to_token.get(¤t_state).unwrap()); - vec.push(embedded_state); - - if self.automaton.is_accepted(¤t_state) { - // accept state - vec.push(AutomatonToken::AcceptState) - } - - for (to_state, condition) in self - .automaton - .transitions_from_state_enumerate_iter(¤t_state) - { - if condition.is_empty() { - continue; - } - let embedded_state = - AutomatonToken::State(*self.state_to_token.get(to_state).unwrap()); - vec.push(embedded_state); - - if condition.is_total() { - vec.push(AutomatonToken::Range(RangeToken::Total)); - } else { - let range = condition - .to_range(self.automaton.get_spanning_set()) - .expect("It should be possible to convert the condition to range."); - self.range_tokenizer - .range_to_embedding(&range) - .unwrap() - .iter() - .for_each(|&e| { - vec.push(AutomatonToken::Range(e)); - }); - } - - if !seen.contains(to_state) { - worklist.push_front(*to_state); - } - } - } - - vec - } - - pub fn from_embedding(&self, vec: &Vec) -> Result { - let mut automaton = FastAutomaton::new_empty(); - automaton.apply_new_spanning_set(self.automaton.get_spanning_set())?; - - let mut from_state = None; - let mut to_state = None; - let mut range = Range::empty(); - for token in vec { - match token { - AutomatonToken::Range(r) => { - range = range.union(self.range_tokenizer.token_to_range(r).unwrap()); - } - AutomatonToken::State(s) => { - while !automaton.has_state((*s).into()) { - automaton.new_state(); - } - if let Some(fs) = from_state { - if let Some(ts) = to_state { - Self::apply_transition(&mut automaton, fs, ts, &range)?; - range = Range::empty(); - } - to_state = Some((*s).into()); - } else { - from_state = Some((*s).into()); - } - } - AutomatonToken::AcceptState => { - automaton.accept(from_state.unwrap()); - } - AutomatonToken::SeparatorState => { - if let Some(to_state) = to_state { - Self::apply_transition( - &mut automaton, - from_state.unwrap(), - to_state, - &range, - )?; - } - from_state = None; - to_state = None; - range = Range::empty(); - } - _ => return Err(EngineError::TokenError(TokenError::UnknownToken)), - }; - } - if let Some(to_state) = to_state { - Self::apply_transition(&mut automaton, from_state.unwrap(), to_state, &range)?; - } - Ok(automaton) - } - - fn apply_transition( - automaton: &mut FastAutomaton, - from_state: State, - to_state: State, - range: &Range, - ) -> Result<(), EngineError> { - let condition = Condition::from_range(range, automaton.get_spanning_set())?; - automaton.add_transition_to(from_state, to_state, &condition); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use embed_automaton::token::Token; - - use crate::regex::RegularExpression; - - use super::*; - - #[test] - fn test_tokenize() -> Result<(), String> { - assert_embedding_convertion_for_fair_and_ai("(a|b)"); - assert_embedding_convertion_for_fair_and_ai("(|a)"); - assert_embedding_convertion_for_fair_and_ai(".*ab"); - assert_embedding_convertion_for_fair_and_ai("toto"); - assert_embedding_convertion_for_fair_and_ai(".{2,3}"); - assert_embedding_convertion_for_fair_and_ai("q(ab|ca|ab|abc)x"); - assert_embedding_convertion_for_fair_and_ai(".*q(ab|ca|ab|abc)x"); - assert_embedding_convertion_for_fair( - "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q)", - ); - assert_embedding_convertion_for_fair("(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])"); - - Ok(()) - } - - fn assert_embedding_convertion_for_fair(regex: &str) { - assert_embedding_convertion(regex, true); - } - - fn assert_embedding_convertion_for_fair_and_ai(regex: &str) { - assert_embedding_convertion(regex, false); - } - - fn assert_embedding_convertion(regex: &str, ignore_ai: bool) { - let regex = RegularExpression::new(regex).unwrap(); - println!("{}", regex); - - let automaton = regex.to_automaton().unwrap().determinize().unwrap(); - - let tokenizer = Tokenizer::new(&automaton); - let embedding = tokenizer.to_embedding(); - - // FAIR - let embedding_u16 = AutomatonToken::to_fair_tokens(&embedding).unwrap(); - let embedding: Vec = embedding_u16 - .iter() - .map(|&t| AutomatonToken::from_fair_token(t)) - .collect(); - - let unembedded_automaton = tokenizer.from_embedding(&embedding).unwrap(); - - assert!(automaton - .subtraction(&unembedded_automaton) - .unwrap() - .is_empty()); - assert!(unembedded_automaton - .subtraction(&automaton) - .unwrap() - .is_empty()); - - if !ignore_ai { - // AI - let embedding_u8 = AutomatonToken::to_ai_tokens(&embedding).unwrap(); - let embedding: Vec = embedding_u8 - .iter() - .map(|&t| AutomatonToken::from_ai_token(t)) - .collect(); - - let unembedded_automaton = tokenizer.from_embedding(&embedding).unwrap(); - - assert!(automaton - .subtraction(&unembedded_automaton) - .unwrap() - .is_empty()); - assert!(unembedded_automaton - .subtraction(&automaton) - .unwrap() - .is_empty()); - } - } -} diff --git a/src/tokenizer/embed_regex.rs b/src/tokenizer/embed_regex.rs deleted file mode 100644 index cb581e6..0000000 --- a/src/tokenizer/embed_regex.rs +++ /dev/null @@ -1,307 +0,0 @@ -use token::TokenError; - -use crate::regex::RegularExpression; - -use self::token::regex_token::RegexToken; - -use super::*; - -impl Tokenizer<'_> { - pub fn to_regex_embedding(&self, regex: &RegularExpression) -> Vec { - let mut vec = self.to_regex_embedding_vec(regex); - - Self::append_counter_if_necessary(&mut vec); - - vec - } - - fn append_counter_if_necessary(vec: &mut Vec) { - if let Some(last) = vec.last() { - match last { - RegexToken::RepetitionNone => {} - RegexToken::Repetition(_) => {} - RegexToken::EndGroup => {} - RegexToken::StartGroup => {} - RegexToken::Alternation => {} - RegexToken::Error => todo!(), - _ => { - vec.push(RegexToken::Repetition(1)); - } - }; - } - } - - fn to_regex_embedding_vec(&self, regex: &RegularExpression) -> Vec { - let mut vec = vec![]; - - match regex { - RegularExpression::Character(range) => { - self.range_tokenizer - .range_to_embedding(range) - .unwrap() - .into_iter() - .for_each(|t| vec.push(RegexToken::Range(t))); - } - RegularExpression::Repetition(regex, min, max_opt) => { - if matches!( - **regex, - RegularExpression::Repetition(_, _, _) | RegularExpression::Concat(_) - ) { - vec.push(RegexToken::StartGroup); - vec.extend(self.to_regex_embedding_vec(regex)); - vec.push(RegexToken::EndGroup); - } else { - vec.extend(self.to_regex_embedding_vec(regex)); - } - - vec.push(RegexToken::Repetition(*min as u16)); - - if let Some(max) = max_opt { - if max != min { - vec.push(RegexToken::Repetition(*max as u16)); - } - } else { - vec.push(RegexToken::RepetitionNone); - } - } - RegularExpression::Concat(elements) => { - for element in elements { - vec.extend(self.to_regex_embedding_vec(element)); - Self::append_counter_if_necessary(&mut vec); - } - } - RegularExpression::Alternation(elements) => { - vec.push(RegexToken::StartGroup); - - for i in 0..elements.len() { - let element = &elements[i]; - vec.extend(self.to_regex_embedding_vec(element)); - Self::append_counter_if_necessary(&mut vec); - if i < elements.len() - 1 { - vec.push(RegexToken::Alternation); - } - } - - vec.push(RegexToken::EndGroup); - } - } - - vec - } - - pub fn from_regex_embedding( - &self, - vec: &[RegexToken], - ) -> Result { - let mut regex_groups = vec![(RegularExpression::new_empty_string(), false)]; - let mut current_range: Option = None; - let mut current_min = None; - for i in 0..vec.len() { - let token = vec[i]; - let current_group = regex_groups.len() - 1; - match token { - RegexToken::Range(range_token) => { - let range = self.range_tokenizer.token_to_range(&range_token).unwrap(); - if let Some(curr_range) = ¤t_range { - current_range = Some(curr_range.union(range)); - } else { - current_range = Some(range.clone()); - } - } - RegexToken::StartGroup => { - regex_groups.push((RegularExpression::new_empty_string(), false)); - } - RegexToken::EndGroup => { - if current_group == 0 { - return Err(TokenError::SyntaxError); - } - if i == vec.len() - 1 || !matches!(vec[i + 1], RegexToken::Repetition(_)) { - let alternation: bool = regex_groups[current_group].1; - Self::pop_regex_group(&mut regex_groups, &None, &None); - if alternation { - Self::pop_regex_group(&mut regex_groups, &None, &None); - } - } - } - RegexToken::Alternation => { - if regex_groups[current_group].1 { - Self::pop_regex_group(&mut regex_groups, &None, &None); - } - regex_groups.push((RegularExpression::new_empty_string(), true)); - } - RegexToken::RepetitionNone => { - if current_min.is_some() { - if let Some(range) = ¤t_range { - Self::add_regex( - &mut regex_groups, - ¤t_min, - &None, - &RegularExpression::Character(range.clone()), - false, - ); - current_range = None; - } else { - Self::pop_regex_group(&mut regex_groups, ¤t_min, &None); - } - current_min = None; - } else { - return Err(TokenError::SyntaxError); - } - } - RegexToken::Repetition(count) => { - if current_min.is_some() - || i == vec.len() - 1 - || !matches!(vec[i + 1], RegexToken::Repetition(_)) - && !matches!(vec[i + 1], RegexToken::RepetitionNone) - { - let min; - let max; - if current_min.is_some() { - min = current_min; - max = Some(count as u32); - } else { - min = Some(count as u32); - max = Some(count as u32); - } - if let Some(range) = ¤t_range { - Self::add_regex( - &mut regex_groups, - &min, - &max, - &RegularExpression::Character(range.clone()), - false, - ); - current_range = None; - } else { - Self::pop_regex_group(&mut regex_groups, &min, &max); - } - current_min = None; - } else { - current_min = Some(count as u32); - } - } - _ => return Err(TokenError::UnknownToken), - }; - } - - Ok(regex_groups[0].0.clone()) - } - - fn pop_regex_group( - regex_groups: &mut Vec<(RegularExpression, bool)>, - current_min: &Option, - current_max: &Option, - ) -> bool { - if regex_groups.len() <= 1 { - return false; - } - - let popped_group = regex_groups.pop().unwrap(); - Self::add_regex( - regex_groups, - current_min, - current_max, - &popped_group.0, - popped_group.1, - ); - true - } - - fn add_regex( - regex_groups: &mut [(RegularExpression, bool)], - current_min: &Option, - current_max: &Option, - regex: &RegularExpression, - alternation: bool, - ) { - let current_group = regex_groups.len() - 1; - let regex_to_use = if let Some(min) = current_min { - if min == &1 && current_max.is_some() { - if current_max.unwrap() == 1 { - regex.clone() - } else { - RegularExpression::Repetition(Box::new(regex.clone()), *min, *current_max) - } - } else { - RegularExpression::Repetition(Box::new(regex.clone()), *min, *current_max) - } - } else { - regex.clone() - }; - - if alternation { - regex_groups[current_group].0 = regex_groups[current_group].0.union(®ex_to_use); - } else { - regex_groups[current_group].0 = - regex_groups[current_group].0.concat(®ex_to_use, true); - } - } -} - -#[cfg(test)] -mod tests { - use embed_regex::token::Token; - - use crate::regex::RegularExpression; - - use super::*; - - #[test] - fn test_tokenize() -> Result<(), String> { - assert_embedding_convertion(".*"); - assert_embedding_convertion("(a|b)"); - assert_embedding_convertion("(|a)"); - assert_embedding_convertion(".*ab"); - assert_embedding_convertion("[a-e]{3}"); - assert_embedding_convertion("[a-e]{3}efg"); - assert_embedding_convertion("toto"); - assert_embedding_convertion(".{2,3}"); - assert_embedding_convertion("q(abc?|ca)x"); - assert_embedding_convertion(".*q(abc?|ca)x"); - assert_embedding_convertion("(abc){3,6}"); - assert_embedding_convertion("((|a)abd+){3}"); - /*assert_embedding_convertion( - "((aad|ads|a)*abc.*def.*uif(aad|ads|x)*abc.*oxs.*def(aad|ads|ax)*abc.*def.*ksd|q)", - );*/ - Ok(()) - } - - fn assert_embedding_convertion(regex: &str) { - let regex = RegularExpression::new(regex).unwrap(); - println!("{}", regex); - - let automaton = regex.to_automaton().unwrap().determinize().unwrap(); - //automaton.to_dot(); - - let tokenizer = Tokenizer::new(&automaton); - let embedding = tokenizer.to_regex_embedding(®ex); - - //println!("{:?}", embedding); - - // FAIR - let embedding_u16 = RegexToken::to_fair_tokens(&embedding).unwrap(); - assert_eq!( - embedding, - embedding_u16 - .iter() - .map(|&t| RegexToken::from_fair_token(t)) - .collect::>() - ); - - let unembedded_regex = tokenizer.from_regex_embedding(&embedding).unwrap(); - assert_eq!(regex, unembedded_regex); - - // AI - let embedding_u8 = RegexToken::to_ai_tokens(&embedding).unwrap(); - assert_eq!( - embedding, - embedding_u8 - .iter() - .map(|&t| RegexToken::from_ai_token(t)) - .collect::>() - ); - - let unembedded_regex = tokenizer.from_regex_embedding(&embedding).unwrap(); - assert_eq!(regex, unembedded_regex); - } -} diff --git a/src/tokenizer/embed_regex_operations.rs b/src/tokenizer/embed_regex_operations.rs deleted file mode 100644 index 4dcb19f..0000000 --- a/src/tokenizer/embed_regex_operations.rs +++ /dev/null @@ -1,119 +0,0 @@ -use token::TokenError; - -use crate::regex::RegularExpression; - -use self::token::regex_operations_token::RegexOperationsToken; - -use super::*; - -impl Tokenizer<'_> { - pub fn to_regex_operations_embedding( - &self, - regex_operations: &[(bool, RegularExpression)], - ) -> Vec { - let mut vec = vec![]; - - for (not, regex) in regex_operations { - if !vec.is_empty() { - vec.push(RegexOperationsToken::And); - } - if *not { - vec.push(RegexOperationsToken::Not); - } - - vec.extend( - self.to_regex_embedding(regex) - .into_iter() - .map(RegexOperationsToken::RegexToken), - ); - } - - vec - } - - pub fn from_regex_operations_embedding( - &self, - vec: &[RegexOperationsToken], - ) -> Result, TokenError> { - let mut operations = vec![]; - let mut current_regex_not = false; - let mut current_regex_token = vec![]; - for token in vec { - match token { - RegexOperationsToken::RegexToken(regex_token) => { - current_regex_token.push(*regex_token) - } - RegexOperationsToken::And => { - let regex = self.from_regex_embedding(¤t_regex_token)?; - operations.push((current_regex_not, regex)); - current_regex_not = false; - current_regex_token.clear(); - } - RegexOperationsToken::Not => current_regex_not = true, - RegexOperationsToken::Error => return Err(TokenError::UnknownToken), - }; - } - - if !current_regex_token.is_empty() { - let regex = self.from_regex_embedding(¤t_regex_token)?; - operations.push((current_regex_not, regex)); - } - - Ok(operations) - } -} - -#[cfg(test)] -mod tests { - use embed_regex_operations::token::Token; - - use crate::regex::RegularExpression; - - use super::*; - - #[test] - fn test_tokenize() -> Result<(), String> { - assert_embedding_convertion(&[(false, "(a|b)")]); - assert_embedding_convertion(&[(false, "(|a)")]); - assert_embedding_convertion(&[(false, ".*ab")]); - assert_embedding_convertion(&[(true, "toto")]); - assert_embedding_convertion(&[(false, ".{2,3}")]); - assert_embedding_convertion(&[(false, "q(abc?|ca)x")]); - assert_embedding_convertion(&[(false, ".*q(abc?|ca)x")]); - assert_embedding_convertion(&[(false, "(abc){3,6}")]); - assert_embedding_convertion(&[(true, "((|a)abd+){3}")]); - - assert_embedding_convertion(&[(false, ".*a.*"), (false, ".*b.*"), (true, ".*abc.*")]); - Ok(()) - } - - fn assert_embedding_convertion(operations: &[(bool, &str)]) { - let mut automaton = FastAutomaton::new_total(); - let operations: Vec<(bool, RegularExpression)> = operations - .iter() - .map(|(not, regex)| { - let regex = RegularExpression::new(regex).unwrap(); - automaton = automaton.intersection(®ex.to_automaton().unwrap()).unwrap(); - (*not, regex) - }) - .collect(); - - let tokenizer = Tokenizer::new(&automaton); - let embedding = tokenizer.to_regex_operations_embedding(&operations); - - // AI - let embedding_u8: Vec = RegexOperationsToken::to_ai_tokens(&embedding).unwrap(); - assert_eq!( - embedding, - embedding_u8 - .iter() - .map(|&t| RegexOperationsToken::from_ai_token(t)) - .collect::>() - ); - - let unembedded_operations = tokenizer - .from_regex_operations_embedding(&embedding) - .unwrap(); - assert_eq!(operations, unembedded_operations); - } -} diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs deleted file mode 100644 index 2e3e4ed..0000000 --- a/src/tokenizer/mod.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::{cmp::Ordering, collections::VecDeque, vec}; - -use ahash::HashMapExt; -use crate::fast_automaton::spanning_set::SpanningSet; -use crate::Range; - -use crate::{ - fast_automaton::{FastAutomaton, State}, - IntMap, IntSet, -}; - -use self::{range_tokenizer::RangeTokenizer, token::automaton_token::AutomatonToken}; - -mod embed_automaton; -mod embed_regex; -mod embed_regex_operations; -pub mod range_tokenizer; -pub mod token; - -#[derive(Debug)] -pub struct Tokenizer<'a> { - range_tokenizer: RangeTokenizer<'a>, - automaton: &'a FastAutomaton, - state_to_token: IntMap, -} - -impl Tokenizer<'_> { - pub fn new(automaton: &FastAutomaton) -> Tokenizer<'_> { - let mut worklist = VecDeque::with_capacity(automaton.get_number_of_states()); - let mut seen = IntSet::default(); - - worklist.push_front(automaton.get_start_state()); - - let mut state_counter: u16 = 0; - let mut state_to_token = IntMap::with_capacity(automaton.get_number_of_states()); - - while let Some(current_state) = worklist.pop_back() { - if !seen.insert(current_state) { - continue; - } - - state_to_token.insert(current_state, state_counter); - state_counter += 1; - - automaton - .transitions_from_state_enumerate_iter(¤t_state) - .filter(|(_, c)| !c.is_empty()) - .for_each(|(to_state, _)| { - if !seen.contains(to_state) { - worklist.push_front(*to_state); - } - }); - } - - Tokenizer { - range_tokenizer: RangeTokenizer::new(automaton.get_spanning_set()), - automaton, - state_to_token, - } - } - - pub fn get_number_of_spanning_ranges(&self) -> usize { - self.range_tokenizer.get_number_of_spanning_ranges() - } - - pub fn get_spanning_set(&self) -> &SpanningSet { - self.range_tokenizer.get_spanning_set() - } -} - -#[cfg(test)] -mod tests {} diff --git a/src/tokenizer/range_tokenizer.rs b/src/tokenizer/range_tokenizer.rs deleted file mode 100644 index 3950033..0000000 --- a/src/tokenizer/range_tokenizer.rs +++ /dev/null @@ -1,74 +0,0 @@ -use self::token::range_token::RangeToken; - -use super::*; - -#[derive(Debug)] -pub struct RangeTokenizer<'a> { - spanning_set: &'a SpanningSet, - total: Range, -} - -impl RangeTokenizer<'_> { - pub fn get_spanning_set(&self) -> &SpanningSet { - self.spanning_set - } - - pub fn new(spanning_set: &SpanningSet) -> RangeTokenizer<'_> { - let total = spanning_set.get_rest().complement(); - RangeTokenizer { - spanning_set, - total, - } - } - - pub fn range_to_embedding(&self, range: &Range) -> Option> { - if range == &self.total { - return Some(vec![RangeToken::Total]); - } else if !range.difference(&self.total).is_empty() { - return None; - } - - let mut vec = vec![]; - for (token, base) in self.spanning_set.get_spanning_ranges().enumerate() { - if range.contains_all(base) { - vec.push(RangeToken::Base(token)); - } - } - vec.sort_unstable(); - - Some(vec) - } - - pub fn embedding_to_range(&self, vec: &[RangeToken]) -> Option { - if vec.is_empty() { - return Some(Range::empty()); - } - - let mut range = Range::empty(); - if vec[0] == RangeToken::Total { - return Some(self.total.clone()); - } - - for token in vec { - if let Some(base) = self.token_to_range(token) { - range = range.union(base); - } else { - return None; - } - } - - Some(range) - } - - pub fn token_to_range(&self, token: &RangeToken) -> Option<&Range> { - match token { - RangeToken::Total => Some(&self.total), - RangeToken::Base(b) => self.spanning_set.get_spanning_range(*b), - RangeToken::Error => panic!("error token"), - } - } - - pub fn get_number_of_spanning_ranges(&self) -> usize { - self.spanning_set.get_number_of_spanning_ranges() - } -} diff --git a/src/tokenizer/token/automaton_token.rs b/src/tokenizer/token/automaton_token.rs deleted file mode 100644 index 215ffed..0000000 --- a/src/tokenizer/token/automaton_token.rs +++ /dev/null @@ -1,115 +0,0 @@ -use self::range_token::RangeToken; - -use super::*; - -#[derive(Debug, Eq, PartialEq, Clone, Copy)] -pub enum AutomatonToken { - Range(RangeToken), - State(u16), - AcceptState, - SeparatorState, - Error, -} - -impl Ord for AutomatonToken { - fn cmp(&self, other: &Self) -> Ordering { - (self.to_fair_token().unwrap()).cmp(&other.to_fair_token().unwrap()) - } -} - -impl PartialOrd for AutomatonToken { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl AutomatonToken { - const TK_AI_RANGE: u8 = 0; - const TK_AI_STATE: u8 = Self::TK_AI_RANGE + RangeToken::AI_VOCABULARY_SIZE; - const TK_AI_ACCEPT_STATE: u8 = Self::TK_AI_STATE + Self::AI_MAX_NUMBER_OF_STATES; - const TK_AI_SEPARATOR_STATE: u8 = Self::TK_AI_ACCEPT_STATE + 1; - - pub const AI_MAX_NUMBER_OF_STATES: u8 = 100; - - pub const AI_VOCABULARY_SIZE: u8 = Self::TK_AI_SEPARATOR_STATE + 1; - - const TK_FAIR_RANGE: u16 = 0; - const TK_FAIR_STATE: u16 = Self::TK_FAIR_RANGE + RangeToken::FAIR_VOCABULARY_SIZE; - const TK_FAIR_ACCEPT_STATE: u16 = Self::TK_FAIR_STATE + Self::FAIR_MAX_NUMBER_OF_STATES; - const TK_FAIR_SEPARATOR_STATE: u16 = Self::TK_FAIR_ACCEPT_STATE + 1; - - pub const FAIR_MAX_NUMBER_OF_STATES: u16 = 65_000; - - pub const FAIR_VOCABULARY_SIZE: u16 = Self::TK_FAIR_SEPARATOR_STATE + 1; -} - -impl Token for AutomatonToken { - fn from_ai_token(token: u8) -> AutomatonToken { - if (Self::TK_AI_RANGE..Self::TK_AI_RANGE + RangeToken::AI_VOCABULARY_SIZE).contains(&token) - { - AutomatonToken::Range(RangeToken::from_ai_token(token)) - } else if (Self::TK_AI_STATE..Self::TK_AI_STATE + Self::AI_MAX_NUMBER_OF_STATES) - .contains(&token) - { - AutomatonToken::State((token - Self::TK_AI_STATE) as u16) - } else if token == Self::TK_AI_ACCEPT_STATE { - AutomatonToken::AcceptState - } else if token == Self::TK_AI_SEPARATOR_STATE { - AutomatonToken::SeparatorState - } else { - AutomatonToken::Error - } - } - - fn to_ai_token(&self) -> Result { - Ok(match self { - AutomatonToken::Range(r) => r.to_ai_token()?, - AutomatonToken::State(s) => { - let max = Self::AI_MAX_NUMBER_OF_STATES; - let s = *s as u8; - if s > max { - return Err(TokenError::TokenOutOfBound("State", max.into(), s.into())); - } - s + Self::TK_AI_STATE - } - AutomatonToken::AcceptState => Self::TK_AI_ACCEPT_STATE, - AutomatonToken::SeparatorState => Self::TK_AI_SEPARATOR_STATE, - AutomatonToken::Error => return Err(TokenError::UnknownToken), - }) - } - - fn from_fair_token(token: u16) -> AutomatonToken { - if (Self::TK_FAIR_RANGE..Self::TK_FAIR_RANGE + RangeToken::FAIR_VOCABULARY_SIZE) - .contains(&token) - { - AutomatonToken::Range(RangeToken::from_fair_token(token)) - } else if (Self::TK_FAIR_STATE..Self::TK_FAIR_STATE + Self::FAIR_MAX_NUMBER_OF_STATES) - .contains(&token) - { - AutomatonToken::State(token - Self::TK_FAIR_STATE) - } else if token == Self::TK_FAIR_ACCEPT_STATE { - AutomatonToken::AcceptState - } else if token == Self::TK_FAIR_SEPARATOR_STATE { - AutomatonToken::SeparatorState - } else { - AutomatonToken::Error - } - } - - fn to_fair_token(&self) -> Result { - Ok(match self { - AutomatonToken::Range(r) => r.to_fair_token()?, - AutomatonToken::State(s) => { - let max = Self::FAIR_MAX_NUMBER_OF_STATES; - let s = *s; - if s > max { - return Err(TokenError::TokenOutOfBound("State", max.into(), s.into())); - } - s + Self::TK_FAIR_STATE - } - AutomatonToken::AcceptState => Self::TK_FAIR_ACCEPT_STATE, - AutomatonToken::SeparatorState => Self::TK_FAIR_SEPARATOR_STATE, - AutomatonToken::Error => return Err(TokenError::UnknownToken), - }) - } -} diff --git a/src/tokenizer/token/mod.rs b/src/tokenizer/token/mod.rs deleted file mode 100644 index 2f28e32..0000000 --- a/src/tokenizer/token/mod.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::fmt::Display; - -use super::*; - -pub mod automaton_token; -pub mod range_token; -pub mod regex_operations_token; -pub mod regex_token; - -#[derive(Debug, PartialEq, Eq)] -pub enum TokenError { - TokenOutOfBound(&'static str, usize, usize), - UnknownToken, - SyntaxError, -} - -impl Display for TokenError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TokenError::TokenOutOfBound(token, expected, got) => write!( - f, - "TokenOutOfBound: {token}, expected: {expected}, got: {got}." - ), - TokenError::UnknownToken => write!(f, "UnknownToken"), - TokenError::SyntaxError => write!(f, "SyntaxError"), - } - } -} - -pub trait Token { - fn from_ai_token(token: u8) -> Self; - - fn to_ai_token(&self) -> Result; - - fn to_ai_tokens(tokens: &[Self]) -> Result, TokenError> - where - Self: Sized, - { - let mut vec = Vec::with_capacity(tokens.len()); - for token in tokens { - vec.push(token.to_ai_token()?); - } - Ok(vec) - } - - fn from_fair_token(token: u16) -> Self; - - fn to_fair_token(&self) -> Result; - - fn to_fair_tokens(tokens: &[Self]) -> Result, TokenError> - where - Self: Sized, - { - let mut vec = Vec::with_capacity(tokens.len()); - for token in tokens { - vec.push(token.to_fair_token()?); - } - Ok(vec) - } -} diff --git a/src/tokenizer/token/range_token.rs b/src/tokenizer/token/range_token.rs deleted file mode 100644 index 62a1753..0000000 --- a/src/tokenizer/token/range_token.rs +++ /dev/null @@ -1,92 +0,0 @@ -use super::*; - -#[derive(Debug, Eq, PartialEq, Clone, Copy)] -pub enum RangeToken { - Total, - Base(usize), - Error, -} - -impl RangeToken { - const TK_AI_TOTAL: u8 = 0; - const TK_AI_BASE: u8 = 1; - - pub const AI_MAX_NUMBER_OF_BASES: u8 = 10; - - pub const AI_VOCABULARY_SIZE: u8 = Self::TK_AI_BASE + Self::AI_MAX_NUMBER_OF_BASES + 1; - - const TK_FAIR_TOTAL: u16 = 0; - const TK_FAIR_BASE: u16 = 1; - - pub const FAIR_MAX_NUMBER_OF_BASES: u16 = 127; - - pub const FAIR_VOCABULARY_SIZE: u16 = Self::TK_FAIR_BASE + Self::FAIR_MAX_NUMBER_OF_BASES + 1; -} - -impl Ord for RangeToken { - fn cmp(&self, other: &Self) -> Ordering { - (self.to_fair_token().unwrap()).cmp(&other.to_fair_token().unwrap()) - } -} - -impl PartialOrd for RangeToken { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Token for RangeToken { - fn from_ai_token(token: u8) -> RangeToken { - if token == Self::TK_AI_TOTAL { - RangeToken::Total - } else if (Self::TK_AI_BASE..Self::TK_AI_BASE + Self::AI_MAX_NUMBER_OF_BASES) - .contains(&token) - { - RangeToken::Base((token - Self::TK_AI_BASE) as usize) - } else { - RangeToken::Error - } - } - - fn to_ai_token(&self) -> Result { - Ok(match self { - RangeToken::Total => Self::TK_AI_TOTAL, - RangeToken::Base(b) => { - let max = Self::AI_MAX_NUMBER_OF_BASES; - let b = *b as u8; - if b > max { - return Err(TokenError::TokenOutOfBound("Base", max.into(), b.into())); - } - b + Self::TK_AI_BASE - } - RangeToken::Error => return Err(TokenError::UnknownToken), - }) - } - - fn from_fair_token(token: u16) -> RangeToken { - if token == Self::TK_FAIR_TOTAL { - RangeToken::Total - } else if (Self::TK_FAIR_BASE..Self::TK_FAIR_BASE + Self::FAIR_MAX_NUMBER_OF_BASES) - .contains(&token) - { - RangeToken::Base((token - Self::TK_FAIR_BASE) as usize) - } else { - RangeToken::Error - } - } - - fn to_fair_token(&self) -> Result { - Ok(match self { - RangeToken::Total => Self::TK_FAIR_TOTAL, - RangeToken::Base(b) => { - let max = Self::FAIR_MAX_NUMBER_OF_BASES; - let b = *b as u16; - if b > max { - return Err(TokenError::TokenOutOfBound("Base", max.into(), b.into())); - } - b + Self::TK_FAIR_BASE - } - RangeToken::Error => return Err(TokenError::UnknownToken), - }) - } -} diff --git a/src/tokenizer/token/regex_operations_token.rs b/src/tokenizer/token/regex_operations_token.rs deleted file mode 100644 index 1074f7f..0000000 --- a/src/tokenizer/token/regex_operations_token.rs +++ /dev/null @@ -1,64 +0,0 @@ -use self::regex_token::RegexToken; - -use super::*; - -#[derive(Debug, Eq, PartialEq, Clone, Copy)] -pub enum RegexOperationsToken { - RegexToken(RegexToken), - And, - Not, - Error, -} - -impl Ord for RegexOperationsToken { - fn cmp(&self, other: &Self) -> Ordering { - (self.to_ai_token().unwrap()).cmp(&other.to_ai_token().unwrap()) - } -} - -impl PartialOrd for RegexOperationsToken { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl RegexOperationsToken { - const TK_AI_REGEX_TOKEN: u8 = 0; - const TK_AI_AND: u8 = Self::TK_AI_REGEX_TOKEN + RegexToken::AI_VOCABULARY_SIZE; - const TK_AI_NOT: u8 = Self::TK_AI_AND + 1; - - pub const AI_VOCABULARY_SIZE: u8 = Self::TK_AI_NOT + 1; -} - -impl Token for RegexOperationsToken { - fn from_ai_token(token: u8) -> RegexOperationsToken { - if (Self::TK_AI_REGEX_TOKEN..Self::TK_AI_REGEX_TOKEN + RegexToken::AI_VOCABULARY_SIZE) - .contains(&token) - { - RegexOperationsToken::RegexToken(RegexToken::from_ai_token(token)) - } else if token == Self::TK_AI_AND { - RegexOperationsToken::And - } else if token == Self::TK_AI_NOT { - RegexOperationsToken::Not - } else { - RegexOperationsToken::Error - } - } - - fn to_ai_token(&self) -> Result { - Ok(match self { - RegexOperationsToken::RegexToken(regex_token) => regex_token.to_ai_token()?, - RegexOperationsToken::And => Self::TK_AI_AND, - RegexOperationsToken::Not => Self::TK_AI_NOT, - RegexOperationsToken::Error => return Err(TokenError::UnknownToken), - }) - } - - fn from_fair_token(_: u16) -> RegexOperationsToken { - panic!("A RegexOperationsToken does not have a FAIR representation.") - } - - fn to_fair_token(&self) -> Result { - panic!("A RegexOperationsToken does not have a FAIR representation.") - } -} diff --git a/src/tokenizer/token/regex_token.rs b/src/tokenizer/token/regex_token.rs deleted file mode 100644 index 2f4c2f2..0000000 --- a/src/tokenizer/token/regex_token.rs +++ /dev/null @@ -1,137 +0,0 @@ -use self::range_token::RangeToken; - -use super::*; - -#[derive(Debug, Eq, PartialEq, Clone, Copy)] -pub enum RegexToken { - Range(RangeToken), - StartGroup, - EndGroup, - Alternation, - RepetitionNone, - Repetition(u16), - Error, -} - -impl Ord for RegexToken { - fn cmp(&self, other: &Self) -> Ordering { - (self.to_fair_token().unwrap()).cmp(&other.to_fair_token().unwrap()) - } -} - -impl PartialOrd for RegexToken { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl RegexToken { - const TK_AI_RANGE: u8 = 0; - const TK_AI_START_GROUP: u8 = Self::TK_AI_RANGE + RangeToken::AI_VOCABULARY_SIZE; - const TK_AI_END_GROUP: u8 = Self::TK_AI_START_GROUP + 1; - const TK_AI_ALTERNATION: u8 = Self::TK_AI_END_GROUP + 1; - const TK_AI_REPETITION_NONE: u8 = Self::TK_AI_ALTERNATION + 1; - const TK_AI_REPETITION: u8 = Self::TK_AI_REPETITION_NONE + 1; - - pub const AI_MAX_NUMBER_OF_REPETITION: u8 = 10; - - pub const AI_VOCABULARY_SIZE: u8 = - Self::TK_AI_REPETITION + Self::AI_MAX_NUMBER_OF_REPETITION + 1; - - const TK_FAIR_RANGE: u16 = 0; - const TK_FAIR_START_GROUP: u16 = Self::TK_FAIR_RANGE + RangeToken::FAIR_VOCABULARY_SIZE; - const TK_FAIR_END_GROUP: u16 = Self::TK_FAIR_START_GROUP + 1; - const TK_FAIR_ALTERNATION: u16 = Self::TK_FAIR_END_GROUP + 1; - const TK_FAIR_REPETITION_NONE: u16 = Self::TK_FAIR_ALTERNATION + 1; - const TK_FAIR_REPETITION: u16 = Self::TK_FAIR_REPETITION_NONE + 1; - - pub const FAIR_MAX_NUMBER_OF_REPETITION: u16 = 1024; - - pub const FAIR_VOCABULARY_SIZE: u16 = - Self::TK_FAIR_REPETITION + Self::FAIR_MAX_NUMBER_OF_REPETITION + 1; -} - -impl Token for RegexToken { - fn from_ai_token(token: u8) -> RegexToken { - if (Self::TK_AI_RANGE..Self::TK_AI_RANGE + RangeToken::AI_VOCABULARY_SIZE).contains(&token) - { - RegexToken::Range(RangeToken::from_ai_token(token)) - } else if token == Self::TK_AI_START_GROUP { - RegexToken::StartGroup - } else if token == Self::TK_AI_END_GROUP { - RegexToken::EndGroup - } else if token == Self::TK_AI_ALTERNATION { - RegexToken::Alternation - } else if token == Self::TK_AI_REPETITION_NONE { - RegexToken::RepetitionNone - } else if (Self::TK_AI_REPETITION - ..Self::TK_AI_REPETITION + Self::AI_MAX_NUMBER_OF_REPETITION) - .contains(&token) - { - RegexToken::Repetition((token - Self::TK_AI_REPETITION) as u16) - } else { - RegexToken::Error - } - } - - fn to_ai_token(&self) -> Result { - Ok(match self { - RegexToken::Range(r) => r.to_ai_token()?, - RegexToken::StartGroup => Self::TK_AI_START_GROUP, - RegexToken::EndGroup => Self::TK_AI_END_GROUP, - RegexToken::Alternation => Self::TK_AI_ALTERNATION, - RegexToken::RepetitionNone => Self::TK_AI_REPETITION_NONE, - RegexToken::Repetition(r) => { - let max = Self::AI_MAX_NUMBER_OF_REPETITION; - let r = *r as u8; - if r > max { - return Err(TokenError::TokenOutOfBound("Repetition", max.into(), r.into())); - } - r + Self::TK_AI_REPETITION - } - RegexToken::Error => return Err(TokenError::UnknownToken), - }) - } - - fn from_fair_token(token: u16) -> RegexToken { - if (Self::TK_FAIR_RANGE..Self::TK_FAIR_RANGE + RangeToken::FAIR_VOCABULARY_SIZE) - .contains(&token) - { - RegexToken::Range(RangeToken::from_fair_token(token)) - } else if token == Self::TK_FAIR_START_GROUP { - RegexToken::StartGroup - } else if token == Self::TK_FAIR_END_GROUP { - RegexToken::EndGroup - } else if token == Self::TK_FAIR_ALTERNATION { - RegexToken::Alternation - } else if token == Self::TK_FAIR_REPETITION_NONE { - RegexToken::RepetitionNone - } else if (Self::TK_FAIR_REPETITION - ..Self::TK_FAIR_REPETITION + Self::FAIR_MAX_NUMBER_OF_REPETITION) - .contains(&token) - { - RegexToken::Repetition(token - Self::TK_FAIR_REPETITION) - } else { - RegexToken::Error - } - } - - fn to_fair_token(&self) -> Result { - Ok(match self { - RegexToken::Range(r) => r.to_fair_token()?, - RegexToken::StartGroup => Self::TK_FAIR_START_GROUP, - RegexToken::EndGroup => Self::TK_FAIR_END_GROUP, - RegexToken::Alternation => Self::TK_FAIR_ALTERNATION, - RegexToken::RepetitionNone => Self::TK_FAIR_REPETITION_NONE, - RegexToken::Repetition(r) => { - let max = Self::FAIR_MAX_NUMBER_OF_REPETITION; - let r = *r; - if r > max { - return Err(TokenError::TokenOutOfBound("Repetition", max.into(), r.into())); - } - r + Self::TK_FAIR_REPETITION - } - RegexToken::Error => return Err(TokenError::UnknownToken), - }) - } -} diff --git a/tests/data/regex.txt b/tests/data/regex.txt index e5fb5df..c65eca9 100644 --- a/tests/data/regex.txt +++ b/tests/data/regex.txt @@ -1,3 +1,5 @@ +(a*,a*)? +(?:\s*,\s*(?:0|1|0?\.\d+))? [\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f] a{2,3} (abc|fg){2} @@ -19,4 +21,59 @@ a+(ba+)* [0-9]+[A-Z]* ù -^\d$ \ No newline at end of file +^\d$ +foo +bar? +baz+ +qux* +quux{3} +quuux{2,5} +quuuux{0,4} +.* +[aeiou] +[^aeiou] +[a-zA-Z0-9] +[\dA-Fa-f] +[\w&&[^_]] +[[:alpha:]]+ +[\p{L}]+ +[0-9]{2,4} +[01]?\d +[1-9][0-9]* +(cat|dog|mouse) +(?:red|green|blue){2} +(gr(a|e)y){1,3} +((ab|cd)ef)+ +(a(b(c|d)e)f)+ +(a|b(c|d(e|f))){2,3} +(?:abc){0,} +(?:abc){1,} +(?:abc){2,5} +a++ +\.\*\?\+\(\)\[\]\{\}\\\| +\u0041\u0042\u0043 +\p{Greek}+ +\p{Sc} +[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[A-Za-z]{2,} +((25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(25[0-5]|2[0-4]\d|[01]?\d?\d) +https?://[^\s/$.?#][^\s]* +\d{4}/\d{2}/\d{2} +\d{1,2}:\d{2}(:\d{2})? +<([A-Za-z][A-Za-z0-9]*)[^>]*?/> +\{(?:[^{}]|\{[^{}]*\})*\} +(?:\d[ -]*?){13,16} +#([A-Fa-f0-9]{8}) +(a|b|c|d|e|f|g|h|i|j){5} +(?:"[^"]*"|[^,]*)(?:,(?:"[^"]*"|[^,]*))* +([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2} +[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12} +[[:alnum:]&&[^0-9]] +[ \t]+ +[\r\n]+ +[^\t\r\n]+ +(a*,a*)* +#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3}) +\{(?:\s*"[^"]*"\s*:\s*(?:\d+|"(...)" )\s*)(?:,\s*"[^"]*"\s*:\s*(?:\d+|"(...)" )\s*)*\} +rgba?\(\s*(?:\d{1,3}\s*,\s*){2}\d{1,3}(?:\s*,\s*(?:0|1|0?\.\d+))?\s*\) +[+-]?(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)? +<\w+(?:\s+\w+(?:="[^"]*")?)*\s*/?> diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1e572a9..37d3ca1 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -4,36 +4,35 @@ use std::{ }; use regex::Regex; -use regexsolver::regex::RegularExpression; +use regexsolver::{fast_automaton::GenerationOptions, regex::RegularExpression}; fn assert_regex(regex: &str) { let re = Regex::new(&format!("(?s)^{}$", regex)).unwrap(); - let regex = RegularExpression::new(regex).unwrap(); + let regex = RegularExpression::parse(regex, true).unwrap(); let automaton = regex.to_automaton().unwrap(); - let strings = automaton.generate_strings(500).unwrap(); + let strings = automaton + .generate_strings(500, 0, GenerationOptions::new()) + .unwrap(); for string in strings { assert!(re.is_match(&string), "'{string}'"); } - assert_eq!( - automaton.get_number_of_states(), - regex.get_number_of_states_in_nfa() - ); - let determinized_automaton = automaton.determinize().unwrap(); - let strings = determinized_automaton.generate_strings(500).unwrap(); + let strings = determinized_automaton + .generate_strings(500, 0, GenerationOptions::new()) + .unwrap(); for string in strings { assert!(re.is_match(&string), "'{string}'"); } - assert!(automaton.is_subset_of(&determinized_automaton).unwrap()); - assert!(determinized_automaton.is_subset_of(&automaton).unwrap()); - assert!(automaton.is_equivalent_of(&determinized_automaton).unwrap()); + assert!(automaton.subset(&determinized_automaton).unwrap()); + assert!(determinized_automaton.subset(&automaton).unwrap()); + assert!(automaton.equivalent(&determinized_automaton).unwrap()); let regex_from_automaton = automaton.to_regex().unwrap(); let automaton_from_regex = regex_from_automaton.to_automaton().unwrap(); - assert!(automaton.is_equivalent_of(&automaton_from_regex).unwrap()); + assert!(automaton.equivalent(&automaton_from_regex).unwrap()); } #[test] @@ -42,7 +41,7 @@ fn test_regular_expression_parsing() { let reader = BufReader::new(file); for regex in reader.lines() { let regex = regex.unwrap(); - println!("{}", ®ex); + println!("{}", regex); assert_regex(®ex); } } diff --git a/tests/proptest_strategies.rs b/tests/proptest_strategies.rs new file mode 100644 index 0000000..06929e2 --- /dev/null +++ b/tests/proptest_strategies.rs @@ -0,0 +1,1222 @@ +//! Property-based tests built on `proptest` strategies that generate random +//! DFAs, NFAs and regular expressions. +//! +//! # Coverage guarantee +//! +//! The strategies are parameterized by a fixed finite alphabet ([`ALPHABET`]) +//! and a maximum number of states ([`MAX_STATES`]). Within those bounds every +//! structure has a strictly positive probability of being generated: +//! +//! * [`arb_dfa`] — every deterministic automaton over the alphabet with +//! `1..=MAX_STATES` states (start state fixed to `0`, any accepting subset, +//! any total transition function) can be produced. A transition function maps +//! each `(state, letter)` to at most one target, which is exactly the +//! definition of a DFA, so the whole DFA space is covered. +//! * [`arb_nfa`] — every nondeterministic automaton is reachable: each ordered +//! `(from, to)` pair may carry any subset of the alphabet letters as its +//! label, plus optional epsilon transitions. Since labels may overlap, this +//! spans all NFAs (and, as a subset, all DFAs). +//! * [`arb_regex`] — every regular expression built from the four +//! [`RegularExpression`] variants up to the configured recursion depth and +//! bound sizes is reachable, including the empty language (`[]`) and `.`. +//! +//! The alphabet is small on purpose so the spaces are finite and the +//! determinization / set operations under test stay cheap. +//! +//! While *coverage* is uniform-in-support, the *distribution* is deliberately +//! shaped — the `inspect::stats` test measures the result and asserts floors: +//! +//! * Per-automaton edge/epsilon/accept densities are sampled, with the accept +//! density centered on ½ (where accepting/rejecting states are hardest to +//! merge, keeping minimal DFAs — and therefore the work done by minimize / +//! equivalence / state elimination — large). +//! * An optional "anchor" state is forced accepting so the empty language is +//! an occasional edge case instead of a fifth of the sample. +//! * A per-automaton acyclic mode (≈⅓ of cases) generates DAGs, whose finite +//! languages exercise the topological-sort paths of the cardinality and +//! max-length analyses that cyclic automata never reach. +//! * The character-class strategy favors single letters so that a `[]` +//! (empty-language) leaf does not collapse most expressions. +//! +//! All weights stay strictly inside (0, 1), preserving the +//! non-null-probability guarantee. + +use proptest::prelude::*; +use regex_charclass::char::Char; +use regexsolver::CharRange; +use regexsolver::cardinality::Cardinality; +use regexsolver::error::EngineError; +use regexsolver::execution_profile::ExecutionProfileBuilder; +use regexsolver::fast_automaton::FastAutomaton; +use regexsolver::regex::RegularExpression; + +/// Fixed alphabet the strategies draw transition labels from. +pub const ALPHABET: &[char] = &['a', 'b']; + +/// Maximum number of states a generated automaton can have. +pub const MAX_STATES: usize = 5; + +/// The single-character range for the `i`-th letter of `alphabet`. +fn letter_over(alphabet: &[char], i: usize) -> CharRange { + let c = Char::new(alphabet[i]); + CharRange::new_from_range(c..=c) +} + +/// The single-character range for the `i`-th [`ALPHABET`] letter. +fn letter(i: usize) -> CharRange { + letter_over(ALPHABET, i) +} + +/// Number of transition-label letters (one per [`ALPHABET`] letter). +fn num_bases() -> usize { + ALPHABET.len() +} + +/// Number of states, biased toward larger automata (the maximum of two uniform +/// draws). Tiny automata are still generated, but the n = 1 space is almost +/// entirely degenerate, so uniform sampling would waste a quarter of all cases +/// on it. +fn arb_num_states() -> impl Strategy { + (1usize..=MAX_STATES, 1usize..=MAX_STATES).prop_map(|(a, b)| a.max(b)) +} + +/// Builds an automaton over `alphabet` from a structural description using +/// only the public API. +/// +/// `accepts[s]` marks state `s` accepting; each `(from, to, mask)` adds a +/// transition whose label is the union of the letters selected by `mask` +/// (`add_transition_from_range` grows the automaton's spanning set as +/// needed); each `(from, to)` in `eps` adds an epsilon transition. The start +/// state is `0`. +fn build_over( + alphabet: &[char], + n: usize, + accepts: &[bool], + char_edges: &[(usize, usize, Vec)], + eps: &[(usize, usize)], +) -> FastAutomaton { + let mut a = FastAutomaton::new_empty(); + for _ in 1..n { + a.new_state(); + } + + for (s, &acc) in accepts.iter().enumerate() { + if acc { + a.accept(s); + } + } + + for (from, to, mask) in char_edges { + let mut range = CharRange::empty(); + for (i, &on) in mask.iter().enumerate() { + if on { + range = range.union(&letter_over(alphabet, i)); + } + } + a.add_transition_from_range(*from, *to, &range) + .expect("adding a union of alphabet letters never fails"); + } + + // Epsilon transitions are added last: `add_epsilon_transition` eagerly + // folds the target's current transitions into the source. + for (from, to) in eps { + a.add_epsilon_transition(*from, *to); + } + + a +} + +/// [`build_over`] with the default [`ALPHABET`]. +fn build( + n: usize, + accepts: &[bool], + char_edges: &[(usize, usize, Vec)], + eps: &[(usize, usize)], +) -> FastAutomaton { + build_over(ALPHABET, n, accepts, char_edges, eps) +} + +/// Strategy producing every DFA over the alphabet with `1..=MAX_STATES` states. +/// +/// Determinism is structural: for each state and each letter we choose at most +/// one target, so transitions leaving a state always carry disjoint labels. +/// +/// An edge density and an accept density are sampled per automaton (both +/// bounded away from 0 and 1, so every DFA keeps a positive probability). +/// Mostly-total transition functions keep the states connected, which makes +/// degenerate (∅ / {""}) languages the exception rather than the rule. The +/// accept density is centered on ½ because that is where accepting/rejecting +/// states are hardest to merge, i.e. where minimal DFAs stay large. +/// +/// A per-automaton `acyclic` flag (≈⅓ of cases) remaps every chosen target +/// into the forward range `from+1..n`, producing a DAG and therefore a +/// **finite** language. Without it nearly every random automaton contains a +/// cycle, and the finite-language paths of the cardinality and max-length +/// analyses go untested. Cyclic mode still reaches every DFA, so coverage is +/// preserved. +pub fn arb_dfa() -> impl Strategy { + ( + arb_num_states(), + 0.6f64..0.97, + 0.25f64..0.75, + prop::bool::weighted(0.35), + ) + .prop_flat_map(|(n, edge_density, accept_density, acyclic)| { + let accepts = prop::collection::vec(prop::bool::weighted(accept_density), n); + // An "anchor" state forced accepting most of the time: without it + // the whole accept vector samples all-false often enough that the + // empty language eats a fifth of the sample. Non-start states are + // preferred — anchoring the start only inflates the {""} corner. + // The `None` branch keeps every accept subset (incl. all-false) + // reachable. + let anchor = prop::option::weighted(0.85, 1usize.min(n - 1)..n); + // transition function: tf[state][base] = optional target state + let tf = prop::collection::vec( + prop::collection::vec(prop::option::weighted(edge_density, 0usize..n), num_bases()), + n, + ); + (Just(n), accepts, anchor, tf, Just(acyclic)) + }) + .prop_map(|(n, mut accepts, anchor, tf, acyclic)| { + if let Some(k) = anchor { + accepts[k] = true; + } + let nb = num_bases(); + let mut edges: Vec<(usize, usize, Vec)> = Vec::new(); + for (from, row) in tf.iter().enumerate() { + // Group the bases by their chosen target so each (from, to) + // edge gets a single, disjoint-from-its-siblings label. + let mut by_target: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for (base, target) in row.iter().enumerate() { + if let Some(t) = target { + let t = if acyclic { + if from + 1 >= n { + // the last state of a DAG has no outgoing edge + continue; + } + // remap into the forward range; every forward + // target keeps a positive probability + from + 1 + (*t % (n - from - 1)) + } else { + *t + }; + by_target.entry(t).or_insert_with(|| vec![false; nb])[base] = true; + } + } + for (to, mask) in by_target { + edges.push((from, to, mask)); + } + } + build(n, &accepts, &edges, &[]) + }) +} + +/// Strategy producing every NFA over the alphabet with `1..=MAX_STATES` states +/// (overlapping labels allowed, plus optional epsilon transitions). +/// +/// Instead of a fixed per-bit probability (which blobs large automata and +/// starves small ones), a per-state branching target is sampled and converted +/// into a bit density of `target / (n · |Σ|)`, so the *local* structure is +/// comparable across sizes. Epsilon and accept densities are sampled too. All +/// densities stay strictly inside (0, 1), so every NFA keeps a positive +/// probability. +/// +/// As in [`arb_dfa`], the accept density is centered on ½ and a per-automaton +/// `acyclic` flag (≈⅓ of cases) keeps only forward (`from < to`) edges, +/// producing finite languages; the density is rescaled to the smaller target +/// pool so the out-degree stays comparable. Epsilon transitions are kept rare +/// because [`FastAutomaton::add_epsilon_transition`] eagerly folds the target +/// state into the source, which merges languages and shrinks minimal DFAs. +pub fn arb_nfa() -> impl Strategy { + arb_nfa_over(ALPHABET) +} + +/// [`arb_nfa`] generalized to an arbitrary alphabet, so two operands of a +/// binary operation can be generated over *different* alphabets — the only +/// way to exercise `SpanningSet::merge` and the `ConditionConverter` +/// re-projection (same-alphabet operands share an identical spanning set and +/// the conversion is the identity). +pub fn arb_nfa_over(alphabet: &'static [char]) -> impl Strategy { + ( + arb_num_states(), + 1.0f64..2.8, + 0.02f64..0.12, + 0.25f64..0.75, + prop::bool::weighted(0.35), + ) + .prop_flat_map( + move |(n, target_out_degree, eps_density, accept_density, acyclic)| { + // In acyclic mode only the upper triangle of the matrix survives, + // so the average target pool is half as big. + let effective_targets = if acyclic { + (n as f64 / 2.0).max(1.0) + } else { + n as f64 + }; + let label_density = (target_out_degree + / (effective_targets * alphabet.len() as f64)) + .clamp(0.02, 0.95); + let accepts = prop::collection::vec(prop::bool::weighted(accept_density), n); + // see arb_dfa: keeps the empty language an edge case, not a fifth + // of the sample + let anchor = prop::option::weighted(0.85, 1usize.min(n - 1)..n); + // labels[from][to] = mask over the alphabet letters + let labels = prop::collection::vec( + prop::collection::vec( + prop::collection::vec(prop::bool::weighted(label_density), alphabet.len()), + n, + ), + n, + ); + // eps[from][to] = whether an epsilon transition is present + let eps = prop::collection::vec( + prop::collection::vec(prop::bool::weighted(eps_density), n), + n, + ); + (Just(n), accepts, anchor, labels, eps, Just(acyclic)) + }, + ) + .prop_map(move |(n, mut accepts, anchor, labels, eps, acyclic)| { + if let Some(k) = anchor { + accepts[k] = true; + } + let mut char_edges = Vec::new(); + for (from, row) in labels.iter().enumerate() { + for (to, mask) in row.iter().enumerate() { + if acyclic && to <= from { + continue; + } + if mask.iter().any(|&b| b) { + char_edges.push((from, to, mask.clone())); + } + } + } + let mut eps_edges = Vec::new(); + for (from, row) in eps.iter().enumerate() { + for (to, &on) in row.iter().enumerate() { + let backward = acyclic && to <= from; + if on && from != to && !backward { + eps_edges.push((from, to)); + } + } + } + build_over(alphabet, n, &accepts, &char_edges, &eps_edges) + }) +} + +/// Strategy for a character class: any subset of the alphabet (including the +/// empty language) and, occasionally, the total range `.`. +/// +/// Single letters dominate: the unbiased subset mask would produce the empty +/// class `[]` a quarter of the time, and a single `[]` anywhere in a +/// concatenation collapses the whole expression to the empty language. The +/// mask branch keeps every subset (including `[]`) at a positive probability. +fn arb_charrange() -> impl Strategy { + prop_oneof![ + 8 => (0..ALPHABET.len()).prop_map(letter), + 3 => prop::collection::vec(any::(), ALPHABET.len()).prop_map(|mask| { + let mut r = CharRange::empty(); + for (i, &on) in mask.iter().enumerate() { + if on { + r = r.union(&letter(i)); + } + } + r + }), + 1 => Just(CharRange::total()), + ] +} + +/// Strategy producing regular expressions over the four [`RegularExpression`] +/// variants up to a bounded recursion depth. +/// +/// Repetition gets the heaviest weight: it is the variant that feeds the +/// `{n,m}` expansion, the simplifier and the loop handling of state +/// elimination, and stacking it (`(a*){2}`-style nesting) is where those +/// paths historically break. Concat and alternation still keep substantial +/// weight so all shapes appear. +pub fn arb_regex() -> impl Strategy { + let leaf = arb_charrange().prop_map(RegularExpression::Character); + leaf.prop_recursive(4, 48, 3, |inner| { + prop_oneof![ + 3 => (inner.clone(), 0u32..=2, 0u32..=2, any::()).prop_map( + |(r, min, extra, has_max)| { + // max is min + extra, so the bounds are always valid + let max = if has_max { Some(min + extra) } else { None }; + RegularExpression::Repetition(Box::new(r), min, max) + } + ), + 2 => prop::collection::vec(inner.clone(), 1..=3) + .prop_map(|v| RegularExpression::Concat(v.into())), + 2 => prop::collection::vec(inner, 1..=3).prop_map(RegularExpression::Alternation), + ] + }) +} + +/// Runs `f` under a bounded execution profile, returning `None` when the +/// operation legitimately exceeds the state/time budget (which is not a bug). +fn bounded Result>(f: F) -> Option { + ExecutionProfileBuilder::new() + .max_number_of_states(8192) + .execution_timeout(3000) + .build() + .run(|| match f() { + Ok(v) => Some(v), + Err(EngineError::AutomatonHasTooManyStates) + | Err(EngineError::OperationTimeOutError) => None, + Err(e) => panic!("unexpected engine error: {e:?}"), + }) +} + +fn determinized(a: &FastAutomaton) -> Option { + bounded(|| a.determinize().map(|c| c.into_owned())) +} + +fn complemented(a: &FastAutomaton) -> Option { + bounded(|| { + let mut c = a.clone(); + c.complement()?; + Ok(c) + }) +} + +/// All strings up to length `max_len` over `alphabet` (plus the empty +/// string). +fn probes_over(alphabet: &[char], max_len: usize) -> Vec { + let mut all = vec![String::new()]; + let mut frontier = vec![String::new()]; + for _ in 0..max_len { + let mut next = Vec::new(); + for w in &frontier { + for &c in alphabet { + let mut s = w.clone(); + s.push(c); + next.push(s); + } + } + all.extend(next.iter().cloned()); + frontier = next; + } + all +} + +/// All strings up to length 4 over [`ALPHABET`] (plus the empty string). +fn probes() -> Vec { + probes_over(ALPHABET, 4) +} + +/// Asserts that intersection / union / difference of `a` and `b` agree with +/// the boolean combination of the operands on every probe string. +fn assert_set_ops_membership( + a: &FastAutomaton, + b: &FastAutomaton, + probes: &[String], +) -> Result<(), TestCaseError> { + if let Some(inter) = bounded(|| a.intersection(b)) { + for s in probes { + prop_assert_eq!( + inter.is_match(s), + a.is_match(s) && b.is_match(s), + "intersection membership for {:?}", + s + ); + } + } + if let Some(union) = bounded(|| a.union(b)) { + for s in probes { + prop_assert_eq!( + union.is_match(s), + a.is_match(s) || b.is_match(s), + "union membership for {:?}", + s + ); + } + } + // `difference` determinizes the subtrahend itself. + if let Some(diff) = bounded(|| a.difference(b)) { + for s in probes { + prop_assert_eq!( + diff.is_match(s), + a.is_match(s) && !b.is_match(s), + "difference membership for {:?}", + s + ); + } + } + Ok(()) +} + +/// Decomposition oracle for repetition: `s` is in L(a){min,max} iff `s` +/// splits into k pieces, each in L(a), for some valid k. Piece counts +/// saturate at `min` once they can only grow (relevant for unbounded +/// maxima and for "" ∈ L(a), which allows padding with empty pieces). +fn repeat_decomposition_oracle(a: &FastAutomaton, s: &str, min: u32, max: Option) -> bool { + let min = min as usize; + let cap = max.map(|m| m as usize).unwrap_or(min).max(min); + let accepts_empty = a.is_match(""); + let len = s.len(); + + // reach[i][k]: the prefix of length i splits into exactly k pieces + // (k saturated at cap + 1 to keep the table finite). + let k_slots = cap + 2; + let mut reach = vec![vec![false; k_slots]; len + 1]; + reach[0][0] = true; + for i in 0..=len { + for k in 0..k_slots { + if !reach[i][k] { + continue; + } + let next_k = (k + 1).min(cap + 1); + // Pad with an empty piece. + if accepts_empty { + reach[i][next_k] = true; + } + // Consume a non-empty piece. + for j in i + 1..=len { + if a.is_match(&s[i..j]) { + reach[j][next_k] = true; + } + } + } + } + + let k_ok = |k: usize| { + k >= min + && match max { + Some(m) => k <= m as usize, + None => true, + } + }; + (0..k_slots).any(|k| reach[len][k] && k_ok(k)) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(192))] + + /// The DFA strategy really does produce deterministic automata. + #[test] + fn dfa_strategy_is_deterministic(a in arb_dfa()) { + prop_assert!(a.is_deterministic(), "arb_dfa produced a non-deterministic automaton"); + } + + /// `a` and `determinize(a)` accept the same language. + #[test] + fn determinize_preserves_language(a in arb_nfa()) { + if let Some(d) = determinized(&a) { + prop_assert!(d.is_deterministic()); + if let Some(eq) = bounded(|| a.equivalent(&d)) { + prop_assert!(eq, "determinize changed the language"); + } + } + } + + /// Minimizing a DFA preserves its language. + #[test] + fn minimize_preserves_language(a in arb_nfa()) { + if let Some(d) = determinized(&a) { + let mut m = d.clone(); + if bounded(|| m.minimize()).is_some() + && let Some(eq) = bounded(|| d.equivalent(&m)) + { + prop_assert!(eq, "minimize changed the language"); + } + } + } + + /// Complement laws: membership flips, `a ∩ ¬a = ∅`, `a ∪ ¬a = Σ*`. + #[test] + fn complement_laws(a in arb_dfa()) { + let d = match determinized(&a) { Some(d) => d, None => return Ok(()) }; + let c = match complemented(&d) { Some(c) => c, None => return Ok(()) }; + + for s in probes() { + prop_assert_eq!(d.is_match(&s), !c.is_match(&s), "complement membership for {:?}", s); + } + + if let Some(inter) = bounded(|| d.intersection(&c)) + && let Some(empty) = bounded(|| inter.equivalent(&FastAutomaton::new_empty())) + { + prop_assert!(empty, "a ∩ ¬a is not empty"); + } + if let Some(union) = bounded(|| d.union(&c)) + && let Some(total) = bounded(|| union.equivalent(&FastAutomaton::new_total())) + { + prop_assert!(total, "a ∪ ¬a is not total"); + } + } + + /// Membership of intersection / union / difference matches the boolean + /// combination of the operands on every probe string. + #[test] + fn set_ops_membership(a in arb_nfa(), b in arb_nfa()) { + assert_set_ops_membership(&a, &b, &probes())?; + } + + /// Set operations across operands built over *overlapping but different* + /// alphabets ({a,b} vs {b,c}): the operands carry different spanning + /// sets, so `SpanningSet::merge` and the `ConditionConverter` + /// re-projection do real work (same-alphabet pairs convert via the + /// identity). The shared letter `b` keeps the intersections non-trivial. + #[test] + fn set_ops_membership_overlapping_alphabets( + a in arb_nfa_over(&['a', 'b']), + b in arb_nfa_over(&['b', 'c']), + ) { + assert_set_ops_membership(&a, &b, &probes_over(&['a', 'b', 'c'], 4))?; + } + + /// Set operations across operands built over *disjoint* alphabets + /// ({a,b} vs {c,d}): the merged spanning set shares no base with either + /// source, the most extreme re-projection. The intersection collapses to + /// at most {""} — itself a worthwhile edge case. + #[test] + fn set_ops_membership_disjoint_alphabets( + a in arb_nfa_over(&['a', 'b']), + b in arb_nfa_over(&['c', 'd']), + ) { + assert_set_ops_membership(&a, &b, &probes_over(&['a', 'b', 'c', 'd'], 3))?; + } + + /// `length` and `cardinality` agree with brute-force enumeration. + /// The probes cover *every* string up to length 4, so they are exactly + /// the language whenever the maximum length is ≤ 4, and a complete + /// census of its short strings otherwise. + #[test] + fn length_cardinality_match_brute_force(a in arb_nfa()) { + let (min, max) = a.length(); + let matched_lengths: Vec = probes() + .iter() + .filter(|s| a.is_match(s)) + .map(|s| s.chars().count() as u32) + .collect(); + + // Minimum: any string of length ≤ 4 is a probe, so a language with + // min ≤ 4 has a matched probe of exactly that length. + match (min, matched_lengths.iter().min()) { + (Some(min_len), Some(&shortest)) => { + prop_assert_eq!(min_len, shortest, "min length disagrees with enumeration"); + } + (Some(min_len), None) => { + prop_assert!(min_len > 4, "min ≤ 4 but no probe matched"); + } + (None, Some(_)) => prop_assert!(false, "empty language matched a probe"), + (None, None) => {} + } + + if let Some(max_len) = max + && max_len <= 4 + { + // The probes enumerate the whole language. + prop_assert_eq!( + Some(max_len), + matched_lengths.iter().max().copied(), + "max length disagrees with enumeration" + ); + if let Some(cardinality) = bounded(|| a.cardinality()) { + prop_assert_eq!( + cardinality, + Cardinality::Integer(matched_lengths.len() as u32), + "cardinality disagrees with enumeration" + ); + } + } else if max.is_none() + && min.is_some() + && let Some(cardinality) = bounded(|| a.cardinality()) + { + // A cycle on an accepting path means infinitely many strings. + prop_assert_eq!( + cardinality, + Cardinality::Infinite, + "infinite language with non-infinite cardinality" + ); + } + } + + /// `FastAutomaton::concat` agrees with the split-membership oracle: + /// s ∈ L(a)·L(b) iff some split s = u·v has u ∈ L(a) and v ∈ L(b). + #[test] + fn automaton_concat_matches_split_oracle(a in arb_nfa(), b in arb_nfa()) { + if let Some(concat) = bounded(|| a.concat(&b)) { + for s in probes() { + let expected = + (0..=s.len()).any(|i| a.is_match(&s[..i]) && b.is_match(&s[i..])); + prop_assert_eq!( + concat.is_match(&s), expected, + "concat membership for {:?}", s + ); + } + } + } + + /// `FastAutomaton::repeat` agrees with a decomposition oracle computed by + /// dynamic programming over (position, piece-count) — independent of the + /// engine's own repeat construction (which the regex route would reuse). + #[test] + fn automaton_repeat_matches_decomposition_oracle( + a in arb_nfa(), + min in 0u32..3, + extra in 0u32..2, + unbounded in any::(), + ) { + let max = if unbounded { None } else { Some(min + extra) }; + if let Some(repeated) = bounded(|| a.repeat(min, max)) { + for s in probes() { + let expected = repeat_decomposition_oracle(&a, &s, min, max); + prop_assert_eq!( + repeated.is_match(&s), expected, + "repeat({}, {:?}) membership for {:?}", min, max, s + ); + } + } + } + + /// `Term::union` / `Term::intersection` over more than 3 operands (the + /// parallel dispatch path when the `parallel` feature is on) agree with + /// sequential pairwise folds. + #[test] + fn many_operand_term_ops_match_pairwise_folds( + a in arb_nfa(), b in arb_nfa(), c in arb_nfa(), d in arb_nfa(), e in arb_nfa(), + ) { + use regexsolver::Term; + + let operands: Vec = [&b, &c, &d, &e] + .into_iter() + .map(|x| Term::from_automaton(x.clone())) + .collect(); + let first = Term::from_automaton(a.clone()); + + if let Some(many) = bounded(|| { + Ok(first.union(&operands)?.to_automaton()?.into_owned()) + }) && let Some(pairwise) = bounded(|| { + let mut acc = a.clone(); + for x in [&b, &c, &d, &e] { + acc = acc.union(x)?; + } + Ok(acc) + }) && let Some(eq) = bounded(|| many.equivalent(&pairwise)) { + prop_assert!(eq, "5-operand union disagrees with pairwise folds"); + } + + if let Some(many) = bounded(|| { + Ok(first.intersection(&operands)?.to_automaton()?.into_owned()) + }) && let Some(pairwise) = bounded(|| { + let mut acc = a.clone(); + for x in [&b, &c, &d, &e] { + acc = acc.intersection(x)?; + } + Ok(acc) + }) && let Some(eq) = bounded(|| many.equivalent(&pairwise)) { + prop_assert!(eq, "5-operand intersection disagrees with pairwise folds"); + } + } + + /// `RegularExpression::union_all` (the alternation accumulator) agrees + /// with a left fold of pairwise `union` — a different composition: the + /// fold runs a fresh two-operand accumulator per step, `union_all` one + /// accumulator over all operands — and with the boolean OR of the + /// operands on every probe string. The `Term`-level test above never + /// reaches this path: it uses automaton-backed operands, and `Term::union` + /// routes through the regex accumulator only when *every* operand is + /// regex-backed. + #[test] + fn regex_union_all_matches_pairwise_fold_and_membership( + regexes in prop::collection::vec(arb_regex(), 2..=6), + ) { + let all = RegularExpression::union_all(®exes); + let pairwise = regexes[1..] + .iter() + .fold(regexes[0].clone(), |acc, r| acc.union(r)); + + let all_a = match bounded(|| all.to_automaton()) { + Some(a) => a, + None => return Ok(()), + }; + if let Some(pairwise_a) = bounded(|| pairwise.to_automaton()) + && let Some(eq) = bounded(|| all_a.equivalent(&pairwise_a)) + { + prop_assert!( + eq, + "union_all disagrees with the pairwise fold: {} vs {}", + all, + pairwise + ); + } + + let mut operand_automata = Vec::with_capacity(regexes.len()); + for r in ®exes { + match bounded(|| r.to_automaton()) { + Some(a) => operand_automata.push(a), + None => return Ok(()), + } + } + for s in probes() { + let expected = operand_automata.iter().any(|a| a.is_match(&s)); + prop_assert_eq!( + all_a.is_match(&s), + expected, + "union_all membership for {:?}", + s + ); + } + } + + /// `subset` and `equivalent` agree: mutual subset iff equivalent; both are + /// reflexive. + #[test] + fn subset_equivalent_consistency(a in arb_nfa(), b in arb_nfa()) { + if let Some(refl) = bounded(|| a.equivalent(&a)) { + prop_assert!(refl, "equivalent is not reflexive"); + } + if let Some(refl) = bounded(|| a.subset(&a)) { + prop_assert!(refl, "subset is not reflexive"); + } + if let (Some(ab), Some(ba), Some(eq)) = ( + bounded(|| a.subset(&b)), + bounded(|| b.subset(&a)), + bounded(|| a.equivalent(&b)), + ) { + prop_assert_eq!(ab && ba, eq, "mutual subset disagrees with equivalent"); + } + } + + /// `a -> regex -> a` round-trips: the regular expression extracted from an + /// automaton compiles back to an equivalent automaton. + #[test] + fn automaton_to_regex_roundtrip(a in arb_nfa()) { + let r = match bounded(|| a.to_regex()) { Some(r) => r, None => return Ok(()) }; + if let Some(a2) = bounded(|| r.to_automaton()) + && let Some(eq) = bounded(|| a.equivalent(&a2)) + { + prop_assert!(eq, "automaton -> regex -> automaton changed the language: {}", r); + } + } + + /// Regular expressions round-trip through an automaton and agree with the + /// reference `regex` crate on every probe string. + #[test] + fn regex_roundtrip_and_oracle(r in arb_regex()) { + let a = match bounded(|| r.to_automaton()) { Some(a) => a, None => return Ok(()) }; + + // regex -> automaton -> regex -> automaton preserves the language. + let r2 = match bounded(|| a.to_regex()) { Some(r2) => r2, None => return Ok(()) }; + if let Some(a2) = bounded(|| r2.to_automaton()) + && let Some(eq) = bounded(|| a.equivalent(&a2)) + { + prop_assert!(eq, "regex round-trip changed the language: {} -> {}", r, r2); + } + + // Cross-check membership against the standard regex engine (anchored, + // dot-matches-newline). Patterns denoting the empty language ("[]") are + // rejected by the `regex` crate, so we only compare when it accepts the + // pattern. + let pattern = r.to_string(); + if let Ok(re) = regex::Regex::new(&format!("(?s)^(?:{})$", pattern)) { + for s in probes() { + prop_assert_eq!( + a.is_match(&s), re.is_match(&s), + "pattern {:?} disagrees with reference engine on {:?}", pattern, s + ); + } + } + } +} + +#[cfg(test)] +mod inspect { + use super::*; + use proptest::strategy::{Strategy, ValueTree}; + use proptest::test_runner::TestRunner; + use regex_charclass::CharacterClass; + + fn samples(strat: S, n: usize) -> Vec { + let mut runner = TestRunner::deterministic(); + (0..n) + .map(|_| strat.new_tree(&mut runner).unwrap().current()) + .collect() + } + + #[test] + fn show_regex() { + for (i, r) in samples(arb_regex(), 30).into_iter().enumerate() { + println!("regex[{i:02}] = {r}"); + } + } + + #[test] + fn show_dfa() { + for (i, a) in samples(arb_dfa(), 30).into_iter().enumerate() { + println!("dfa[{i:02}] det={} Graphviz={}", a.is_deterministic(), a); + } + } + + #[test] + fn show_nfa() { + for (i, a) in samples(arb_nfa(), 30).into_iter().enumerate() { + println!("nfa[{i:02}] det={} Graphviz={}", a.is_deterministic(), a); + } + } + + /// The language class of a generated entity, ordered from degenerate to + /// rich. + /// + /// `Empty`, `EmptyString` and `Total` are the corners of the language + /// lattice: useful as occasional edge cases (they hit the `is_empty` / + /// complement / difference fast paths) but they exercise nothing else. + /// `Finite` languages take the topological-sort path of the cardinality + /// and max-length analyses; `Infinite` ones take the cycle paths of state + /// elimination and repeat synthesis. A quality sample needs both in bulk. + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] + enum LangClass { + Empty, + EmptyString, + Total, + Finite, + Infinite, + } + + /// Reduces an automaton to its canonical minimal DFA. + fn minimal_dfa(a: &FastAutomaton) -> FastAutomaton { + let mut m = determinized(a).expect("generated automata always determinize in budget"); + bounded(|| m.minimize()).expect("generated automata always minimize in budget"); + m + } + + /// Classifies the language of a **minimal DFA** into the + /// [`LangClass`] variants: the trivial cases (empty, empty-string, total) + /// and, for the rest, finite vs. infinite languages, which exercise + /// disjoint code paths. + fn classify(m: &FastAutomaton) -> LangClass { + if m.is_empty() { + LangClass::Empty + } else if m.is_empty_string() { + LangClass::EmptyString + } else if m.is_total() { + // exact on a DFA + LangClass::Total + } else if m.length().1.is_some() { + LangClass::Finite + } else { + LangClass::Infinite + } + } + + /// Canonical fingerprint of a language: the minimal DFA, renumbered in + /// BFS order with the outgoing transitions of each state sorted by label. + /// Minimal DFAs are unique up to isomorphism, so two automata share a key + /// iff they accept the same language — this is what lets the stats count + /// *distinct* languages instead of distinct syntax trees. + fn language_key(m: &FastAutomaton) -> String { + use std::fmt::Write; + let ss = m.spanning_set(); + let mut order = vec![m.start_state()]; + let mut ids = std::collections::HashMap::new(); + ids.insert(m.start_state(), 0usize); + let mut key = String::new(); + let mut i = 0; + while i < order.len() { + let s = order[i]; + i += 1; + // In a DFA the labels leaving a state are disjoint, hence unique, + // so sorting by label gives a deterministic traversal order. + let mut out: Vec<(String, usize)> = m + .transitions_from_vec(s) + .into_iter() + .map(|(c, t)| { + ( + c.to_range(ss) + .expect("condition always converts to a range") + .to_regex(), + t, + ) + }) + .collect(); + out.sort(); + write!(key, "{}", if m.is_accepted(s) { 'A' } else { 'r' }).unwrap(); + for (label, t) in out { + let id = match ids.get(&t) { + Some(&id) => id, + None => { + let id = order.len(); + ids.insert(t, id); + order.push(t); + id + } + }; + write!(key, " {label}>{id}").unwrap(); + } + key.push(';'); + } + key + } + + /// Everything we measure about one generated automaton. + struct Measure { + // Structure of the generated entity itself. + edges: usize, + multi_base_edges: usize, + deterministic: bool, + // Properties of its *language*, computed on the minimal DFA. + class: LangClass, + minimal_states: usize, + accepts_empty_string: bool, + key: String, + } + + fn measure(a: &FastAutomaton) -> Measure { + let mut edges = 0; + let mut multi_base_edges = 0; + for s in a.states_vec() { + for (cond, _) in a.transitions_from_vec(s) { + edges += 1; + if cond.binary_representation().iter().filter(|&&b| b).count() > 1 { + multi_base_edges += 1; + } + } + } + let m = minimal_dfa(a); + Measure { + edges, + multi_base_edges, + deterministic: a.is_deterministic(), + class: classify(&m), + minimal_states: m.number_of_states(), + accepts_empty_string: m.is_match(""), + key: language_key(&m), + } + } + + /// Aggregated quality metrics over a sample; what the strategies are + /// evaluated (and asserted) on. + struct Quality { + /// Share of `Empty` + `EmptyString` + `Total` languages. Wanted as a + /// small minority: present (they are real edge cases) but not eating + /// the sample. + degenerate_pct: f64, + finite_pct: f64, + infinite_pct: f64, + /// Distinct languages (by canonical minimal DFA) over sample size. + /// Duplicates re-test the same language and are wasted cases. + distinct_pct: f64, + /// Average minimal-DFA size: the number of Myhill-Nerode classes is + /// what minimize / equivalence / state elimination actually scale + /// with, so this — not the raw state count — is language complexity. + avg_minimal_states: f64, + /// Share of languages needing a minimal DFA of ≥ 3 states. + rich_pct: f64, + accepts_empty_string_pct: f64, + /// Share of transitions whose condition spans more than one base of + /// the spanning set (exercises the bitvector paths beyond single + /// bits). + multi_base_edge_pct: f64, + /// Share of genuinely nondeterministic automata (only meaningful for + /// the NFA strategy: a "NFA" that is already deterministic never + /// exercises subset construction). + nondeterministic_pct: f64, + } + + fn quality(name: &str, measures: &[Measure]) -> Quality { + let n = measures.len() as f64; + let count = |f: &dyn Fn(&Measure) -> bool| { + 100.0 * measures.iter().filter(|m| f(m)).count() as f64 / n + }; + + let distinct: std::collections::HashSet<&str> = + measures.iter().map(|m| m.key.as_str()).collect(); + let edges: usize = measures.iter().map(|m| m.edges).sum(); + let multi: usize = measures.iter().map(|m| m.multi_base_edges).sum(); + + let mut histogram = std::collections::BTreeMap::new(); + for m in measures { + *histogram.entry(m.minimal_states).or_insert(0usize) += 1; + } + + let q = Quality { + degenerate_pct: count(&|m| { + matches!( + m.class, + LangClass::Empty | LangClass::EmptyString | LangClass::Total + ) + }), + finite_pct: count(&|m| m.class == LangClass::Finite), + infinite_pct: count(&|m| m.class == LangClass::Infinite), + distinct_pct: 100.0 * distinct.len() as f64 / n, + avg_minimal_states: measures.iter().map(|m| m.minimal_states).sum::() as f64 / n, + rich_pct: count(&|m| m.minimal_states >= 3), + accepts_empty_string_pct: count(&|m| m.accepts_empty_string), + multi_base_edge_pct: 100.0 * multi as f64 / edges.max(1) as f64, + nondeterministic_pct: count(&|m| !m.deterministic), + }; + + println!( + "{name}: degenerate {:>4.1}% (∅ {:.1}% | {{\"\"}} {:.1}% | Σ* {:.1}%) | finite {:>4.1}% | infinite {:>4.1}%", + q.degenerate_pct, + count(&|m| m.class == LangClass::Empty), + count(&|m| m.class == LangClass::EmptyString), + count(&|m| m.class == LangClass::Total), + q.finite_pct, + q.infinite_pct, + ); + println!( + "{name}: distinct languages {:>4.1}% | accepts \"\" {:>4.1}% | nondet {:>4.1}% | multi-base edges {:>4.1}%", + q.distinct_pct, + q.accepts_empty_string_pct, + q.nondeterministic_pct, + q.multi_base_edge_pct, + ); + println!( + "{name}: minimal-DFA states avg {:.2}, ≥3 {:>4.1}%, histogram {:?}", + q.avg_minimal_states, q.rich_pct, histogram, + ); + q + } + + /// Operator coverage of a generated regular expression; the round-trip + /// (state elimination) and simplification code paths are keyed on these + /// shapes. + #[derive(Default)] + struct RegexFacets { + unbounded_repetition: bool, + bounded_repetition: bool, + nested_repetition: bool, + alternation: bool, + multi_char_class: bool, + } + + fn regex_facets(r: &RegularExpression, inside_repetition: bool, f: &mut RegexFacets) { + match r { + RegularExpression::Character(range) => { + let letters = (0..ALPHABET.len()) + .filter(|&i| !range.intersection(&letter(i)).is_empty()) + .count(); + if letters > 1 || range.is_total() { + f.multi_char_class = true; + } + } + RegularExpression::Repetition(inner, _, max) => { + if max.is_some() { + f.bounded_repetition = true; + } else { + f.unbounded_repetition = true; + } + if inside_repetition { + f.nested_repetition = true; + } + regex_facets(inner, true, f); + } + RegularExpression::Concat(parts) => { + for p in parts { + regex_facets(p, inside_repetition, f); + } + } + RegularExpression::Alternation(parts) => { + f.alternation = true; + for p in parts { + regex_facets(p, inside_repetition, f); + } + } + } + } + + /// Quantitative quality summary over a larger sample, with floors the + /// strategies must keep. The sample runner is deterministic, so the + /// numbers — and therefore the assertions — are reproducible. + #[test] + fn stats() { + const N: usize = 300; + + let regexes = samples(arb_regex(), N); + let regex_measures: Vec = regexes + .iter() + .map(|r| measure(&r.to_automaton().expect("small regexes always convert"))) + .collect(); + let regex_q = quality("regex", ®ex_measures); + + let avg_len = regexes.iter().map(|r| r.to_string().len()).sum::() as f64 / N as f64; + let mut facet_counts = [0usize; 5]; + for r in ®exes { + let mut f = RegexFacets::default(); + regex_facets(r, false, &mut f); + for (i, hit) in [ + f.unbounded_repetition, + f.bounded_repetition, + f.nested_repetition, + f.alternation, + f.multi_char_class, + ] + .into_iter() + .enumerate() + { + facet_counts[i] += hit as usize; + } + } + let fpct = |i: usize| 100.0 * facet_counts[i] as f64 / N as f64; + println!( + "regex: avg pattern length {avg_len:.1} | unbounded-rep {:.1}% | bounded-rep {:.1}% | nested-rep {:.1}% | alternation {:.1}% | multi-char class {:.1}%", + fpct(0), + fpct(1), + fpct(2), + fpct(3), + fpct(4), + ); + + let dfa_q = quality( + "dfa ", + &samples(arb_dfa(), N) + .iter() + .map(measure) + .collect::>(), + ); + let nfa_q = quality( + "nfa ", + &samples(arb_nfa(), N) + .iter() + .map(measure) + .collect::>(), + ); + + for (name, q) in [("regex", ®ex_q), ("dfa", &dfa_q), ("nfa", &nfa_q)] { + // The trivial corner languages should be present but a minority. + assert!( + q.degenerate_pct < 25.0, + "{name}: too many degenerate languages" + ); + // Both bulk classes must be well represented. + assert!( + q.finite_pct >= 15.0, + "{name}: finite languages under-represented" + ); + assert!( + q.infinite_pct >= 15.0, + "{name}: infinite languages under-represented" + ); + // The sample must not keep re-testing the same languages. These + // floors are measured on the *canonical* minimal DFA (dead states + // removed), so genuinely-equivalent samples dedupe correctly; they + // sit below the pre-canonicalization figures, when dead-state + // differences inflated both the distinct-language and state counts. + assert!( + q.distinct_pct >= 38.0, + "{name}: not enough distinct languages" + ); + // Language complexity: minimal DFAs must not collapse to 1-2 states. + assert!(q.rich_pct >= 30.0, "{name}: minimal DFAs too small"); + // Both "" ∈ L and "" ∉ L need bulk representation. + assert!( + (20.0..=80.0).contains(&q.accepts_empty_string_pct), + "{name}: empty-string acceptance unbalanced" + ); + // Conditions spanning several bases must show up regularly. + assert!( + q.multi_base_edge_pct >= 10.0, + "{name}: multi-base conditions too rare" + ); + } + // An NFA strategy that mostly produces DFAs never exercises subset + // construction. + assert!( + nfa_q.nondeterministic_pct >= 50.0, + "nfa: mostly deterministic" + ); + } +} diff --git a/tests/readme_examples.rs b/tests/readme_examples.rs new file mode 100644 index 0000000..0cd03d0 --- /dev/null +++ b/tests/readme_examples.rs @@ -0,0 +1,150 @@ +//! Keeps the README's examples honest: these tests are the README snippets, +//! verbatim. If one fails, update the README. + +use regexsolver::Term; +use regexsolver::error::EngineError; +use regexsolver::execution_profile::ExecutionProfileBuilder; +use regexsolver::fast_automaton::{GenerationOptions, PathOrder}; + +#[test] +fn readme_automaton_building_example() -> Result<(), EngineError> { + use regexsolver::CharRange; + use regexsolver::fast_automaton::FastAutomaton; + use regexsolver::regex_charclass::char::Char; + + // Build an automaton matching "[a-c][0-9]*" by hand: + let mut automaton = FastAutomaton::new_empty(); + let s1 = automaton.new_state(); + automaton.accept(s1); + + let a_to_c = CharRange::new_from_range(Char::new('a')..=Char::new('c')); + let digits = CharRange::new_from_range(Char::new('0')..=Char::new('9')); + automaton.add_transition_from_range(0, s1, &a_to_c)?; + automaton.add_transition_from_range(s1, s1, &digits)?; + + assert!(automaton.is_match("b42")); + assert!(!automaton.is_match("4b")); + assert_eq!(automaton.to_regex()?.to_string(), "[a-c][0-9]*"); + + Ok(()) +} + +#[test] +fn readme_hero_example() -> Result<(), EngineError> { + let a: Term = "(ab|xy){2}".parse()?; + let b: Term = ".*xy".parse()?; + + // Which strings match BOTH patterns? Get the answer as a regex: + let both = a.intersection([&b])?; + assert_eq!(both.to_pattern()?, "(ab|xy)xy"); + + // Test a concrete string against the result (matching is anchored): + assert!(both.matches("abxy")?); + + // ...and sample them: + assert_eq!( + both.generate_strings(2, 0, PathOrder::Sweep)?, + ["xyxy", "abxy"] + ); + + Ok(()) +} + +#[test] +fn readme_regular_expression_example() -> Result<(), EngineError> { + use regexsolver::regex::RegularExpression; + + // A validation pattern for an order id, e.g. "ORD-2024-12345". + let pattern = RegularExpression::new("ORD-20[0-9]{2}-[0-9]{4,6}")?; + + // How long can matching ids get? Size your database column accordingly. + assert_eq!(pattern.length(), (Some(13), Some(15))); + + // The AST is a plain enum: walk it to lint patterns, e.g. reject + // validation rules that accept unboundedly long input. + fn has_unbounded_repetition(regex: &RegularExpression) -> bool { + match regex { + RegularExpression::Character(_) => false, + RegularExpression::Repetition(inner, _, max) => { + max.is_none() || has_unbounded_repetition(inner) + } + RegularExpression::Concat(parts) => parts.iter().any(has_unbounded_repetition), + RegularExpression::Alternation(parts) => parts.iter().any(has_unbounded_repetition), + } + } + assert!(!has_unbounded_repetition(&pattern)); + assert!(has_unbounded_repetition(&RegularExpression::new( + ".*@example\\.com" + )?)); + + Ok(()) +} + +// Timing-dependent in one direction only: the assertion needs the +// 100-million-string generation to *not* finish within 50ms, which no +// machine can do, so the test cannot flake. +#[test] +fn readme_time_bounded_execution_example() -> Result<(), EngineError> { + let term = Term::from_pattern(".*abc.*cdef.*sqdsqf.*")?; + + let execution_profile = ExecutionProfileBuilder::new() + .execution_timeout(50) // limit in milliseconds + .build(); + + // Asking for 100 million strings cannot finish within the budget, so the + // generation aborts instead of running to completion. + execution_profile.run(|| { + assert_eq!( + EngineError::OperationTimeOutError, + term.generate_strings(100_000_000, 0, GenerationOptions::new()) + .unwrap_err() + ); + }); + + Ok(()) +} + +#[test] +fn readme_state_limited_execution_example() -> Result<(), EngineError> { + let term1 = Term::from_pattern(".*abcdef.*")?; + let term2 = Term::from_pattern(".*defabc.*")?; + + let execution_profile = ExecutionProfileBuilder::new() + .max_number_of_states(5) // we set the limit + .build(); + + // We run the operation with the defined limitation + execution_profile.run(|| { + assert_eq!( + EngineError::AutomatonHasTooManyStates, + term1.intersection(&[term2]).unwrap_err() + ); + }); + + Ok(()) +} + +#[test] +fn readme_disabling_implicit_determinization_example() -> Result<(), EngineError> { + // Any non-deterministic FastAutomaton; ".*abc" compiles to one. + let nfa = Term::from_pattern(".*abc")?.to_automaton()?.into_owned(); + assert!(!nfa.is_deterministic()); + + let execution_profile = ExecutionProfileBuilder::new() + .implicit_determinization(false) // default is true + .build(); + + execution_profile.run(|| { + let mut cannot_minimize = nfa.clone(); + assert_eq!( + EngineError::DeterministicAutomatonRequired, + cannot_minimize.minimize().unwrap_err() + ); + + // Determinizing explicitly is always allowed. + let mut dfa = nfa.determinize().unwrap().into_owned(); + assert!(dfa.minimize().is_ok()); + }); + + Ok(()) +} diff --git a/tests/state_elimination_quality.rs b/tests/state_elimination_quality.rs new file mode 100644 index 0000000..a9f77df --- /dev/null +++ b/tests/state_elimination_quality.rs @@ -0,0 +1,69 @@ +//! Measures the quality of automaton→regex conversion (state elimination) +//! over the shared corpus. Not a pass/fail test of absolute numbers — it +//! prints aggregate metrics so a heuristic change can be compared before/after +//! (`cargo test --test state_elimination_quality -- --ignored --nocapture`), +//! while still asserting that every conversion round-trips (correctness). + +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +use regexsolver::regex::RegularExpression; + +#[test] +#[ignore = "measurement harness; run explicitly with --ignored --nocapture"] +fn measure_state_elimination_quality() { + let file = File::open("tests/data/regex.txt").unwrap(); + let reader = BufReader::new(file); + + let mut count = 0usize; + let mut total_complexity_nfa = 0.0f64; + let mut total_len_nfa = 0usize; + let mut total_complexity_dfa = 0.0f64; + let mut total_len_dfa = 0usize; + + for line in reader.lines() { + let line = line.unwrap(); + if line.trim().is_empty() { + continue; + } + let input = match RegularExpression::parse(&line, true) { + Ok(r) => r, + Err(_) => continue, + }; + let automaton = input.to_automaton().unwrap(); + + // NFA-derived conversion. + let out_nfa = automaton.to_regex().unwrap(); + assert!( + automaton + .equivalent(&out_nfa.to_automaton().unwrap()) + .unwrap(), + "NFA round-trip mismatch for {line:?} -> {out_nfa}" + ); + total_complexity_nfa += out_nfa.evaluate_complexity(); + total_len_nfa += out_nfa.to_string().chars().count(); + + // DFA-derived conversion. + let dfa = automaton.determinize().unwrap(); + let out_dfa = dfa.to_regex().unwrap(); + assert!( + dfa.equivalent(&out_dfa.to_automaton().unwrap()).unwrap(), + "DFA round-trip mismatch for {line:?} -> {out_dfa}" + ); + total_complexity_dfa += out_dfa.evaluate_complexity(); + total_len_dfa += out_dfa.to_string().chars().count(); + + count += 1; + } + + println!("=== state elimination quality over {count} patterns ==="); + println!("NFA: total_complexity = {total_complexity_nfa:.3}, total_len = {total_len_nfa}"); + println!("DFA: total_complexity = {total_complexity_dfa:.3}, total_len = {total_len_dfa}"); + println!( + "SUM: total_complexity = {:.3}, total_len = {}", + total_complexity_nfa + total_complexity_dfa, + total_len_nfa + total_len_dfa + ); +}