From 22e261842e538188aea281896c578c5f3d1e38b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 10 Sep 2026 16:17:55 +0300 Subject: [PATCH] Implement opt-in SPSS Frontend 0.3 --- CHANGELOG.md | 15 ++ README.md | 10 +- docs/architecture.md | 4 +- docs/release-readiness.md | 7 +- docs/transformations.md | 48 ++++- src/Core/CapabilityDeclaration.php | 3 + src/Frontend/Spss/Ast/FormatTarget.php | 2 +- src/Frontend/Spss/Ast/NotPredicate.php | 17 ++ src/Frontend/Spss/Ast/Program.php | 2 +- src/Frontend/Spss/Ast/RecodeStatement.php | 2 +- src/Frontend/Spss/Ast/ValueLabelGroup.php | 2 +- .../Spss/Ast/ValueLabelsStatement.php | 1 + .../Spss/Ast/VariableLabelAssignment.php | 2 +- src/Frontend/Spss/Ast/VariableLevelGroup.php | 2 +- src/Frontend/Spss/Ast/VariableRange.php | 16 ++ src/Frontend/Spss/Binder.php | 160 ++++++++++------ src/Frontend/Spss/Binding/SchemaState.php | 53 +++++- src/Frontend/Spss/Lexer.php | 49 ++++- src/Frontend/Spss/Parser.php | 150 +++++++++------ src/Frontend/Spss/Request/InputVariable.php | 2 +- .../Spss/Request/SpssFrontendRequest.php | 3 +- src/Frontend/Spss/SpssCompiler.php | 2 +- src/Frontend/Spss/TokenType.php | 1 + tests/Core/CapabilityDeclarationTest.php | 3 + .../Spss/Conformance/SpssFrontend02Test.php | 54 ++++++ tests/Frontend/Spss/SpssFrontend03Test.php | 173 ++++++++++++++++++ .../OfficialInPlaceTransformation01Test.php | 63 +++++++ 27 files changed, 720 insertions(+), 126 deletions(-) create mode 100644 src/Frontend/Spss/Ast/NotPredicate.php create mode 100644 src/Frontend/Spss/Ast/VariableRange.php create mode 100644 tests/Frontend/Spss/SpssFrontend03Test.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 8985e40..ab1c4d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## Unreleased + +### Added + +- Explicitly selected official SPSS Frontend 0.3 using the existing strict + request/parser/binder/compiler pipeline; Frontend 0.2 remains the default. + Support comments, dictionary-order TO and grouped commands, NOT and comparison + aliases, finite open RECODE ranges, and ordered typed ADD VALUE LABELS. +- Check all 90 effective official cases against the unchanged specification pin, + inherited Plan 0.1/0.2 objects and hashes, and native SQLite in-place provenance + and identity. Service CI is pending; no new network runtime claim is made. +- Document the parent precedence decision: comparisons, NOT, AND, OR; + parentheses override. No STRING/DELETE or schema extension, version, codec, + dependency, publication or prepared 0.7.2 scope change. + ## [0.7.2] - Unreleased ### Fixed diff --git a/README.md b/README.md index 8e15700..6892399 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,12 @@ Pass only loss codes consciously accepted for that conversion. `operation_catalo The adapter claims official Transformation Plan 0.1/0.2, SPSS Syntax Frontend 0.2, and In-Place Transformation 0.1/0.2 conformance. Compile an alias-based frontend request, then bind that alias to the existing dataset at apply time. -The specification pin does not claim support for optional Transformation Plan, -SPSS Syntax Frontend, or In-Place Transformation 0.3. +Unreleased opt-in SPSS Syntax Frontend 0.3 is implemented: explicitly set the +request `contract` to `openstatspec-spss-syntax-frontend-v0.3` (or use +`SpssFrontendRequest::CONTRACT_V03`). It emits only Plan 0.1/0.2; default 0.2 +syntax and APIs remain unchanged. All 90 effective fixtures and native SQLite +apply are tested locally; service CI is pending, not a new runtime claim. +Transformation Plan and In-Place Transformation 0.3 remain unimplemented. ```php use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; @@ -182,7 +186,7 @@ supported operations, and atomicity guarantees. - `src/Sql` - PDO profiles, strict-wide DDL, import/export and catalogues. - `src/Spss` - SAV/ZSAV gating, typed V3 engine bridge and public adapter API. - `src/Transformation` - official plans, compact apply audit, and in-place execution. -- `src/Frontend/Spss` - official SPSS Frontend 0.2 request compilation. +- `src/Frontend/Spss` - SPSS Frontend 0.2 and explicitly selected 0.3 request compilation. See [docs/architecture.md](docs/architecture.md) for the complete relational contract. See [docs/transformations.md](docs/transformations.md) for frontend boundaries, diff --git a/docs/architecture.md b/docs/architecture.md index 6d62ae7..fecafd9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,7 +90,9 @@ Encrypted files, Portable (`.por`) files and arbitrary external-engine formats a `src/Transformation/Plan` owns the official Transformation Plan 0.1/0.2 model, strict codec, validation, and deterministic plan identity. -`src/Frontend/Spss` is the separate official SPSS Frontend 0.2 boundary. +`src/Frontend/Spss` is the separate SPSS Frontend 0.2 boundary, with opt-in +Frontend 0.3 using the same strict request validator, parser, binder and compiler. +Frontend 0.3 emits only Plan 0.1/0.2; its service CI claim is pending. `src/Transformation/Execution` binds an alias-based plan to one existing dataset and applies it through the active PDO profile; it never invokes the frontend. The Stata and SAS directories are placeholders only. diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 985f879..cf9cc62 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -37,8 +37,11 @@ files. Unknown patches, including 2.2.4, fail before mutation. Read-only export still verifies server identity but does not require a write-version claim. Transformation claims remain Transformation Plan 0.1/0.2, SPSS Syntax Frontend -0.2, and In-Place Transformation 0.1/0.2. The specification's optional 0.3 -contracts are not implemented or claimed. Existing target pre-provisioning and +0.2, and In-Place Transformation 0.1/0.2 for the prepared patch. +Separate Unreleased work implements opt-in SPSS Syntax Frontend 0.3 against +this unchanged pin: 90 effective fixtures and native SQLite apply are locally +covered; exact-commit service CI remains pending. Transformation Plan 0.3 and +In-Place Transformation 0.3 remain unimplemented. Existing target pre-provisioning and caller-owned Dolt commit rules remain unchanged; see the [transformation migration notes](transformations.md#v060-migration). diff --git a/docs/transformations.md b/docs/transformations.md index 72e37ff..0d15db6 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -10,6 +10,12 @@ The PHP adapter conforms to these pinned OpenStatSpec contracts: - `openstatspec-in-place-transformation-v0.1` and `openstatspec-in-place-transformation-v0.2`. +Unreleased opt-in `openstatspec-spss-syntax-frontend-v0.3` is also implemented, +with local fixture/SQLite evidence and service CI pending. It is separately +listed as `implemented_service_ci_pending` in the capability declaration, not +added to its existing conformant contract lists. Plan 0.3 and In-Place 0.3 are +not implemented. + Plans are source-neutral, alias-based, and deterministic. The canonical plan contains only its contract, input alias, and ordered operations. Dataset UUIDs, SQL identifiers, actor identity, and Dolt context are supplied only when the @@ -81,6 +87,37 @@ Names bind ASCII case-insensitively while plans retain exact catalog spelling. Multi-variable `RECODE` reads all sources from the pre-command schema. Commands, expressions, comments, and syntax outside the official subset fail closed. +## Opt-in Frontend 0.3 + +Select the exact `openstatspec-spss-syntax-frontend-v0.3` request contract, also +available as `SpssFrontendRequest::CONTRACT_V03`. `CONTRACT` and the public +parse/bind defaults remain 0.2. The existing `fromArray()` production boundary +rejects unknown/missing fields and wrong types, including nested schema and +typed-label fields. No physical identifiers or schema extensions are accepted. + +The shared parser/binder adds command-boundary `*` and `COMMENT` comments, +non-nested block comments wherever whitespace is legal, dictionary-order `TO`, +grouped RECODE/labels/formats/levels, numeric `NOT` and `NE`/`<>`/`~=`, finite +LOWEST/HIGHEST bounds, and ordered typed `ADD VALUE LABELS`. Comments end at +the next command period (or `*/` for blocks); nested/unterminated blocks and +comment-only programs fail. Hashing retains original comments and normalizes +only CRLF/CR to LF. ADD updates existing typed codes in place and appends new +codes, including after preceding VALUE LABELS replacements. Its initial label +state comes from the supplied input schema; callers must supply the current +dictionary when compiling. + +**Parent decision:** use conventional SPSS precedence: comparisons, then NOT, +then AND, then OR. Thus `NOT a = 1 AND b = 2 OR c = 3` means +`((NOT (a = 1)) AND (b = 2)) OR (c = 3)`. Parentheses override that order. +Negation complements comparisons and applies De Morgan's laws without changing +UNKNOWN; canonical lowering flattens maximal same-operator nodes in source +order, including across parentheses. Double NOT retains the original predicate. + +These additions emit exact Plan 0.1 when possible and Plan 0.2 for existing +0.2-only operations. `STRING`, `DELETE VARIABLES`, arbitrary expressions and +implicit type coercions remain rejected. No codec, catalog or executor schema +change is involved. + ## In-place and atomicity contract Every successful apply preserves the existing logical dataset UUID, registered @@ -143,6 +180,15 @@ composer check ``` They cover all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, -and 11 In-Place 0.2 manifest cases. SQLite runs locally. PostgreSQL, MySQL, +and 11 In-Place 0.2 manifest cases, plus all 90 effective Frontend 0.3 cases +(35 declared plus inherited cases minus the two comment supersessions). +Frontend checks compare source/plan hashes, exact inherited plans, diagnostics +and declared metadata preservation. Use +`OPENSTATSPEC_SPECIFICATION_DIR=/tmp/openstatspec-alignment-spec` for the +alignment checkout at unchanged commit `864e84479f554b8ee250ffed44c4dfb963750d4a`. +The public Frontend 0.3 → native SQLite apply regression checks UNKNOWN, +metadata, provenance and unchanged dataset/table identity without extra data +artifacts or history. This change has no local network-service evidence; its +service CI gate remains pending. SQLite runs locally. PostgreSQL, MySQL, MariaDB, and Dolt cases run when their `OPENSTATSPEC_*` service configuration is supplied; CI configures every service family. diff --git a/src/Core/CapabilityDeclaration.php b/src/Core/CapabilityDeclaration.php index a6ab29f..7860720 100644 --- a/src/Core/CapabilityDeclaration.php +++ b/src/Core/CapabilityDeclaration.php @@ -65,6 +65,9 @@ public function toArray(): array 'official_frontend_contracts' => [ 'openstatspec-spss-syntax-frontend-v0.2', ], + 'opt_in_frontend_contracts' => [ + 'openstatspec-spss-syntax-frontend-v0.3' => 'implemented_service_ci_pending', + ], 'official_binding_contracts' => [ 'openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2', diff --git a/src/Frontend/Spss/Ast/FormatTarget.php b/src/Frontend/Spss/Ast/FormatTarget.php index 2706753..e0ba1b9 100644 --- a/src/Frontend/Spss/Ast/FormatTarget.php +++ b/src/Frontend/Spss/Ast/FormatTarget.php @@ -9,7 +9,7 @@ final readonly class FormatTarget { public function __construct( - public string $variable, + public string|VariableRange $variable, public string $family, public int $width, public int $decimals, diff --git a/src/Frontend/Spss/Ast/NotPredicate.php b/src/Frontend/Spss/Ast/NotPredicate.php new file mode 100644 index 0000000..fabf2b1 --- /dev/null +++ b/src/Frontend/Spss/Ast/NotPredicate.php @@ -0,0 +1,17 @@ +span; + } +} diff --git a/src/Frontend/Spss/Ast/Program.php b/src/Frontend/Spss/Ast/Program.php index eea677e..f7ba6b9 100644 --- a/src/Frontend/Spss/Ast/Program.php +++ b/src/Frontend/Spss/Ast/Program.php @@ -7,5 +7,5 @@ final readonly class Program { /** @param list $statements */ - public function __construct(public array $statements) {} + public function __construct(public array $statements, public bool $officialV03 = false) {} } diff --git a/src/Frontend/Spss/Ast/RecodeStatement.php b/src/Frontend/Spss/Ast/RecodeStatement.php index a332991..3edd467 100644 --- a/src/Frontend/Spss/Ast/RecodeStatement.php +++ b/src/Frontend/Spss/Ast/RecodeStatement.php @@ -9,7 +9,7 @@ final readonly class RecodeStatement implements Statement { /** - * @param non-empty-list $sources + * @param non-empty-list $sources * @param non-empty-list $rules * @param list $targets */ diff --git a/src/Frontend/Spss/Ast/ValueLabelGroup.php b/src/Frontend/Spss/Ast/ValueLabelGroup.php index 3f87715..9aa5b9a 100644 --- a/src/Frontend/Spss/Ast/ValueLabelGroup.php +++ b/src/Frontend/Spss/Ast/ValueLabelGroup.php @@ -9,7 +9,7 @@ final readonly class ValueLabelGroup { /** - * @param non-empty-list $variables + * @param non-empty-list $variables * @param non-empty-list $labels */ public function __construct( diff --git a/src/Frontend/Spss/Ast/ValueLabelsStatement.php b/src/Frontend/Spss/Ast/ValueLabelsStatement.php index 966e986..9d58e98 100644 --- a/src/Frontend/Spss/Ast/ValueLabelsStatement.php +++ b/src/Frontend/Spss/Ast/ValueLabelsStatement.php @@ -13,6 +13,7 @@ public function __construct( public int $sourceLine, public array $groups, public SourceSpan $span, + public bool $add = false, ) {} public function line(): int diff --git a/src/Frontend/Spss/Ast/VariableLabelAssignment.php b/src/Frontend/Spss/Ast/VariableLabelAssignment.php index b1f4e42..e1352e7 100644 --- a/src/Frontend/Spss/Ast/VariableLabelAssignment.php +++ b/src/Frontend/Spss/Ast/VariableLabelAssignment.php @@ -9,7 +9,7 @@ final readonly class VariableLabelAssignment { public function __construct( - public string $variable, + public string|VariableRange $variable, public string $label, public SourceSpan $variableSpan, public SourceSpan $span, diff --git a/src/Frontend/Spss/Ast/VariableLevelGroup.php b/src/Frontend/Spss/Ast/VariableLevelGroup.php index 28b0625..8a2c2d5 100644 --- a/src/Frontend/Spss/Ast/VariableLevelGroup.php +++ b/src/Frontend/Spss/Ast/VariableLevelGroup.php @@ -8,7 +8,7 @@ final readonly class VariableLevelGroup { - /** @param non-empty-list $variables */ + /** @param non-empty-list $variables */ public function __construct( public array $variables, /** @var non-empty-list */ diff --git a/src/Frontend/Spss/Ast/VariableRange.php b/src/Frontend/Spss/Ast/VariableRange.php new file mode 100644 index 0000000..e336b83 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableRange.php @@ -0,0 +1,16 @@ +officialV03); $bound = []; foreach ($program->statements as $statement) { if ($statement instanceof RecodeStatement) { @@ -86,22 +87,23 @@ public function bind(string $inputAlias, InputSchema $inputSchema, Program $prog } if ($statement instanceof FormatsStatement) { foreach ($statement->targets as $target) { - $variable = $schema->resolve($target->variable, $target->variableSpan); - if ($variable->storageKind !== 'numeric') { - $this->fail( - 'expression_type_unsupported', - 'Numeric F formats cannot target string variables.', - $target->variableSpan, + foreach ($schema->expand($target->variable, $target->variableSpan) as $variable) { + if ($variable->storageKind !== 'numeric') { + $this->fail( + 'expression_type_unsupported', + 'Numeric F formats cannot target string variables.', + $target->variableSpan, + ); + } + $this->validateFormat($target->width, $target->decimals, $target->span); + $bound[] = new BoundFormat( + $variable->name, + $target->family, + $target->width, + $target->decimals, + $target->span, ); } - $this->validateFormat($target->width, $target->decimals, $target->span); - $bound[] = new BoundFormat( - $variable->name, - $target->family, - $target->width, - $target->decimals, - $target->span, - ); } continue; } @@ -109,8 +111,9 @@ public function bind(string $inputAlias, InputSchema $inputSchema, Program $prog foreach ($statement->groups as $group) { foreach ($group->variables as $index => $name) { $span = $group->variableSpans[$index]; - $variable = $schema->resolve($name, $span); - $bound[] = new BoundMeasurementLevel($variable->name, $group->level, $span); + foreach ($schema->expand($name, $span) as $variable) { + $bound[] = new BoundMeasurementLevel($variable->name, $group->level, $span); + } } } continue; @@ -121,8 +124,9 @@ public function bind(string $inputAlias, InputSchema $inputSchema, Program $prog } if ($statement instanceof VariableLabelsStatement) { foreach ($statement->assignments as $assignment) { - $variable = $schema->resolve($assignment->variable, $assignment->variableSpan); - $bound[] = new BoundVariableLabel($variable->name, $assignment->label, $assignment->span); + foreach ($schema->expand($assignment->variable, $assignment->variableSpan) as $variable) { + $bound[] = new BoundVariableLabel($variable->name, $assignment->label, $assignment->span); + } } continue; } @@ -226,8 +230,14 @@ private function operand(AstExpressionOperand $operand, SchemaState $schema): ar throw new \LogicException(sprintf('Unsupported operand %s.', $operand::class)); } - private function predicate(AstPredicate $predicate, SchemaState $schema): Predicate + private function predicate(AstPredicate $predicate, SchemaState $schema, bool $negated = false): Predicate { + if ($predicate instanceof NotPredicate) { + if (!$schema->officialV03) { + $this->fail('expression_type_unsupported', 'NOT requires Frontend 0.3.', $predicate->span); + } + return $this->predicate($predicate->operand, $schema, !$negated); + } if ($predicate instanceof AstComparison) { [$left, $leftType] = $this->operand($predicate->left, $schema); [$right, $rightType] = $this->operand($predicate->right, $schema); @@ -239,13 +249,30 @@ private function predicate(AstPredicate $predicate, SchemaState $schema): Predic ); } - return new Comparison($left, $predicate->operator, $right); + $operator = strtoupper($predicate->operator); + if (in_array($operator, ['NE', '<>', '~='], true)) { + if (!$schema->officialV03) { + $this->fail('spss_syntax_error', 'Comparison aliases require Frontend 0.3.', $predicate->span); + } + return new BooleanPredicate( + $negated ? 'and' : 'or', + new Comparison($left, $negated ? '>=' : '<', $right), + new Comparison($left, $negated ? '<=' : '>', $right), + ); + } + if ($negated && $operator === '=') { + return new BooleanPredicate('or', new Comparison($left, '<', $right), new Comparison($left, '>', $right)); + } + return new Comparison($left, $negated ? ['<' => '>=', '<=' => '>', '>' => '<=', '>=' => '<'][$operator] : $operator, $right); } if ($predicate instanceof AstBooleanPredicate) { - return new BooleanPredicate( - $predicate->operator, - ...array_map(fn(AstPredicate $operand): Predicate => $this->predicate($operand, $schema), $predicate->operands), - ); + $operator = $negated ? ($predicate->operator === 'and' ? 'or' : 'and') : $predicate->operator; + $operands = []; + foreach ($predicate->operands as $operand) { + $lowered = $this->predicate($operand, $schema, $negated); + array_push($operands, ...($lowered instanceof BooleanPredicate && $lowered->operator === $operator ? $lowered->operands : [$lowered])); + } + return new BooleanPredicate($operator, ...$operands); } throw new \LogicException(sprintf('Unsupported predicate %s.', $predicate::class)); @@ -255,8 +282,12 @@ private function predicate(AstPredicate $predicate, SchemaState $schema): Predic private function recode(RecodeStatement $statement, SchemaState $schema): array { $sources = []; + $sourceSpans = []; foreach ($statement->sources as $index => $sourceName) { - $sources[] = $schema->resolve($sourceName, $statement->sourceSpans[$index]); + foreach ($schema->expand($sourceName, $statement->sourceSpans[$index]) as $source) { + $sources[] = $source; + $sourceSpans[] = $statement->sourceSpans[$index]; + } } $targetMode = $statement->targets === [] ? TargetMode::Replace : TargetMode::Create; @@ -274,7 +305,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array foreach ($targetNames as $index => $targetName) { $targetSpan = $targetMode === TargetMode::Create ? $statement->targetSpans[$index] - : $statement->sourceSpans[$index]; + : $sourceSpans[$index]; $this->validateTargetName($targetName, $targetSpan); if ($targetMode === TargetMode::Create) { $key = mb_strtolower($targetName, 'UTF-8'); @@ -320,7 +351,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array $unmatched = $result; continue; } - $rules[] = new RecodeRule($this->recodeMatch($rule->input, $source), $result); + $rules[] = new RecodeRule($this->recodeMatch($rule->input, $source, $schema->officialV03), $result); } $unmatched ??= $targetMode === TargetMode::Create ? new SystemMissingResult() : new CopyResult(); $resultTypes = array_values(array_unique(array_map( @@ -329,7 +360,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array ))); $targetSpan = $targetMode === TargetMode::Create ? $statement->targetSpans[$index] - : $statement->sourceSpans[$index]; + : $sourceSpans[$index]; if ($targetMode === TargetMode::Create && in_array('string', $resultTypes, true)) { $this->fail( 'string_target_requires_declaration', @@ -370,7 +401,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array return $bound; } - private function recodeMatch(RecodeInput $input, InputVariable $source): RecodeMatch + private function recodeMatch(RecodeInput $input, InputVariable $source, bool $officialV03): RecodeMatch { if ($input instanceof MissingInput) { $this->fail( @@ -391,16 +422,16 @@ private function recodeMatch(RecodeInput $input, InputVariable $source): RecodeM return new SystemMissingMatch(); } if ($input instanceof RangeInput) { - if ($input->lower === null || $input->upper === null) { + if (!$officialV03 && ($input->lower === null || $input->upper === null)) { $this->fail('spss_syntax_error', 'RECODE ranges require two finite endpoints.', $input->span); } - $lower = $this->typedValue($input->lower); - $upper = $this->typedValue($input->upper); + $lower = $input->lower === null ? Binary64Value::fromBits('ffefffffffffffff') : $this->typedValue($input->lower); + $upper = $input->upper === null ? Binary64Value::fromBits('7fefffffffffffff') : $this->typedValue($input->upper); if (!$lower instanceof Binary64Value || !$upper instanceof Binary64Value || $source->storageKind !== 'numeric') { $this->fail('type_mismatch', 'RECODE ranges require numeric sources and endpoints.', $input->span); } if ($lower->number() > $upper->number()) { - $this->fail('invalid_numeric_range', 'RECODE range lower bound exceeds its upper bound.', $input->span); + $this->fail($officialV03 ? 'invalid_variable_range' : 'invalid_numeric_range', 'RECODE range lower bound exceeds its upper bound.', $input->span); } return new RangeMatch($lower, $upper); @@ -460,6 +491,9 @@ private function resultType(Result $result, InputVariable $source): string /** @return list */ private function valueLabels(ValueLabelsStatement $statement, SchemaState $schema): array { + if ($statement->add && !$schema->officialV03) { + $this->fail('unsupported_spss_command', 'ADD VALUE LABELS requires Frontend 0.3.', $statement->span); + } $bound = []; foreach ($statement->groups as $group) { $labels = array_map( @@ -470,28 +504,48 @@ private function valueLabels(ValueLabelsStatement $statement, SchemaState $schem $group->labels, ); foreach ($group->variables as $index => $name) { - $variable = $schema->resolve($name, $group->variableSpans[$index]); - $expectedType = $this->storageType($variable); - $seen = []; - foreach ($labels as $label) { - if ($this->typedValueType($label->value) !== $expectedType) { - $this->fail( - 'type_mismatch', - 'VALUE LABELS codes must match the variable storage kind.', - $group->span, - ); + foreach ($schema->expand($name, $group->variableSpans[$index]) as $variable) { + $expectedType = $this->storageType($variable); + $seen = []; + foreach ($labels as $label) { + if ($this->typedValueType($label->value) !== $expectedType) { + $this->fail( + 'type_mismatch', + 'VALUE LABELS codes must match the variable storage kind.', + $group->span, + ); + } + $key = $label->value->canonicalKey(); + if (isset($seen[$key])) { + $this->fail( + 'duplicate_value_label', + 'VALUE LABELS contains duplicate canonical codes.', + $group->span, + ); + } + $seen[$key] = true; } - $key = $label->value->canonicalKey(); - if (isset($seen[$key])) { - $this->fail( - 'duplicate_value_label', - 'VALUE LABELS contains duplicate canonical codes.', - $group->span, - ); + $replacement = $labels; + if ($statement->add) { + $ordered = []; + foreach ($schema->valueLabels($variable) as $label) { + if ($this->typedValueType($label->value) !== $expectedType) { + $this->fail('type_mismatch', 'Existing value labels must match the variable type.', $group->span); + } + $key = $label->value->canonicalKey(); + if (isset($ordered[$key])) { + $this->fail('duplicate_value_label', 'Existing value labels have duplicate codes.', $group->span); + } + $ordered[$key] = $label; + } + foreach ($labels as $label) { + $ordered[$label->value->canonicalKey()] = $label; + } + $replacement = array_values($ordered); } - $seen[$key] = true; + $schema->setValueLabels($variable, $replacement); + $bound[] = new BoundValueLabels($variable->name, $replacement, $group->span); } - $bound[] = new BoundValueLabels($variable->name, $labels, $group->span); } } diff --git a/src/Frontend/Spss/Binding/SchemaState.php b/src/Frontend/Spss/Binding/SchemaState.php index 5106ed8..d7d7e1f 100644 --- a/src/Frontend/Spss/Binding/SchemaState.php +++ b/src/Frontend/Spss/Binding/SchemaState.php @@ -4,7 +4,11 @@ namespace OpenStatSpec\Frontend\Spss\Binding; +use OpenStatSpec\Frontend\Spss\Ast\VariableRange; use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; +use OpenStatSpec\Transformation\Plan\Value\StringValue; use OpenStatSpec\Frontend\Spss\Request\InputVariable; use OpenStatSpec\Transformation\Diagnostic\SourceSpan; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; @@ -17,7 +21,10 @@ final class SchemaState /** @var array> */ private array $index = []; - public function __construct(InputSchema $schema) + /** @var array> */ + private array $labels = []; + + public function __construct(InputSchema $schema, public readonly bool $officialV03 = false) { $this->variables = $schema->variables; foreach ($this->variables as $variable) { @@ -43,6 +50,50 @@ public function resolve( return $matches[0]; } + /** @return non-empty-list */ + public function expand(string|VariableRange $name, SourceSpan $span): array + { + if (is_string($name)) { + return [$this->resolve($name, $span)]; + } + if (!$this->officialV03) { + throw TransformationFailure::at('spss_syntax_error', '$.source_text', 'TO requires Frontend 0.3.', $span); + } + $resolved = $this->expand($name->first, $name->span); + $first = array_search($resolved[array_key_last($resolved)], $this->variables, true); + $last = array_search($this->resolve($name->last, $name->span), $this->variables, true); + if ($first === false || $last === false) { + throw new \LogicException('Resolved variables must be in dictionary order.'); + } + if ($first > $last) { + throw TransformationFailure::at('invalid_variable_range', '$.source_text', 'TO range is reversed.', $span); + } + foreach (array_slice($this->variables, $first + 1, $last - $first) as $variable) { + $resolved[] = $this->resolve($variable->name, $span); + } + return $resolved; + } + + /** @return list */ + public function valueLabels(InputVariable $variable): array + { + return $this->labels[$variable->name] ?? array_map( + static fn(array $label): ValueLabel => new ValueLabel( + $label['value']['type'] === 'binary64' + ? Binary64Value::fromBits($label['value']['bits']) + : new StringValue($label['value']['value']), + $label['label'], + ), + $variable->valueLabels, + ); + } + + /** @param list $labels */ + public function setValueLabels(InputVariable $variable, array $labels): void + { + $this->labels[$variable->name] = $labels; + } + public function find(string $name, SourceSpan $span): ?InputVariable { $matches = $this->index[$this->key($name)] ?? []; diff --git a/src/Frontend/Spss/Lexer.php b/src/Frontend/Spss/Lexer.php index d464670..fc34556 100644 --- a/src/Frontend/Spss/Lexer.php +++ b/src/Frontend/Spss/Lexer.php @@ -10,7 +10,7 @@ final class Lexer { /** @return list */ - public function tokenize(string $source): array + public function tokenize(string $source, bool $officialV03 = false): array { $tokens = []; $offset = str_starts_with($source, "\xEF\xBB\xBF") ? 3 : 0; @@ -30,7 +30,15 @@ public function tokenize(string $source): array continue; } + if ($officialV03 && substr($source, $offset, 2) === '/*') { + $this->comment($source, $offset, $line, $column, true); + continue; + } if ($atStatementStart && $character === '*') { + if ($officialV03) { + $this->comment($source, $offset, $line, $column, false); + continue; + } $this->fail( new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), 'Leading-star comments are not supported.', @@ -41,6 +49,7 @@ public function tokenize(string $source): array $tokenOffset = $offset; $tokenLine = $line; $tokenColumn = $column; + $wasStatementStart = $atStatementStart; $atStatementStart = false; if ($this->startsNumber($source, $offset)) { @@ -56,6 +65,7 @@ public function tokenize(string $source): array ); } $punctuation = match (true) { + $officialV03 && (($character === '<' && $next === '>') || ($character === '~' && $next === '=')) => TokenType::NotEqual, $character === '<' && $next === '=' => TokenType::LessThanOrEqual, $character === '>' && $next === '=' => TokenType::GreaterThanOrEqual, $character === '<' => TokenType::LessThan, @@ -70,7 +80,7 @@ public function tokenize(string $source): array default => null, }; if ($punctuation !== null) { - $lexeme = ($punctuation === TokenType::LessThanOrEqual || $punctuation === TokenType::GreaterThanOrEqual) + $lexeme = in_array($punctuation, [TokenType::LessThanOrEqual, TokenType::GreaterThanOrEqual, TokenType::NotEqual], true) ? $character . $next : $character; $this->advance($character, $offset, $line, $column); @@ -108,6 +118,11 @@ public function tokenize(string $source): array $this->advance($identifierCharacter, $offset, $line, $column); } $lexeme = substr($source, $start, $offset - $start); + if ($officialV03 && $wasStatementStart && strtoupper($lexeme) === 'COMMENT') { + $this->comment($source, $offset, $line, $column, false); + $atStatementStart = true; + continue; + } $type = match (strtoupper($lexeme)) { 'COMPUTE' => TokenType::Compute, 'IF' => TokenType::If, @@ -146,6 +161,36 @@ public function tokenize(string $source): array return $tokens; } + private function comment(string $source, int &$offset, int &$line, int &$column, bool $block): void + { + $start = new SourceSpan($offset, $offset, $line, $column, $line, $column); + if ($block) { + $this->advance('/', $offset, $line, $column); + $this->advance('*', $offset, $line, $column); + } + while ($offset < strlen($source)) { + $pair = substr($source, $offset, 2); + if ($block && $pair === '/*') { + $this->fail($start, 'Nested block comments are not supported.'); + } + if ($block && $pair === '*/') { + $this->advance('*', $offset, $line, $column); + $this->advance('/', $offset, $line, $column); + return; + } + $character = $this->characterAt($source, $offset, $line, $column); + if ($character === "\r") { + $this->advanceCarriageReturn($source, $offset, $line, $column); + } else { + $this->advance($character, $offset, $line, $column); + } + if (!$block && $character === '.') { + return; + } + } + $this->fail($start, 'Unterminated comment.'); + } + private function string(string $source, int &$offset, int &$line, int &$column): Token { $quote = $this->characterAt($source, $offset, $line, $column); diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php index 3d4e812..1f54b38 100644 --- a/src/Frontend/Spss/Parser.php +++ b/src/Frontend/Spss/Parser.php @@ -15,6 +15,8 @@ use OpenStatSpec\Frontend\Spss\Ast\IfStatement; use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; +use OpenStatSpec\Frontend\Spss\Ast\NotPredicate; +use OpenStatSpec\Frontend\Spss\Ast\VariableRange; use OpenStatSpec\Frontend\Spss\Ast\Program; use OpenStatSpec\Frontend\Spss\Ast\Predicate; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; @@ -44,17 +46,19 @@ final class Parser /** @var list */ private array $tokens = []; private int $position = 0; + private bool $officialV03 = false; public function __construct(private readonly Lexer $lexer = new Lexer()) {} - public function parse(string $source): Program + public function parse(string $source, bool $officialV03 = false): Program { - return $this->parseTokens($this->lexer->tokenize($source)); + return $this->parseTokens($this->lexer->tokenize($source, $officialV03), $officialV03); } /** @param list $tokens */ - public function parseTokens(array $tokens): Program + public function parseTokens(array $tokens, bool $officialV03 = false): Program { + $this->officialV03 = $officialV03; $this->tokens = $tokens; $this->position = 0; $statements = []; @@ -63,6 +67,9 @@ public function parseTokens(array $tokens): Program $command = $this->current(); if ($this->matchKeyword('RECODE')) { $statements[] = $this->recode($command); + while ($this->officialV03 && $this->match(TokenType::Slash)) { + $statements[] = $this->recode($this->current()); + } } elseif ($this->matchKeyword('VARIABLE')) { if ($this->matchKeyword('LABELS')) { $statements[] = $this->variableLabels($command); @@ -71,6 +78,10 @@ public function parseTokens(array $tokens): Program } else { $this->fail($this->current(), 'Expected LABELS or LEVEL after VARIABLE.'); } + } elseif ($this->officialV03 && $this->matchKeyword('ADD')) { + $this->consumeKeyword('VALUE', 'Expected VALUE after ADD.'); + $this->consumeKeyword('LABELS', 'Expected LABELS after ADD VALUE.'); + $statements[] = $this->valueLabels($command, true); } elseif ($this->matchKeyword('VALUE')) { $this->consumeKeyword('LABELS', 'Expected LABELS after VALUE.'); $statements[] = $this->valueLabels($command); @@ -101,7 +112,7 @@ public function parseTokens(array $tokens): Program $this->consume(TokenType::Terminator, 'Expected a period after the SPSS command.'); } - return new Program($statements); + return new Program($statements, $this->officialV03); } private function compute(Token $command): ComputeStatement @@ -155,14 +166,24 @@ private function parseOr(): Predicate private function parseAnd(): Predicate { - $predicate = $this->comparison(); + $predicate = $this->parseNot(); while ($this->matchKeyword('AND')) { - $predicate = $this->merge('and', $predicate, $this->comparison()); + $predicate = $this->merge('and', $predicate, $this->parseNot()); } return $predicate; } + private function parseNot(): Predicate + { + if ($this->officialV03 && $this->matchKeyword('NOT')) { + $start = $this->previous(); + $operand = $this->parseNot(); + return new NotPredicate($operand, SourceSpan::cover($start->span, $operand->span())); + } + return $this->comparison(); + } + private function comparison(): Predicate { if ($this->match(TokenType::LeftParenthesis)) { @@ -192,6 +213,9 @@ private function comparisonOperator(): Token 'expression_type_unsupported', ); } + if ($this->officialV03 && ($this->match(TokenType::NotEqual) || $this->matchKeyword('NE'))) { + return $this->previous(); + } foreach ([ TokenType::Equals, TokenType::LessThan, @@ -204,7 +228,11 @@ private function comparisonOperator(): Token } } - $this->fail($this->current(), 'Expected a supported comparison operator.'); + $this->fail( + $this->current(), + 'Expected a supported comparison operator.', + $this->officialV03 && $this->check(TokenType::RightParenthesis) ? 'expression_type_unsupported' : 'spss_syntax_error', + ); } private function expressionOperand(string $message): ExpressionOperand @@ -272,7 +300,7 @@ private function formats(Token $command): FormatsStatement { $targets = []; do { - $variable = $this->consumeIdentifier('FORMATS requires a variable and format pair.'); + [$variables, $spans] = $this->existingVariables('FORMATS requires variables and a format.', $this->officialV03); $this->consume(TokenType::LeftParenthesis, 'Expected ( before the format.'); $familyAndWidth = $this->consumeIdentifier('Expected an F format such as F8.2.'); if (preg_match('/\AF([1-9][0-9]*)\z/iD', $familyAndWidth->lexeme, $matches) !== 1) { @@ -286,15 +314,17 @@ private function formats(Token $command): FormatsStatement $this->fail($decimalToken, 'F format must use the Fwidth.decimals form.', 'invalid_format'); } $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the format.'); - $targets[] = new FormatTarget( - $variable->lexeme, - 'F', - (int) $matches[1], - (int) $decimalMatches[1], - $variable->span, - SourceSpan::cover($variable->span, $rightParenthesis->span), - ); - } while ($this->current()->isWord()); + foreach ($variables as $index => $variable) { + $targets[] = new FormatTarget( + $variable, + 'F', + (int) $matches[1], + (int) $decimalMatches[1], + $spans[$index], + SourceSpan::cover($spans[$index], $rightParenthesis->span), + ); + } + } while (($this->officialV03 && $this->match(TokenType::Slash)) || $this->current()->isWord()); return new FormatsStatement( $targets, @@ -306,16 +336,7 @@ private function variableLevel(Token $command): VariableLevelStatement { $groups = []; do { - $variables = []; - $variableSpans = []; - $first = $this->consumeIdentifier('VARIABLE LEVEL requires at least one variable.'); - $variables[] = $first->lexeme; - $variableSpans[] = $first->span; - while ($this->current()->isWord()) { - $variable = $this->advance(); - $variables[] = $variable->lexeme; - $variableSpans[] = $variable->span; - } + [$variables, $variableSpans] = $this->existingVariables('VARIABLE LEVEL requires at least one variable.'); $this->consume(TokenType::LeftParenthesis, 'Expected ( before the measurement level.'); $level = $this->current(); if ( @@ -330,7 +351,7 @@ private function variableLevel(Token $command): VariableLevelStatement $variables, $variableSpans, strtolower($level->lexeme), - SourceSpan::cover($first->span, $rightParenthesis->span), + SourceSpan::cover($variableSpans[0], $rightParenthesis->span), ); } while ($this->match(TokenType::Slash)); @@ -342,13 +363,7 @@ private function variableLevel(Token $command): VariableLevelStatement private function recode(Token $command): RecodeStatement { - $sources = []; - $sourceSpans = []; - do { - $source = $this->consumeIdentifier('Expected a source variable after RECODE.'); - $sources[] = $source->lexeme; - $sourceSpans[] = $source->span; - } while ($this->current()->isWord() && !$this->current()->isKeyword('INTO')); + [$sources, $sourceSpans] = $this->existingVariables('Expected a source variable after RECODE.', stopAtInto: true); $rules = []; while ($this->match(TokenType::LeftParenthesis)) { @@ -382,6 +397,9 @@ private function recode(Token $command): RecodeStatement $targetSpans = []; if ($this->matchKeyword('INTO')) { do { + if ($this->officialV03 && $this->current()->isKeyword('TO')) { + $this->fail($this->current(), 'TO is forbidden in generated INTO targets.'); + } $target = $this->consumeIdentifier('Expected a target variable after INTO.'); $targets[] = $target->lexeme; $targetSpans[] = $target->span; @@ -452,15 +470,20 @@ private function variableLabels(Token $command): VariableLabelsStatement $labels = []; $assignments = []; while (!$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { - $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.'); + [$variables, $spans] = $this->existingVariables('Expected a variable name in VARIABLE LABELS.', $this->officialV03); $label = $this->consume(TokenType::String, 'Expected a quoted variable label.'); - $labels[$variable->lexeme] = $label->lexeme; - $assignments[] = new VariableLabelAssignment( - $variable->lexeme, - $label->lexeme, - $variable->span, - SourceSpan::cover($variable->span, $label->span), - ); + foreach ($variables as $index => $variable) { + $labels[is_string($variable) ? $variable : $variable->last] = $label->lexeme; + $assignments[] = new VariableLabelAssignment( + $variable, + $label->lexeme, + $spans[$index], + SourceSpan::cover($spans[$index], $label->span), + ); + } + if ($this->officialV03 && $this->match(TokenType::Slash) && !$this->current()->isWord()) { + $this->fail($this->current(), 'Expected another variable label group.'); + } } if ($labels === []) { $this->fail($this->current(), 'VARIABLE LABELS requires at least one variable and label.'); @@ -474,20 +497,11 @@ private function variableLabels(Token $command): VariableLabelsStatement ); } - private function valueLabels(Token $command): ValueLabelsStatement + private function valueLabels(Token $command, bool $add = false): ValueLabelsStatement { $groups = []; do { - $variables = []; - $variableSpans = []; - while ($this->current()->isWord()) { - $variable = $this->advance(); - $variables[] = $variable->lexeme; - $variableSpans[] = $variable->span; - } - if ($variables === []) { - $this->fail($this->current(), 'VALUE LABELS requires at least one variable before its value-label pairs.'); - } + [$variables, $variableSpans] = $this->existingVariables('VALUE LABELS requires at least one variable before its value-label pairs.'); $labels = []; while (!$this->check(TokenType::Slash) && !$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { @@ -510,9 +524,37 @@ private function valueLabels(Token $command): ValueLabelsStatement $command->line, $groups, SourceSpan::cover($command->span, $groups[array_key_last($groups)]->span), + $add, ); } + /** @return array{non-empty-list, non-empty-list} */ + private function existingVariables(string $message, bool $multiple = true, bool $stopAtInto = false): array + { + $first = $this->consumeIdentifier($message); + if ($this->officialV03 && $first->isKeyword('TO')) { + $this->fail($first, 'TO requires a preceding variable.'); + } + $variables = [$first->lexeme]; + $spans = [$first->span]; + while ($multiple && $this->current()->isWord() && !($stopAtInto && $this->current()->isKeyword('INTO'))) { + if ($this->officialV03 && $this->matchKeyword('TO')) { + $last = $this->consumeIdentifier('TO requires an endpoint.'); + if ($last->isKeyword('TO')) { + $this->fail($last, 'TO requires an endpoint.'); + } + $index = array_key_last($variables); + $spans[$index] = SourceSpan::cover($spans[$index], $last->span); + $variables[$index] = new VariableRange($variables[$index], $last->lexeme, $spans[$index]); + } else { + $variable = $this->advance(); + $variables[] = $variable->lexeme; + $spans[] = $variable->span; + } + } + return [$variables, array_values($spans)]; + } + private function scalar(string $message): ScalarValue { if ($this->match(TokenType::String)) { diff --git a/src/Frontend/Spss/Request/InputVariable.php b/src/Frontend/Spss/Request/InputVariable.php index e23bbb2..9ff6a27 100644 --- a/src/Frontend/Spss/Request/InputVariable.php +++ b/src/Frontend/Spss/Request/InputVariable.php @@ -79,7 +79,7 @@ public static function fromArray(mixed $raw, string $path): self $name, $storageKind, $variableLabel, - self::valueLabels($raw['value_labels'] ?? [], $path . '.value_labels'), + self::valueLabels(array_key_exists('value_labels', $raw) ? $raw['value_labels'] : [], $path . '.value_labels'), $formatFamily, $width, $decimals, diff --git a/src/Frontend/Spss/Request/SpssFrontendRequest.php b/src/Frontend/Spss/Request/SpssFrontendRequest.php index f2c1fe6..a186494 100644 --- a/src/Frontend/Spss/Request/SpssFrontendRequest.php +++ b/src/Frontend/Spss/Request/SpssFrontendRequest.php @@ -9,6 +9,7 @@ final readonly class SpssFrontendRequest { public const CONTRACT = 'openstatspec-spss-syntax-frontend-v0.2'; + public const CONTRACT_V03 = 'openstatspec-spss-syntax-frontend-v0.3'; public function __construct( public string $contract, @@ -16,7 +17,7 @@ public function __construct( public InputSchema $inputSchema, public string $sourceText, ) { - if ($contract !== self::CONTRACT) { + if (!in_array($contract, [self::CONTRACT, self::CONTRACT_V03], true)) { self::schema('$.contract', 'Unsupported SPSS frontend contract.'); } self::nonEmptyString($inputAlias, '$.input_alias'); diff --git a/src/Frontend/Spss/SpssCompiler.php b/src/Frontend/Spss/SpssCompiler.php index d382b36..7a95242 100644 --- a/src/Frontend/Spss/SpssCompiler.php +++ b/src/Frontend/Spss/SpssCompiler.php @@ -31,7 +31,7 @@ public function bind(string $inputAlias, InputSchema $inputSchema, Program $prog public function compile(SpssFrontendRequest $request): SpssCompilationResult { $sourceHash = $request->sourceHash(); - $program = $this->parser->parse($request->sourceText); + $program = $this->parser->parse($request->sourceText, $request->contract === SpssFrontendRequest::CONTRACT_V03); $bound = $this->binder->bind($request->inputAlias, $request->inputSchema, $program); return new SpssCompilationResult($this->compiler->compile($bound), $sourceHash); diff --git a/src/Frontend/Spss/TokenType.php b/src/Frontend/Spss/TokenType.php index f369b98..7d9786c 100644 --- a/src/Frontend/Spss/TokenType.php +++ b/src/Frontend/Spss/TokenType.php @@ -22,6 +22,7 @@ enum TokenType case LeftParenthesis; case RightParenthesis; case Equals; + case NotEqual; case LessThan; case LessThanOrEqual; case GreaterThan; diff --git a/tests/Core/CapabilityDeclarationTest.php b/tests/Core/CapabilityDeclarationTest.php index 9cba19f..0da3aae 100644 --- a/tests/Core/CapabilityDeclarationTest.php +++ b/tests/Core/CapabilityDeclarationTest.php @@ -61,6 +61,9 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], $contracts['official_binding_contracts'], ); + self::assertSame([ + 'openstatspec-spss-syntax-frontend-v0.3' => 'implemented_service_ci_pending', + ], $contracts['opt_in_frontend_contracts']); self::assertArrayNotHasKey('legacy_plan_contracts', $contracts); self::assertFalse(class_exists(\OpenStatSpec\Transformation\Model\TransformationPlan::class)); self::assertSame('3.1.1', $declaration['engine']['active_version']); diff --git a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php index 03ec1a0..d75765c 100644 --- a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php +++ b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php @@ -43,11 +43,62 @@ public function testOfficialFrontend02Case(array $case): void (string) $case['id'], ); + foreach ($case['expected_output_metadata'] ?? [] as $name => $metadata) { + $variables = array_column($request->inputSchema->variables, null, 'name'); + self::assertSame($metadata, [ + 'variable_label' => $variables[$name]->variableLabel, + 'value_labels' => $variables[$name]->valueLabels, + ]); + // This inherited case promises RECODE leaves dictionary metadata alone. + self::assertSame([], array_values(array_filter( + $result->plan->canonicalArray()['operations'], + static fn(array $operation): bool => ($operation['variable'] ?? null) === $name, + ))); + } if (isset($case['expected_plan_contract'])) { self::assertSame($case['expected_plan_contract'], $result->plan->contract->value, (string) $case['id']); } } + /** @param array $case */ + #[DataProvider('official03Cases')] + public function testOfficialFrontend03Case(array $case): void + { + $this->testOfficialFrontend02Case($case); + } + + public function testOfficial03ManifestCoverage(): void + { + self::assertCount(35, SpecificationManifest::load('conformance/spss-syntax-frontend-0.3.json')['cases']); + self::assertCount(90, iterator_to_array(self::official03Cases())); + } + + /** @return iterable}> */ + public static function official03Cases(): iterable + { + $manifest = SpecificationManifest::load('conformance/spss-syntax-frontend-0.3.json'); + foreach ($manifest['inherited_manifests'] as $inheritance) { + foreach (SpecificationManifest::load('conformance/' . $inheritance['manifest'])['cases'] as $case) { + if (isset($inheritance['superseded_cases'][$case['id']])) { + continue; + } + if ($inheritance['manifest'] === 'spss-syntax-frontend-0.1.json' && isset($case['expected_plan_case'])) { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $planCase) { + if ($planCase['id'] === $case['expected_plan_case']) { + $case['expected_plan'] = $planCase['plan']; + $case['expected_plan_hash'] = $planCase['expected_plan_hash']; + } + } + } + $case['request']['contract'] = $inheritance['request_contract_override']; + yield $inheritance['manifest'] . '/' . $case['id'] => [$case]; + } + } + foreach ($manifest['cases'] as $case) { + yield '0.3/' . $case['id'] => [$case]; + } + } + /** @return iterable}> */ public static function cases(): iterable { @@ -65,6 +116,9 @@ public static function cases(): iterable */ private function expectedPlan(array $case): array { + if (isset($case['expected_plan']) && is_array($case['expected_plan'])) { + return $case['expected_plan']; + } if (isset($case['expected_plan_case'])) { return $this->namedPlan('conformance/transformation-plan-0.2.json', (string) $case['expected_plan_case'], 'plan'); } diff --git a/tests/Frontend/Spss/SpssFrontend03Test.php b/tests/Frontend/Spss/SpssFrontend03Test.php new file mode 100644 index 0000000..686efa0 --- /dev/null +++ b/tests/Frontend/Spss/SpssFrontend03Test.php @@ -0,0 +1,173 @@ + $payload */ + #[DataProvider('invalidRequests')] + public function testProductionRequestBoundaryRejectsInvalidShapes(array $payload): void + { + try { + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($payload)); + self::fail('Invalid request compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + } + } + + /** @return iterable}> */ + public static function invalidRequests(): iterable + { + foreach (SpssCompilerTest::invalidRequestProvider() as $name => [$payload]) { + if ($name !== 'wrong contract') { + $payload['contract'] = SpssFrontendRequest::CONTRACT_V03; + } + yield $name => [$payload]; + } + $valid = self::payload('EXECUTE.'); + foreach (['contract' => 3, 'source_text' => [], 'input_alias' => false, 'input_schema' => 'schema'] as $key => $value) { + yield 'wrong type ' . $key => [[...$valid, $key => $value]]; + } + foreach (['value_labels' => null, 'width' => '8', 'decimals' => false, 'variable_label' => 1, 'format_family' => [], 'measurement_level' => true] as $key => $value) { + $payload = $valid; + $payload['input_schema']['variables'][0][$key] = $value; + yield 'wrong variable type ' . $key => [$payload]; + } + } + + #[DataProvider('lowerings')] + public function testOfficialLoweringMatchesExplicitInheritedPlan(string $source, string $expanded): void + { + $compiler = new SpssCompiler(); + $official = $compiler->compile(SpssFrontendRequest::fromArray(self::payload($source))); + $legacy = $compiler->compile(SpssFrontendRequest::fromArray([ + ...self::payload($expanded), 'contract' => SpssFrontendRequest::CONTRACT, + ])); + self::assertSame($legacy->plan->canonicalArray(), $official->plan->canonicalArray()); + } + + /** @return iterable */ + public static function lowerings(): iterable + { + foreach (['=' => 'a < 1 OR a > 1', '<' => 'a >= 1', '<=' => 'a > 1', '>' => 'a <= 1', '>=' => 'a < 1', 'NE' => 'a >= 1 AND a <= 1', '<>' => 'a >= 1 AND a <= 1', '~=' => 'a >= 1 AND a <= 1'] as $operator => $predicate) { + yield 'NOT ' . $operator => ["IF (NOT a $operator 1) target = 1.", "IF ($predicate) target = 1."]; + } + yield 'comparison then NOT then AND then OR' => [ + 'IF (NOT a = 1 AND b = 2 OR c = 3) target = 1.', + 'IF (((a < 1 OR a > 1) AND b = 2) OR c = 3) target = 1.', + ]; + yield 'parentheses override NOT scope' => [ + 'IF (NOT (a = 1 AND b = 2 OR c = 3)) target = 1.', + 'IF ((a < 1 OR a > 1 OR b < 2 OR b > 2) AND (c < 3 OR c > 3)) target = 1.', + ]; + yield 'aliases flatten with parenthesized siblings' => [ + 'IF (a NE 1 OR (b <> 2 OR c ~= 3)) target = 1.', + 'IF (a < 1 OR a > 1 OR b < 2 OR b > 2 OR c < 3 OR c > 3) target = 1.', + ]; + yield 'double NOT remains comparison' => ['IF (NOT NOT a = 1) target = 1.', 'IF (a = 1) target = 1.']; + yield 'grouped recodes and labels use ordered evolving schema' => [ + "RECODE a TO c (1 = 0) INTO x y z / x TO z (0 = 2).\nVARIABLE LABELS a TO c 'Group' / target 'Target'.\nFORMATS a b (F8.2) / c target (F9.3).\nVARIABLE LEVEL a TO c (ORDINAL) / target (SCALE).", + "RECODE a b c (1 = 0) INTO x y z. RECODE x y z (0 = 2).\nVARIABLE LABELS a 'Group' b 'Group' c 'Group' target 'Target'.\nFORMATS a (F8.2) b (F8.2) c (F9.3) target (F9.3).\nVARIABLE LEVEL a b c (ORDINAL) / target (SCALE).", + ]; + yield 'chained dictionary ranges retain order without duplicate joints' => [ + "VALUE LABELS a TO b TO c 1 'One'.", "VALUE LABELS a b c 1 'One'.", + ]; + yield 'ADD typed zero updates ordinal and groups see replacement' => [ + "VALUE LABELS a 5 'Five' 0 'Old'. ADD VALUE LABELS a -0 'Zero' 2 'Two' / a 5 'Cinq'.", + "VALUE LABELS a 5 'Five' 0 'Old'. VALUE LABELS a 5 'Five' 0 'Zero' 2 'Two'. VALUE LABELS a 5 'Cinq' 0 'Zero' 2 'Two'.", + ]; + } + + public function testAddStringLabelsUsesExactContentsAndKeepsExistingOrdinals(): void + { + $payload = self::payload("ADD VALUE LABELS Status '1' 'One' '' 'Empty'."); + $payload['input_schema']['variables'] = [[ + 'name' => 'Status', 'storage_kind' => 'string', + 'value_labels' => [ + ['value' => ['type' => 'string', 'value' => '01'], 'label' => 'Leading zero'], + ['value' => ['type' => 'string', 'value' => '1'], 'label' => 'Old'], + ], + ]]; + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($payload)); + self::assertSame([ + ['value' => ['type' => 'string', 'value' => '01'], 'label' => 'Leading zero'], + ['value' => ['type' => 'string', 'value' => '1'], 'label' => 'One'], + ['value' => ['type' => 'string', 'value' => ''], 'label' => 'Empty'], + ], $result->plan->canonicalArray()['operations'][0]['labels']); + } + + public function testCommentsRetainNormalizedHashQuotedMarkersAndExactDiagnosticCoordinates(): void + { + $source = " /* header */ * ignored ' quote.\r\nCOMMENT ignored \" quote.\rVARIABLE LABELS a '/* literal */ * COMMENT'."; + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload($source))); + self::assertSame(hash('sha256', str_replace(["\r\n", "\r"], "\n", $source)), $result->sourceHash); + self::assertSame('/* literal */ * COMMENT', $result->plan->canonicalArray()['operations'][0]['label']); + try { + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload("/* comment */\r\nCOMPUTE target = absent."))); + self::fail('Unknown variable compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame('unknown_variable', $failure->diagnosticCode()); + $span = $failure->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame([32, 38, 2, 18], [$span->startOffset, $span->endOffset, $span->startLine, $span->startColumn]); + } + } + + #[DataProvider('rejectedSyntax')] + public function testOfficialSyntaxFailsAtomically(string $source, string $code): void + { + try { + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload('EXECUTE. ' . $source))); + self::fail('Unsupported syntax compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame($code, $failure->diagnosticCode()); + self::assertNotNull($failure->diagnostics[0]->span); + } + } + + /** @return iterable */ + public static function rejectedSyntax(): iterable + { + foreach (['/* nested /* inner */ */', '/* unclosed', '* no period', 'COMMENT no period', 'FORMATS a (F8.2) / .', "VARIABLE LABELS a 'A' / .", 'RECODE a (1 = 0) / .', 'RECODE a (LOWEST THRU 1e4000 = 0).', 'RECODE a (1 THRU LOWEST = 0).'] as $source) { + yield $source => [$source, 'spss_syntax_error']; + } + yield 'reverse finite range' => ['RECODE a (2 THRU 1 = 0).', 'invalid_variable_range']; + yield 'STRING' => ['STRING new (A8).', 'unsupported_spss_command']; + yield 'DELETE' => ['DELETE VARIABLES a.', 'unsupported_spss_command']; + } + + public function testExplicitSelectionDoesNotLeakIntoDefaultParserOrCompiler(): void + { + $compiler = new SpssCompiler(); + foreach (['* comment. EXECUTE.', 'COMMENT text. EXECUTE.', '/* comment */ EXECUTE.', 'IF (NOT a = 1) target = 1.', 'IF (a NE 1) target = 1.', 'IF (a <> 1) target = 1.', 'IF (a ~= 1) target = 1.', 'RECODE a (LOWEST THRU HIGHEST = 0).', "ADD VALUE LABELS a 1 'One'.", 'FORMATS a TO c (F8.2).', "VARIABLE LABELS a b 'Both'.", 'RECODE a (1 = 0) / b (1 = 0).'] as $source) { + $compiler->compile(SpssFrontendRequest::fromArray(self::payload($source))); + try { + $compiler->compile(SpssFrontendRequest::fromArray([...self::payload($source), 'contract' => SpssFrontendRequest::CONTRACT])); + self::fail('Default 0.2 accepted: ' . $source); + } catch (TransformationFailure $failure) { + self::assertNotNull($failure->diagnostics[0]->span); + } + } + self::assertCount(1, $compiler->bind('parent', SpssFrontendRequest::fromArray(self::payload('EXECUTE.'))->inputSchema, $compiler->parse('EXECUTE.'))->statements); + } + + /** @return array */ + private static function payload(string $source): array + { + return [ + 'contract' => SpssFrontendRequest::CONTRACT_V03, + 'input_alias' => 'parent', + 'input_schema' => ['variables' => array_map(static fn(string $name): array => ['name' => $name, 'storage_kind' => 'numeric'], ['a', 'b', 'c', 'target'])], + 'source_text' => $source, + ]; + } +} diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php index 135df21..7c4edf6 100644 --- a/tests/Integration/OfficialInPlaceTransformation01Test.php +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -152,6 +152,69 @@ public function testSqliteStringValueLabelsAreReplacedExactlyWithoutChangingOthe ); } + public function testOptInFrontend03AppliesNativelyWithProvenanceAndNoExtraDataArtifacts(): void + { + $pdo = $this->fixture(); + $tablesBefore = $this->tables($pdo); + $datasetBefore = $this->rows($pdo, 'SELECT * FROM dataset'); + $source = "* Frontend03 native SQLite.\r\n" + . "COMPUTE Flag = 9.\n" + . "IF (NOT q1 = 1 AND q1 < 3 OR q1 = 6) Flag = 1.\n" + . "IF (q1 NE q1 OR q1 <> q1 OR q1 ~= q1) Flag = 2.\n" + . "IF (q1 = 3) q1 = -1.7976931348623157e308. IF (q1 = 4) q1 = 1.7976931348623157e308.\n" + . "RECODE q1 (LOWEST THRU HIGHEST = 0).\n" + . "VARIABLE LABELS q1 TO Flag 'Grouped'.\n" + . "VALUE LABELS Flag 9 'Retained'. ADD VALUE LABELS Flag 1 'Selected' 9 'Unchanged'.\n" + . "FORMATS q1 TO Flag (F8.2). VARIABLE LEVEL q1 TO Flag (NOMINAL). EXECUTE."; + $request = SpssFrontendRequest::fromArray([ + 'contract' => SpssFrontendRequest::CONTRACT_V03, + 'input_alias' => 'parent', + 'input_schema' => ['variables' => [['name' => 'q1', 'storage_kind' => 'numeric']]], + 'source_text' => $source, + ]); + $compiled = (new SpssCompiler())->compile($request); + $result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute(new InPlaceApplyRequest( + $compiled->plan, + 'parent', + self::DATASET_ID, + $compiled->sourceHash, + 'frontend03-test', + )); + + self::assertSame(self::DATASET_ID, $result->datasetId()); + self::assertSame($datasetBefore, $this->rows($pdo, 'SELECT * FROM dataset')); + self::assertSame($tablesBefore, $this->tables($pdo)); + self::assertSame([], $this->rows($pdo, "SELECT name FROM sqlite_temp_master WHERE type = 'table'")); + self::assertSame([1, 2, 3, 4, 5, 6, 7], array_map('intval', $this->column($pdo, 'SELECT __case_ordinal FROM data_plan01 ORDER BY __case_ordinal'))); + self::assertSame([9.0, 1.0, 9.0, 9.0, 9.0, 1.0, 9.0], array_map('floatval', $this->column($pdo, 'SELECT Flag FROM data_plan01 ORDER BY __case_ordinal'))); + self::assertSame([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, null], array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column($pdo, 'SELECT q1 FROM data_plan01 ORDER BY __case_ordinal'), + )); + self::assertSame(['q1', 'Flag'], $this->column($pdo, 'SELECT source_name FROM variable ORDER BY source_ordinal')); + foreach ($this->rows($pdo, 'SELECT variable_label, print_format_family, print_format_width, print_format_decimals, measurement_level FROM variable') as $variable) { + self::assertSame(['Grouped', 'F', 8, 2, 'nominal'], array_values($variable)); + } + self::assertSame(['Unchanged', 'Selected'], $this->column($pdo, 'SELECT label FROM value_label ORDER BY ordinal')); + $audit = $this->rows($pdo, 'SELECT * FROM transformation_apply'); + self::assertCount(1, $audit); + self::assertSame('openstatspec-in-place-transformation-v0.2', $audit[0]['contract_id']); + self::assertSame('sqlite', $audit[0]['database_profile']); + self::assertSame(self::DATASET_ID, $audit[0]['dataset_id']); + self::assertSame('data_plan01', $audit[0]['physical_table_name']); + self::assertSame(hash('sha256', str_replace("\r\n", "\n", $source)), $audit[0]['source_hash']); + self::assertSame((new PlanCodec())->hash($compiled->plan), $audit[0]['plan_hash']); + self::assertSame((new PlanCodec())->canonicalJson($compiled->plan), $audit[0]['canonical_plan_json']); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM operation')); + self::assertSame(0, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM fidelity_event')); + self::assertSame('frontend03-test', $audit[0]['actor']); + self::assertSame('succeeded', $audit[0]['status']); + self::assertSame(count($compiled->plan->operations), (int) $audit[0]['operation_count']); + self::assertNull($audit[0]['dolt_head_before']); + self::assertNull($audit[0]['dolt_head_after']); + self::assertFalse($pdo->inTransaction()); + } + private function fixture(): PDO { if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) {