From 1a4dbdbdcaaa84631217eb3112d80a10a1e0f4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 10 Sep 2026 15:34:06 +0300 Subject: [PATCH] Namespace Python schema-change contracts and preserve legacy plans --- CHANGELOG.md | 15 +++++++ docs/transformations.md | 34 ++++++++++++++ src/openstatspec/frontends/spss/compiler.py | 3 +- src/openstatspec/transform/plan.py | 20 ++++++--- tests/test_inplace_transform.py | 46 ++++++++++++++++++- tests/test_transform_frontend.py | 49 ++++++++++++++++----- 6 files changed, 146 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eab417..9f8492a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this reference implementation are documented here. +## Unreleased + +### Fixed + +- Namespace new Python schema-change plans and SPSS output as + `openstatspec-python-schema-change-plan-v0.1` and + `openstatspec-python-schema-change-spss-v0.1`, retaining exported constant names. + The former `openstatspec-transformation-plan-v0.3` remains accepted as a + legacy Python extension with unchanged semantics, canonical JSON, and hashes; + stored audits are not migrated. New compilation changes schema-plan hashes. +- Clarify that specification `v0.5.0` has no official Plan 0.3 and that its + syntax-only Frontend 0.3 over Plan 0.1/0.2 is not implemented here. Legacy + acceptance and Python extension tests do not claim official conformance. + See [compatibility and migration](docs/transformations.md#contract-ownership-and-legacy-compatibility). + ## 0.8.1 - 2026-09-10 ### Fixed diff --git a/docs/transformations.md b/docs/transformations.md index ea870bd..448c5be 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -30,6 +30,40 @@ This is a trust boundary. JSON plans must pass `transformation_plan_from_dict()` and live-schema validation before mutation; they are never treated as arbitrary SQL. +## Contract ownership and legacy compatibility + +Published OpenStatSpec `v0.5.0` defines Transformation Plan 0.1/0.2, not +Plan 0.3. Its optional SPSS Frontend 0.3 is a syntax-only expansion emitting +Plan 0.1/0.2; this adapter does **not** implement that frontend yet. + +Explicit `create_variable` / `delete_variable` operations (including SPSS +`STRING` / `DELETE VARIABLES`) belong to a **Python extension**, not official +OpenStatSpec conformance. New schema-changing compilations emit: + +- Plan: `openstatspec-python-schema-change-plan-v0.1` +- Frontend: `openstatspec-python-schema-change-spss-v0.1` + +The exported names `TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT` and +`SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT` remain unchanged; their values now use +these Python-owned identifiers. Plans without these explicit schema operations +retain their existing Plan 0.1/0.2 selection and frontend identifier. Official +Plan 0.1/0.2 still reject explicit create/delete operations. + +The loader and executor continue to accept the old Python plan identifier +`openstatspec-transformation-plan-v0.3` with its original operation semantics. +This is legacy compatibility, **not** recognition of an official Plan 0.3. +Loading, canonical serialization, hashing, and apply preserve the supplied +identifier; existing canonical JSON, hashes, and stored audits are not rewritten. +Historical `openstatspec-spss-syntax-frontend-v0.3` audit values described this +Python extension and are not evidence of official Frontend 0.3 conformance. + +No database migration is needed. Consumers should accept the new extension IDs +before using new compiler output. Keep stored legacy plans and audits intact; +recompiling schema-changing syntax now produces a new plan identity/hash even +when its operations are identical. An intentional change of a saved plan's +contract likewise creates a new artifact with a new hash, not an audit migration. +Older adapter versions cannot load the new IDs. + ## Install the audit schema Install the compact audit relation once before the first apply: diff --git a/src/openstatspec/frontends/spss/compiler.py b/src/openstatspec/frontends/spss/compiler.py index ecd05f2..a2fb8e2 100644 --- a/src/openstatspec/frontends/spss/compiler.py +++ b/src/openstatspec/frontends/spss/compiler.py @@ -18,7 +18,8 @@ SPSS_FRONTEND_CONTRACT = "openstatspec-spss-syntax-frontend-v0.2" -SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT = "openstatspec-spss-syntax-frontend-v0.3" +# Python-owned schema extension, not the official syntax-only Frontend 0.3. +SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT = "openstatspec-python-schema-change-spss-v0.1" @dataclass(frozen=True) diff --git a/src/openstatspec/transform/plan.py b/src/openstatspec/transform/plan.py index 74bef7c..cd31701 100644 --- a/src/openstatspec/transform/plan.py +++ b/src/openstatspec/transform/plan.py @@ -16,8 +16,15 @@ TRANSFORMATION_PLAN_V1_CONTRACT = "openstatspec-transformation-plan-v0.1" TRANSFORMATION_PLAN_CONTRACT = "openstatspec-transformation-plan-v0.2" -TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT = "openstatspec-transformation-plan-v0.3" -_TRANSFORMATION_PLAN_CONTRACTS = {TRANSFORMATION_PLAN_V1_CONTRACT, TRANSFORMATION_PLAN_CONTRACT, TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT} +TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT = "openstatspec-python-schema-change-plan-v0.1" +# Python extension compatibility only: never normalize legacy IDs or audit hashes. +_SCHEMA_CHANGE_CONTRACTS = { + TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT, + "openstatspec-transformation-plan-v0.3", +} +_TRANSFORMATION_PLAN_CONTRACTS = { + TRANSFORMATION_PLAN_V1_CONTRACT, TRANSFORMATION_PLAN_CONTRACT, +} | _SCHEMA_CHANGE_CONTRACTS _BINARY64 = re.compile(r"[0-9a-f]{16}") @@ -589,7 +596,7 @@ def __post_init__(self) -> None: if not isinstance(self.contract, str) or self.contract not in _TRANSFORMATION_PLAN_CONTRACTS: _invalid("Plan contract is not a supported transformation-plan contract.") if ( - self.contract != TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT + self.contract not in _SCHEMA_CHANGE_CONTRACTS and any( isinstance(operation, ( CreateVariableOperation, DeleteVariableOperation, @@ -598,8 +605,9 @@ def __post_init__(self) -> None: ) ): _invalid( - "Create/delete schema operations require " - "openstatspec-transformation-plan-v0.3." + "Create/delete schema operations require the Python extension " + f"{TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT} " + "(legacy Python v0.3 plans are also accepted, not official conformance)." ) if self.contract == TRANSFORMATION_PLAN_V1_CONTRACT and any( isinstance(operation, ( @@ -723,7 +731,7 @@ def _match(raw: Any) -> RecodeMatch: _invalid("Unknown recode match kind.") def transformation_plan_from_dict(raw: Mapping[str, Any]) -> TransformationPlan: - """Strictly validate canonical v0.1, v0.2, or schema-change v0.3 plans.""" + """Validate Plan 0.1/0.2 or Python schema-change plans, including legacy IDs.""" if not isinstance(raw, Mapping): _invalid("Transformation plan must be an object.") _exact(raw, {"contract", "input_alias", "operations"}, "Transformation plan") diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 48b61bd..baab261 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -904,7 +904,7 @@ def test_public_apply_supports_non_dolt_without_building_undo(catalog) -> None: )) == (1, 1), "public SPSS apply must encode/hash the canonical plan once" -def test_schema_commands_record_the_v03_frontend_contract(catalog) -> None: +def test_schema_commands_record_python_extension_frontend_contract(catalog) -> None: url, path, dataset_id, _table_name = catalog openstatspec.apply_spss_in_place( @@ -919,10 +919,52 @@ def test_schema_commands_record_the_v03_frontend_contract(catalog) -> None: ).fetchone() assert audit == ( "spss_syntax", - "openstatspec-spss-syntax-frontend-v0.3", + "openstatspec-python-schema-change-spss-v0.1", ) +@pytest.mark.parametrize("contract", [ + "openstatspec-transformation-plan-v0.3", # Legacy acceptance, not conformance. + "openstatspec-python-schema-change-plan-v0.1", +]) +def test_schema_plan_apply_preserves_identity_and_audit(catalog, contract) -> None: + url, path, dataset_id, table_name = catalog + plan = openstatspec.TransformationPlan( + (openstatspec.CreateVariableOperation("note", "string", 4),), + contract=contract, + ) + with sqlite3.connect(path) as connection: + tables = connection.execute("SELECT name FROM sqlite_master ORDER BY name").fetchall() + identity = connection.execute("SELECT * FROM dataset").fetchall() + result = openstatspec.apply_transformation_plan_in_place( + database_url=url, dataset_id=dataset_id, plan=plan.as_dict(), actor="test-agent", + ) + assert (result["dataset_id"], result["physical_table_name"]) == (dataset_id, table_name) + with sqlite3.connect(path) as connection: + assert connection.execute( + f'SELECT score, note FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == [(1.0, ""), (2.0, ""), (3.0, "")] + audit = connection.execute("SELECT * FROM transformation_apply").fetchone() + assert connection.execute( + "SELECT canonical_plan_json, plan_hash, source_hash, frontend_contract " + "FROM transformation_apply" + ).fetchone() == (plan.canonical_json(), plan.sha256(), plan.sha256(), None) + # New compilation must not migrate an existing (including legacy) audit row. + openstatspec.apply_spss_in_place( + database_url=url, dataset_id=dataset_id, + source_text="DELETE VARIABLES note.", actor="test-agent", + ) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT name FROM sqlite_master ORDER BY name").fetchall() == tables + assert connection.execute("SELECT * FROM dataset").fetchall() == identity + assert connection.execute( + "SELECT * FROM transformation_apply WHERE apply_id = ?", (result["apply_id"],), + ).fetchone() == audit + assert [row[1] for row in connection.execute( + f'PRAGMA table_info("{table_name}")' + )] == ["__case_ordinal", "score"] + + @pytest.mark.parametrize("as_mapping", [False, True]) def test_public_generic_plan_apply_accepts_object_and_mapping( catalog, as_mapping, diff --git a/tests/test_transform_frontend.py b/tests/test_transform_frontend.py index ed01ee2..2edc70f 100644 --- a/tests/test_transform_frontend.py +++ b/tests/test_transform_frontend.py @@ -375,14 +375,16 @@ def test_source_normalization_hash_and_positions_are_stable() -> None: assert compilation.frontend_contract == "openstatspec-spss-syntax-frontend-v0.2" -def test_schema_commands_use_the_v03_frontend_contract() -> None: +def test_schema_commands_use_python_extension_contracts() -> None: compilation = compile_spss_syntax( "STRING note (A4).", _schema(VariableDefinition("q1", "numeric")), ) - assert compilation.plan.contract == "openstatspec-transformation-plan-v0.3" - assert compilation.frontend_contract == "openstatspec-spss-syntax-frontend-v0.3" + assert compilation.plan.contract == "openstatspec-python-schema-change-plan-v0.1" + assert compilation.frontend_contract == "openstatspec-python-schema-change-spss-v0.1" + assert transform_module.TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT == compilation.plan.contract + assert transform_module.SPSS_FRONTEND_SCHEMA_CHANGE_CONTRACT == compilation.frontend_contract def test_string_comparison_fails_closed_until_exact_collation_is_supported() -> None: error = _error( @@ -584,26 +586,49 @@ def test_v02_plan_and_schema_reject_decimal_format_that_cannot_fit() -> None: ) -def test_schema_operations_require_v03_contract() -> None: - operation = CreateVariableOperation("note", "string", 8) +@pytest.mark.parametrize("contract", [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2", +]) +@pytest.mark.parametrize("operation", [ + CreateVariableOperation("note", "string", 8), DeleteVariableOperation("note"), +]) +def test_official_plan_contracts_reject_schema_operations(contract, operation) -> None: with pytest.raises(TransformationFrontendError) as caught: - TransformationPlan((operation,)) + transformation_plan_from_dict({ + "contract": contract, "input_alias": "parent", + "operations": [operation.as_dict()], + }) assert caught.value.code == "invalid_transformation_plan" - plan = TransformationPlan( - (operation,), contract="openstatspec-transformation-plan-v0.3", + +def test_legacy_schema_plan_roundtrip_preserves_exact_hash() -> None: + serialized = ( + '{"contract":"openstatspec-transformation-plan-v0.3","input_alias":"parent",' + '"operations":[{"declared_string_width":4,"op":"create_variable",' + '"storage_kind":"string","variable":"note"}]}' ) - assert plan.contract == "openstatspec-transformation-plan-v0.3" + raw = json.loads(serialized) + legacy = transformation_plan_from_dict(raw) + assert canonical_plan_json(raw) == legacy.canonical_json() == serialized + assert canonical_plan_hash(raw) == legacy.sha256() == ( + "3c1933c6df1ea664d506d3b7879af5141f912181884129d2861bf19e44f4e131" + ) + new = transformation_plan_from_dict({ + **raw, "contract": "openstatspec-python-schema-change-plan-v0.1", + }) + assert new.operations == legacy.operations + assert new.sha256() != legacy.sha256() -def test_spss_schema_commands_emit_v03_contract() -> None: +def test_spss_schema_commands_emit_python_extension_contract() -> None: schema = _schema(VariableDefinition("q1", "numeric")) assert bind_spss_syntax( parse_spss_syntax("STRING note (A8)."), schema, - ).plan.contract == "openstatspec-transformation-plan-v0.3" + ).plan.contract == "openstatspec-python-schema-change-plan-v0.1" assert bind_spss_syntax( parse_spss_syntax("COMPUTE other = q1. DELETE VARIABLES q1."), schema, - ).plan.contract == "openstatspec-transformation-plan-v0.3" + ).plan.contract == "openstatspec-python-schema-change-plan-v0.1" def test_custom_nonempty_input_alias_is_canonical() -> None: