From 3222e633ee9474be06c44e91f98c51993a62506c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 8 Sep 2026 16:40:26 +0300 Subject: [PATCH 1/3] test: bound export metadata reads independently of dictionary size --- .../MySqlFamilySpssRoundTripTestCase.php | 18 +- .../PostgreSqlSpssRoundTripTest.php | 18 +- tests/Spss/SpssAdapterTest.php | 215 ++++++++++++++++++ tests/Sql/MySqlWideTableExporterTest.php | 10 +- tests/Sql/PostgreSqlWideTableExporterTest.php | 6 +- tests/Support/ExportCountingPdo.php | 70 ++++++ 6 files changed, 329 insertions(+), 8 deletions(-) create mode 100644 tests/Support/ExportCountingPdo.php diff --git a/tests/Integration/MySqlFamilySpssRoundTripTestCase.php b/tests/Integration/MySqlFamilySpssRoundTripTestCase.php index 0cf8378..00f468a 100644 --- a/tests/Integration/MySqlFamilySpssRoundTripTestCase.php +++ b/tests/Integration/MySqlFamilySpssRoundTripTestCase.php @@ -5,6 +5,9 @@ namespace OpenStatSpec\Tests\Integration; use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Sql\CanonicalWideTableExporter; +use OpenStatSpec\Sql\MySqlWideTableExporter; +use OpenStatSpec\Tests\Support\ExportCountingPdo; use OpenStatSpec\Sql\MySqlProfile; use OpenStatSpec\Sql\MySqlWideTableImporter; use OpenStatSpec\Spss\PhpSpssEngine; @@ -47,6 +50,7 @@ abstract class MySqlFamilySpssRoundTripTestCase extends TestCase public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void { $pdo = $this->mysql(); + self::assertInstanceOf(ExportCountingPdo::class, $pdo); $engine = new PhpSpssEngine(); foreach (['sav' => ['$FL2', 1], 'zsav' => ['$FL3', 2]] as $format => [$header, $compression]) { @@ -112,6 +116,16 @@ public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void self::assertSame($format, $technical[0]['source_format']); self::assertSame($compression, (int) $technical[0]['compression']); + $executionCounts = []; + foreach ([CanonicalWideTableExporter::class, MySqlWideTableExporter::class] as $exporterClass) { + $pdo->executions = []; + $direct = (new $exporterClass($pdo))->export($datasetName, $format); + $executionCounts[] = count($pdo->captured()); + $pdo->executions = null; + self::assertSame([], $direct['diagnostics']); + self::assertSame($fixture->rows(), $direct['dataset']->rows()); + self::assertEquals($fixture->metadata, $direct['dataset']->metadata); + } $result = $adapter->export($datasetName, $targetPath); self::assertSame([], $result->diagnostics); self::assertSame(2, $result->caseCount); @@ -142,6 +156,8 @@ public function testRealEngineRoundTripsSavAndZsavThroughMySqlFamily(): void self::assertCount(1, $roundTrip->metadata->multipleResponseSets()); self::assertSame(MultipleResponseSetType::DICHOTOMY, $roundTrip->metadata->multipleResponseSets()[0]->type); self::assertSame(['Reason'], $roundTrip->metadata->multipleResponseSets()[0]->variableNames()); + self::assertLessThanOrEqual(14, $executionCounts[0], 'Canonical export SQL executions'); + self::assertLessThanOrEqual(18, $executionCounts[1], 'Legacy export SQL executions'); } finally { $this->cleanup($pdo, $datasetName, $tableName); @unlink($sourcePath); @@ -262,7 +278,7 @@ protected function mysql(): PDO $user = getenv($prefix . '_USER'); $password = getenv($prefix . '_PASSWORD'); - return new PDO( + return new ExportCountingPdo( $dsn, is_string($user) ? $user : null, is_string($password) ? $password : null, diff --git a/tests/Integration/PostgreSqlSpssRoundTripTest.php b/tests/Integration/PostgreSqlSpssRoundTripTest.php index ff2af59..a092c61 100644 --- a/tests/Integration/PostgreSqlSpssRoundTripTest.php +++ b/tests/Integration/PostgreSqlSpssRoundTripTest.php @@ -7,6 +7,9 @@ use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; use OpenStatSpec\Sql\CatalogOwnership; +use OpenStatSpec\Sql\CanonicalWideTableExporter; +use OpenStatSpec\Sql\PostgreSqlWideTableExporter; +use OpenStatSpec\Tests\Support\ExportCountingPdo; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Sql\PostgreSqlWideTableImporter; @@ -60,6 +63,7 @@ final class PostgreSqlSpssRoundTripTest extends TestCase public function testRealEngineRoundTripsSavAndZsavThroughPostgreSql(): void { $pdo = $this->postgres(); + self::assertInstanceOf(ExportCountingPdo::class, $pdo); $engine = new PhpSpssEngine(); foreach (['sav' => ['$FL2', 1], 'zsav' => ['$FL3', 2]] as $format => [$header, $compression]) { @@ -96,6 +100,16 @@ public function testRealEngineRoundTripsSavAndZsavThroughPostgreSql(): void self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM multiple_response_sets WHERE dataset_name = ?', [$datasetName])); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT variable_ordinal FROM dataset_weight_variables WHERE dataset_name = ?', [$datasetName])); + $executionCounts = []; + foreach ([CanonicalWideTableExporter::class, PostgreSqlWideTableExporter::class] as $exporterClass) { + $pdo->executions = []; + $direct = (new $exporterClass($pdo))->export($datasetName, $format); + $executionCounts[] = count($pdo->captured()); + $pdo->executions = null; + self::assertSame([], $direct['diagnostics']); + self::assertSame($fixture->rows(), $direct['dataset']->rows()); + self::assertEquals($fixture->metadata, $direct['dataset']->metadata); + } $result = $adapter->export($datasetName, $targetPath); self::assertSame([], $result->diagnostics); self::assertSame(2, $result->caseCount); @@ -126,6 +140,8 @@ public function testRealEngineRoundTripsSavAndZsavThroughPostgreSql(): void self::assertCount(1, $roundTrip->metadata->multipleResponseSets()); self::assertSame(MultipleResponseSetType::DICHOTOMY, $roundTrip->metadata->multipleResponseSets()[0]->type); self::assertSame(['Reason'], $roundTrip->metadata->multipleResponseSets()[0]->variableNames()); + self::assertLessThanOrEqual(14, $executionCounts[0], 'Canonical export SQL executions'); + self::assertLessThanOrEqual(18, $executionCounts[1], 'Legacy export SQL executions'); } finally { $this->cleanup($pdo, $datasetName, $tableName); @unlink($sourcePath); @@ -365,7 +381,7 @@ private function postgres(): PDO $user = getenv('OPENSTATSPEC_PG_USER'); $password = getenv('OPENSTATSPEC_PG_PASSWORD'); - return new PDO( + return new ExportCountingPdo( $dsn, is_string($user) ? $user : null, is_string($password) ? $password : null, diff --git a/tests/Spss/SpssAdapterTest.php b/tests/Spss/SpssAdapterTest.php index 4c6ef3f..1709664 100644 --- a/tests/Spss/SpssAdapterTest.php +++ b/tests/Spss/SpssAdapterTest.php @@ -11,6 +11,11 @@ use OpenStatSpec\Spss\SpssAdapter; use OpenStatSpec\Spss\SpssSourceNormalizer; use OpenStatSpec\Sql\CatalogOwnership; +use OpenStatSpec\Sql\CanonicalWideTableExporter; +use OpenStatSpec\Sql\SqliteWideTableExporter; +use OpenStatSpec\Sql\MySqlWideTableExporter; +use OpenStatSpec\Sql\PostgreSqlWideTableExporter; +use OpenStatSpec\Tests\Support\ExportCountingPdo; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; @@ -1369,6 +1374,216 @@ public function testImportRejectsCallerOwnedTransactionWithoutMutation(bool $dir self::assertSame(DiagnosticCode::UnsupportedOperation, $failure->diagnosticCode); } + /** @return iterable */ + public static function groupedExporters(): iterable + { + foreach ([CanonicalWideTableExporter::class, SqliteWideTableExporter::class, MySqlWideTableExporter::class, PostgreSqlWideTableExporter::class, SpssAdapter::class] as $class) { + yield substr($class, strrpos($class, '\\') + 1) => [$class]; + } + } + + /** @param class-string $class */ + #[DataProvider('groupedExporters')] + public function testGroupedExportExecutionCountDoesNotGrow(string $class): void + { + $counts = []; + $overhead = 0; + foreach ([[2, 0, false], [20, 0, false], [20, 10, false], [2, 0, true]] as [$variables, $sets, $empty]) { + $pdo = new ExportCountingPdo('sqlite::memory:'); + $source = $this->groupedExportFixture($variables, $sets, $empty); + $engine = new FakeSpssEngine($source); + $adapter = new SpssAdapter($pdo, $engine); + $adapter->import('fixture.sav', 'A'); + $exporter = $class === SpssAdapter::class ? $adapter : new $class($pdo); + $pdo->exec('PRAGMA query_only = ON'); + if ($class === SpssAdapter::class) { + $pdo->executions = []; + CatalogOwnership::assertReadyForUseReadOnly($pdo); + $overhead = count($pdo->captured()); + } + $pdo->executions = []; + $pdo->prepares = 0; + $export = $this->publicExport($exporter, $engine, 'A'); + $counts[] = count($pdo->captured()) - $overhead; + self::assertSame($source->rowCount(), $export['caseCount']); + self::assertSame([], $export['diagnostics']); + self::assertSame($source->rows(), $export['dataset']->rows()); + foreach ($pdo->captured() as $execution) { + self::assertMatchesRegularExpression('/^SELECT\b/i', $execution['sql']); + } + } + $limit = in_array($class, [CanonicalWideTableExporter::class, SpssAdapter::class], true) ? 14 : 18; + self::assertLessThanOrEqual($limit, max($counts), 'Executed SQL for (V,S,R)=(2,0,0),(20,0,0),(20,10,10),empty: ' . json_encode($counts)); + self::assertSame($counts[0], $counts[1], 'Adding variables must not add executions.'); + } + + /** @param class-string $class */ + #[DataProvider('groupedExporters')] + public function testGroupedExportPreservesOrderedMetadataIsolationAndFreshness(string $class): void + { + $pdo = new ExportCountingPdo('sqlite::memory:'); + $source = $this->groupedExportFixture(4, 3); + $engine = new FakeSpssEngine($source); + $adapter = new SpssAdapter($pdo, $engine); + $adapter->import('fixture.sav', 'A'); + (new SpssAdapter($pdo, new FakeSpssEngine($this->groupedExportFixture(4, 3, changed: true))))->import('fixture.sav', 'B'); + (new SpssAdapter($pdo, new FakeSpssEngine($this->groupedExportFixture(2, 0, true))))->import('fixture.sav', 'C'); + // Legacy format-zero/absent rules deliberately ignore their value rows. + $pdo->exec("DELETE FROM missing_rules WHERE dataset_name = 'C' AND variable_ordinal = 2"); + $pdo->exec("INSERT INTO missing_rule_values (dataset_name, variable_ordinal, ordinal, value_kind) VALUES ('C', 1, 1, 'text'), ('C', 2, 1, 'text')"); + // Valid shared label set: both owners are in A, never a foreign-label policy test. + $pdo->exec("UPDATE variable_value_label_set SET value_label_set_id = (SELECT link.value_label_set_id FROM variable_value_label_set link JOIN variable v ON v.variable_id = link.variable_id JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 1) WHERE variable_id = (SELECT v.variable_id FROM variable v JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 3)"); + $exporter = $class === SpssAdapter::class ? $adapter : new $class($pdo); + foreach (['A', 'B', 'A', 'C'] as $name) { + $expected = match ($name) { + 'A' => $source, + 'B' => $this->groupedExportFixture(4, 3, changed: true), + default => $this->groupedExportFixture(2, 0, true), + }; + $before = self::rows($pdo, 'SELECT total_changes() AS changes'); + $pdo->exec('PRAGMA query_only = ON'); + $export = $this->publicExport($exporter, $engine, $name, 'zsav'); + $this->assertGroupedExport($expected, $export, 'zsav'); + self::assertSame($before, self::rows($pdo, 'SELECT total_changes() AS changes')); + $pdo->exec('PRAGMA query_only = OFF'); + } + $pdo->beginTransaction(); + // Mutate each grouped child family, scoped to A; same object must re-read them. + $pdo->exec("UPDATE value_label SET label = 'Updated' WHERE ordinal = 1 AND value_label_set_id IN (SELECT link.value_label_set_id FROM variable_value_label_set link JOIN variable v ON v.variable_id = link.variable_id JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 1)"); + $pdo->exec("UPDATE missing_rule SET numeric_upper = 4 WHERE variable_id IN (SELECT v.variable_id FROM variable v JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 1)"); + $pdo->exec("INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind, code_kind, numeric_value) SELECT 'new-missing', v.variable_id, 2, 'discrete', 'numeric', 99 FROM variable v JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 1"); + $pdo->exec("UPDATE missing_rules SET missing_format = -3 WHERE dataset_name = 'A' AND variable_ordinal = 1"); + $pdo->exec("INSERT INTO missing_rule_values (dataset_name, variable_ordinal, ordinal, value_kind, numeric_value) VALUES ('A', 1, 3, 'numeric', 99)"); + $pdo->exec("UPDATE variable_attribute SET attribute_value = 'updated' WHERE attribute_name = 'Origin' AND array_ordinal = 2 AND variable_id IN (SELECT v.variable_id FROM variable v JOIN dataset d ON d.dataset_id = v.dataset_id WHERE d.dataset_name = 'A' AND v.source_ordinal = 1)"); + $pdo->exec("UPDATE variable SET variable_role = 0, display_width = 13 WHERE dataset_id = (SELECT dataset_id FROM dataset WHERE dataset_name = 'A') AND source_ordinal = 1"); + $pdo->exec("UPDATE value_labels SET label = 'Updated' WHERE dataset_name = 'A' AND variable_ordinal IN (1, 3) AND ordinal = 1"); + $pdo->exec("UPDATE missing_rule_values SET numeric_value = 4 WHERE dataset_name = 'A' AND variable_ordinal = 1 AND ordinal = 2"); + $pdo->exec("UPDATE variable_attributes SET value = 'updated' WHERE dataset_name = 'A' AND variable_ordinal = 1 AND attribute_name = 'Origin' AND ordinal = 2"); + $pdo->exec("UPDATE variable_roles SET role = 0 WHERE dataset_name = 'A' AND variable_ordinal = 1"); + $pdo->exec("UPDATE variable_display_metadata SET display_width = 13 WHERE dataset_name = 'A' AND variable_ordinal = 1"); + $pdo->exec("DELETE FROM variable_set_member WHERE source_ordinal = 2 AND variable_set_id IN (SELECT s.variable_set_id FROM variable_set s JOIN dataset d ON d.dataset_id = s.dataset_id WHERE d.dataset_name = 'A')"); + $pdo->exec("DELETE FROM multiple_response_member WHERE source_ordinal = 2 AND multiple_response_set_id IN (SELECT s.multiple_response_set_id FROM multiple_response_set s JOIN dataset d ON d.dataset_id = s.dataset_id WHERE d.dataset_name = 'A')"); + $pdo->exec("DELETE FROM variable_set_members WHERE dataset_name = 'A' AND member_ordinal = 2"); + $pdo->exec("DELETE FROM multiple_response_set_members WHERE dataset_name = 'A' AND member_ordinal = 2"); + $pdo->commit(); + $pdo->exec('PRAGMA query_only = ON'); + $this->assertGroupedExport($this->groupedExportFixture(4, 3, changed: true), $this->publicExport($exporter, $engine, 'A'), 'sav'); + } + + /** @param class-string $class */ + #[DataProvider('groupedExporters')] + public function testGroupedExportRetainsMalformedMissingAndMemberDiagnostics(string $class): void + { + $pdo = new PDO('sqlite::memory:'); + $engine = new FakeSpssEngine($this->fixture()); + $adapter = new SpssAdapter($pdo, $engine); + $adapter->import('fixture.sav', 'A'); + $exporter = $class === SpssAdapter::class ? $adapter : new $class($pdo); + $canonical = in_array($class, [CanonicalWideTableExporter::class, SpssAdapter::class], true); + $profile = match ($class) { + MySqlWideTableExporter::class => 'MySQL-family', + PostgreSqlWideTableExporter::class => 'PostgreSQL', + default => 'SQLite', + }; + $corruptions = $canonical ? [ + ["INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind) SELECT 'bad', variable_id, 2, 'numeric_range' FROM missing_rule", 'Invalid discrete missing rule.'], + ["UPDATE variable_set_member SET variable_id = 'unknown' WHERE source_ordinal = 1", 'Invalid string value.'], + ["UPDATE multiple_response_member SET variable_id = 'unknown' WHERE source_ordinal = 1", 'Invalid string value.'], + ] : [ + ['DELETE FROM missing_rule_values WHERE ordinal = 2', $profile === 'SQLite' ? 'A numeric user-missing rule contains a non-numeric catalogue value.' : "The $profile user-missing rule has an incomplete ordered value list."], + ['UPDATE variable_set_members SET variable_ordinal = 999 WHERE member_ordinal = 1', "A $profile variable set references an unknown variable."], + ['UPDATE multiple_response_set_members SET variable_ordinal = 999 WHERE member_ordinal = 1', "A $profile multiple-response set references an unknown variable."], + ]; + $pdo->exec('PRAGMA foreign_keys = OFF'); + foreach ($corruptions as [$sql, $message]) { + $pdo->beginTransaction(); + try { + $pdo->exec($sql); + $this->publicExport($exporter, $engine, 'A'); + self::fail('Malformed grouped metadata was silently discarded.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertSame($message, $exception->getMessage()); + } finally { + $pdo->rollBack(); + } + } + } + + /** @return array{dataset: Dataset, caseCount: int, diagnostics: list<\OpenStatSpec\Core\FidelityDiagnostic>} */ + private function publicExport(CanonicalWideTableExporter|SqliteWideTableExporter|MySqlWideTableExporter|PostgreSqlWideTableExporter|SpssAdapter $exporter, FakeSpssEngine $engine, string $name, string $format = 'sav'): array + { + if (!$exporter instanceof SpssAdapter) { + return $exporter->export($name, $format); + } + $target = sys_get_temp_dir() . '/oss-grouped-' . bin2hex(random_bytes(8)) . '.' . $format; + try { + $result = $exporter->export($name, $target); + return ['dataset' => $engine->lastWrite()['dataset'], 'caseCount' => $result->caseCount, 'diagnostics' => $result->diagnostics]; + } finally { + @unlink($target); + } + } + + /** @param array{dataset: Dataset, caseCount: int, diagnostics: list<\OpenStatSpec\Core\FidelityDiagnostic>} $export */ + private function assertGroupedExport(Dataset $expected, array $export, string $format): void + { + self::assertSame([], $export['diagnostics']); + self::assertSame($expected->rowCount(), $export['caseCount']); + self::assertSame($expected->rows(), $export['dataset']->rows()); + // Compare values and ordering, not object sharing between dictionary entries. + self::assertSame(array_map('serialize', $expected->variables()), array_map('serialize', $export['dataset']->variables())); + self::assertSame(serialize($expected->metadata), serialize($export['dataset']->metadata)); + self::assertEquals(new FileTechnicalMetadata( + sourceFormat: $format, + recordType: $format === 'zsav' ? '$FL3' : '$FL2', + sourceVersion: 'OpenStatSpec 0.1', + provenance: 'Päritolu: küsitlus', + encoding: 'UTF-8', + productName: 'OpenStatSpec tööriist', + compression: $format === 'zsav' ? 2 : 1, + ), $export['dataset']->technicalMetadata); + } + + private function groupedExportFixture(int $count, int $setCount, bool $empty = false, bool $changed = false): Dataset + { + $base = $this->fixture(); + $variables = []; + for ($i = 1; $i <= $count; ++$i) { + $template = $base->variables()[$i === 2 ? 1 : 0]; + $name = $i <= 2 ? $template->name : 'Extra' . $i; + $variables[] = new VariableMetadata(...array_replace(get_object_vars($template), [ + 'name' => $name, + 'dictionaryIndex' => $i, + 'writeFormat' => $i === 2 ? $template->writeFormat : new VariableFormat(5, 12, 2), + 'valueLabels' => new ValueLabelSet($empty || $i === 2 ? [] : [new ValueLabel(7.0, $changed && in_array($i, [1, 3], true) ? 'Updated' : 'Seven'), new ValueLabel(5.0, 'Viis')], [$name]), + 'missingValues' => $empty ? MissingValues::none() : ($changed && $i === 1 ? MissingValues::rangeAndValue(1.0, 4.0, 99.0) : $template->missingValues), + 'role' => $changed && $i === 1 ? VariableRole::INPUT : $template->role, + 'columns' => $changed && $i === 1 ? 13 : $template->columns, + 'attributes' => $empty ? [] : ($i === 2 ? $template->attributes() : [new VariableAttribute($name, 'Origin', ['customer', $changed && $i === 1 ? 'updated' : 'identifier']), new VariableAttribute($name, 'Source', ['õ', ''])]), + ])); + } + $sets = $multiple = []; + for ($i = 1; $i <= $setCount; ++$i) { + $members = $i === 1 ? [] : ($changed ? ['Favourite colour'] : ['Favourite colour', 'Respondent ID']); + $sets[] = new VariableSet('Set' . $i, $members); + $multiple[] = new MultipleResponseSet('$Set' . $i, MultipleResponseSetType::CATEGORY, $members, 'Set ' . $i); + } + return new Dataset( + new VariableDictionary($variables), + $empty ? [] : [array_merge([7.0, 'õ'], array_fill(0, $count - 2, 7.0)), array_merge([null, ''], array_fill(0, $count - 2, null))], + new FileMetadata( + $empty ? null : $base->metadata->label, + weightVariableName: $empty ? null : $base->metadata->weightVariableName, + documents: $empty ? [] : $base->metadata->documents(), + attributes: $empty ? [] : $base->metadata->attributes(), + variableSets: $sets, + multipleResponseSets: $multiple, + ), + $base->technicalMetadata, + ); + } + private function fixture(string $sourceFormat = 'zsav'): Dataset { return new Dataset( diff --git a/tests/Sql/MySqlWideTableExporterTest.php b/tests/Sql/MySqlWideTableExporterTest.php index 861a205..45c8506 100644 --- a/tests/Sql/MySqlWideTableExporterTest.php +++ b/tests/Sql/MySqlWideTableExporterTest.php @@ -45,14 +45,14 @@ public function testRestoresCoreDictionaryDisplayAndFileMetadata(): void ['value_kind' => 'numeric', 'numeric_value' => '1', 'text_value' => null, 'label' => 'Yes'], ['value_kind' => 'numeric', 'numeric_value' => '2', 'text_value' => null, 'label' => 'No'], ]), - 'missing' => $this->statement([], [], -3), + 'missing' => $this->statement([['missing_format' => -3]], [], -3), 'missing_values' => $this->statement([ ['value_kind' => 'numeric', 'numeric_value' => '-99', 'text_value' => null], ['value_kind' => 'numeric', 'numeric_value' => '99', 'text_value' => null], ['value_kind' => 'numeric', 'numeric_value' => '-1', 'text_value' => null], ]), 'display' => $this->statement([['measurement_level' => '3', 'display_width' => '12', 'alignment' => '1']]), - 'roles' => $this->statement([], [], 0), + 'roles' => $this->statement([['role' => 0]], [], 0), 'variable_attributes' => $this->statement(), 'file_attributes' => $this->statement(), 'variable_sets' => $this->statement(), @@ -204,7 +204,7 @@ private function extensionExport(array $catalog): array 'labels' => $this->statement(), 'missing' => $this->statement([], [], false), 'display' => $this->statement(), - 'roles' => $this->statement([], [], $catalog['roles']), + 'roles' => $this->statement(array_map(static fn(array $variable): array => ['variable_ordinal' => $variable['ordinal'], 'role' => $catalog['roles']], array_values($catalog['variables'])), [], $catalog['roles']), 'variable_attributes' => $this->statement($catalog['variable_attributes']), 'dataset_metadata' => $this->statement([], [], false), 'documents' => $this->statement([], []), @@ -266,9 +266,11 @@ private function variable(string $name, int $ordinal = 1): array */ private function statement(array $rows = [], array $all = [], mixed $column = false): PDOStatement { + // Include owner keys for grouped reads while retaining real-driver numeric strings. + $rows = array_map(static fn(array $row): array => $row + ['dataset_name' => 'fixture', 'variable_ordinal' => 1, 'set_ordinal' => 1], $rows); $statement = $this->createStub(PDOStatement::class); $statement->method('execute')->willReturn(true); - $statement->method('fetchAll')->willReturn($all); + $statement->method('fetchAll')->willReturn($all !== [] ? $all : $rows); $statement->method('fetchColumn')->willReturn($column); $index = 0; $statement->method('fetch')->willReturnCallback(static function () use ($rows, &$index): array|false { diff --git a/tests/Sql/PostgreSqlWideTableExporterTest.php b/tests/Sql/PostgreSqlWideTableExporterTest.php index ce360e6..9eac314 100644 --- a/tests/Sql/PostgreSqlWideTableExporterTest.php +++ b/tests/Sql/PostgreSqlWideTableExporterTest.php @@ -120,7 +120,7 @@ private function export(array $catalog): array 'labels' => $this->statement(), 'missing' => $this->statement([], [], false), 'display' => $this->statement(), - 'roles' => $this->statement([], [], $catalog['roles']), + 'roles' => $this->statement(array_map(static fn(array $variable): array => ['variable_ordinal' => $variable['ordinal'], 'role' => $catalog['roles']], array_values($catalog['variables'])), [], $catalog['roles']), 'variable_attributes' => $this->statement($catalog['variable_attributes']), 'dataset_metadata' => $this->statement([], [], false), 'documents' => $this->statement([], []), @@ -163,9 +163,11 @@ private function export(array $catalog): array */ private function statement(array $rows = [], array $all = [], mixed $column = false): PDOStatement { + // Include owner keys for either per-owner or grouped public-export reads. + $rows = array_map(static fn(array $row): array => $row + ['dataset_name' => 'fixture', 'variable_ordinal' => 1, 'set_ordinal' => 1], $rows); $statement = $this->createStub(PDOStatement::class); $statement->method('execute')->willReturn(true); - $statement->method('fetchAll')->willReturn($all); + $statement->method('fetchAll')->willReturn($all !== [] ? $all : $rows); $statement->method('fetchColumn')->willReturn($column); $index = 0; $statement->method('fetch')->willReturnCallback(static function () use ($rows, &$index): array|false { diff --git a/tests/Support/ExportCountingPdo.php b/tests/Support/ExportCountingPdo.php new file mode 100644 index 0000000..42da6f8 --- /dev/null +++ b/tests/Support/ExportCountingPdo.php @@ -0,0 +1,70 @@ +}> */ + public ?array $executions = null; + public int $prepares = 0; + + /** @param array $options */ + public function __construct(string $dsn, ?string $username = null, ?string $password = null, array $options = []) + { + parent::__construct($dsn, $username, $password, $options); + $this->setAttribute(PDO::ATTR_STATEMENT_CLASS, [ExportCountingStatement::class, [$this]]); + } + + /** @return list}> */ + public function captured(): array + { + return $this->executions ?? []; + } + + /** @param null|array $parameters */ + public function record(string $sql, ?array $parameters = null): void + { + if ($this->executions !== null) { + $this->executions[] = ['sql' => $sql, 'parameters' => $parameters]; + } + } + + /** @param array $options */ + public function prepare(string $query, array $options = []): PDOStatement|false + { + if ($this->executions !== null) { + ++$this->prepares; + } + return parent::prepare($query, $options); + } + + public function query(string $query, ?int $fetchMode = null, mixed ...$fetchModeArgs): PDOStatement|false + { + $this->record($query); + return parent::query($query, $fetchMode, ...$fetchModeArgs); + } + + public function exec(string $statement): int|false + { + $this->record($statement); + return parent::exec($statement); + } +} + +final class ExportCountingStatement extends PDOStatement +{ + protected function __construct(private ExportCountingPdo $pdo) {} + + /** @param null|array $params */ + public function execute(?array $params = null): bool + { + $this->pdo->record($this->queryString, $params); + return parent::execute($params); + } +} From 5e00c742426508da711d0f32bad8e38b49f6bda7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 8 Sep 2026 16:55:01 +0300 Subject: [PATCH 2/3] perf: load export dictionary children in dataset-scoped groups --- src/Sql/CanonicalWideTableExporter.php | 44 +++++++---- src/Sql/MySqlWideTableExporter.php | 100 +++++++++++++----------- src/Sql/PostgreSqlWideTableExporter.php | 100 +++++++++++++----------- src/Sql/SqliteWideTableExporter.php | 93 ++++++++++++---------- 4 files changed, 195 insertions(+), 142 deletions(-) diff --git a/src/Sql/CanonicalWideTableExporter.php b/src/Sql/CanonicalWideTableExporter.php index 11a5168..f4a37b6 100644 --- a/src/Sql/CanonicalWideTableExporter.php +++ b/src/Sql/CanonicalWideTableExporter.php @@ -50,6 +50,9 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array if ($variables === []) { throw $this->invalid('The canonical dataset has no variables.'); } + $labelsByVariable = $this->all('SELECT link.variable_id, label.* FROM variable_value_label_set link JOIN variable ON variable.variable_id = link.variable_id JOIN value_label label ON label.value_label_set_id = link.value_label_set_id WHERE variable.dataset_id = ? ORDER BY link.variable_id, label.ordinal', [$id], 'variable_id'); + $attributesByVariable = $this->all('SELECT attribute.* FROM variable_attribute attribute JOIN variable ON variable.variable_id = attribute.variable_id WHERE variable.dataset_id = ? ORDER BY attribute.variable_id, attribute.attribute_name, attribute.array_ordinal', [$id], 'variable_id'); + $missingByVariable = $this->all('SELECT rule.* FROM missing_rule rule JOIN variable ON variable.variable_id = rule.variable_id WHERE variable.dataset_id = ? ORDER BY rule.variable_id, rule.ordinal', [$id], 'variable_id'); $typedVariables = []; $columns = []; foreach ($variables as $variable) { @@ -62,11 +65,11 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array default => throw $this->invalid('Invalid variable storage kind.'), }; $labels = []; - foreach ($this->all('SELECT label.* FROM variable_value_label_set link JOIN value_label label ON label.value_label_set_id = link.value_label_set_id WHERE link.variable_id = ? ORDER BY label.ordinal', [$variableId]) as $label) { + foreach ($labelsByVariable[$variableId] ?? [] as $label) { $labels[] = new ValueLabel($this->value($label['code_kind'], $label['numeric_code'], $label['string_code']), $this->text($label['label'])); } $attributes = []; - foreach ($this->attributes('SELECT attribute_name, attribute_value FROM variable_attribute WHERE variable_id = ? ORDER BY attribute_name, array_ordinal', $variableId) as $attributeName => $values) { + foreach ($this->attributes($attributesByVariable[$variableId] ?? []) as $attributeName => $values) { $attributes[] = new VariableAttribute($name, $attributeName, $values); } $typedVariables[] = new VariableMetadata( @@ -77,7 +80,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array writeFormat: $this->format($variable, 'write', $type), label: $variable['variable_label'] === null ? null : $this->text($variable['variable_label']), valueLabels: new ValueLabelSet($labels, [$name]), - missingValues: $this->missingValues($variableId), + missingValues: $this->missingValues($missingByVariable[$variableId] ?? []), measure: match ($variable['measurement_level']) { 'nominal' => Measure::NOMINAL, 'ordinal' => Measure::ORDINAL, @@ -111,7 +114,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array $rows[] = $values; } $attributes = []; - foreach ($this->attributes('SELECT attribute_name, attribute_value FROM dataset_attribute WHERE dataset_id = ? ORDER BY attribute_name, array_ordinal', $id) as $name => $values) { + foreach ($this->attributes($this->all('SELECT attribute_name, attribute_value FROM dataset_attribute WHERE dataset_id = ? ORDER BY attribute_name, array_ordinal', [$id])) as $name => $values) { $attributes[] = new FileAttribute($name, $values); } $weight = $this->all('SELECT variable.source_name FROM dataset_weight_variable weight LEFT JOIN variable ON variable.variable_id = weight.variable_id AND variable.dataset_id = weight.dataset_id WHERE weight.dataset_id = ?', [$id]); @@ -167,9 +170,9 @@ private function format(array $variable, string $prefix, VariableType $type): Va ); } - private function missingValues(string $variableId): MissingValues + /** @param list> $rules */ + private function missingValues(array $rules): MissingValues { - $rules = $this->all('SELECT * FROM missing_rule WHERE variable_id = ? ORDER BY ordinal', [$variableId]); if ($rules === []) { return MissingValues::none(); } @@ -197,8 +200,9 @@ private function missingValues(string $variableId): MissingValues private function variableSets(string $datasetId): array { $sets = []; + $membersBySet = $this->all('SELECT member.variable_set_id, variable.source_name FROM variable_set_member member JOIN variable_set owner ON owner.variable_set_id = member.variable_set_id LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = owner.dataset_id WHERE owner.dataset_id = ? ORDER BY member.variable_set_id, member.source_ordinal', [$datasetId], 'variable_set_id'); foreach ($this->all('SELECT variable_set_id, set_name FROM variable_set WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]) as $set) { - $members = $this->all('SELECT variable.source_name FROM variable_set_member member LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = ? WHERE member.variable_set_id = ? ORDER BY member.source_ordinal', [$datasetId, $set['variable_set_id']]); + $members = $membersBySet[$set['variable_set_id']] ?? []; $sets[] = new VariableSet($this->string($set['set_name']), array_map(fn(array $member): string => $this->string($member['source_name']), $members)); } return $sets; @@ -208,8 +212,9 @@ private function variableSets(string $datasetId): array private function multipleResponseSets(string $datasetId): array { $sets = []; + $membersBySet = $this->all('SELECT member.multiple_response_set_id, variable.source_name FROM multiple_response_member member JOIN multiple_response_set owner ON owner.multiple_response_set_id = member.multiple_response_set_id LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = owner.dataset_id WHERE owner.dataset_id = ? ORDER BY member.multiple_response_set_id, member.source_ordinal', [$datasetId], 'multiple_response_set_id'); foreach ($this->all('SELECT * FROM multiple_response_set WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]) as $set) { - $members = $this->all('SELECT variable.source_name FROM multiple_response_member member LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = ? WHERE member.multiple_response_set_id = ? ORDER BY member.source_ordinal', [$datasetId, $set['multiple_response_set_id']]); + $members = $membersBySet[$set['multiple_response_set_id']] ?? []; $counted = $set['counted_value_kind'] === null ? null : $this->value($set['counted_value_kind'], $set['counted_numeric_value'], $set['counted_string_value']); if (is_float($counted)) { if (floor($counted) !== $counted || $counted >= (float) PHP_INT_MAX || $counted < PHP_INT_MIN) { @@ -238,11 +243,14 @@ private function multipleResponseSets(string $datasetId): array return $sets; } - /** @return array> */ - private function attributes(string $sql, string $id): array + /** + * @param list> $rows + * @return array> + */ + private function attributes(array $rows): array { $attributes = []; - foreach ($this->all($sql, [$id]) as $row) { + foreach ($rows as $row) { $attributes[$this->string($row['attribute_name'])][] = $this->text($row['attribute_value']); } return $attributes; @@ -297,15 +305,23 @@ private function optionalText(mixed $value): ?string /** * @param list $parameters - * @return list> + * @return ($groupBy is null ? list> : array>>) */ - private function all(string $sql, array $parameters): array + private function all(string $sql, array $parameters, ?string $groupBy = null): array { $statement = $this->pdo->prepare($sql); if ($statement === false || !$statement->execute($parameters)) { throw $this->invalid('Could not read the canonical dataset.'); } - return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if ($groupBy === null) { + return array_values($rows); + } + $grouped = []; + foreach ($rows as $row) { + $grouped[$row[$groupBy]][] = $row; + } + return $grouped; } private function quote(string $identifier): string diff --git a/src/Sql/MySqlWideTableExporter.php b/src/Sql/MySqlWideTableExporter.php index 24621dd..7925dc6 100644 --- a/src/Sql/MySqlWideTableExporter.php +++ b/src/Sql/MySqlWideTableExporter.php @@ -66,13 +66,19 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array $rows[] = $values; } + $labels = $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $rules = $this->grouped('SELECT variable_ordinal, missing_format FROM missing_rules WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $missingValues = $rules === [] ? [] : $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $displays = $this->grouped('SELECT variable_ordinal, measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $roles = $this->grouped('SELECT variable_ordinal, role FROM variable_roles WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $attributes = $this->grouped('SELECT variable_ordinal, attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? ORDER BY variable_ordinal, attribute_name, ordinal', $datasetName); $typedVariables = []; foreach ($variables as $variable) { $isString = $variable['storage_kind'] === 'string'; $printFormatWidth = $isString ? min($variable['format_width'], 255) : $variable['format_width']; $writeFormatWidth = $isString ? min($variable['write_format_width'], 255) : $variable['write_format_width']; - $dictionary = $this->dictionary($datasetName, $variable['ordinal']); - $display = $this->display($datasetName, $variable['ordinal']); + $dictionary = $this->dictionary($labels[$variable['ordinal']] ?? [], $rules[$variable['ordinal']][0]['missing_format'] ?? null, $missingValues[$variable['ordinal']] ?? []); + $display = $this->display($displays[$variable['ordinal']][0] ?? null); $typedVariables[] = new VariableMetadata( name: $variable['source_name'], type: $isString ? VariableType::STRING : VariableType::NUMERIC, @@ -85,8 +91,8 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array measure: $display['measure'], alignment: $display['alignment'], columns: $display['columns'], - role: $this->role($datasetName, $variable['ordinal']), - attributes: $this->variableAttributes($datasetName, $variable['ordinal'], $variable['source_name']), + role: $this->role($roles[$variable['ordinal']][0]['role'] ?? null), + attributes: $this->variableAttributes($attributes[$variable['ordinal']] ?? [], $variable['source_name']), dictionaryIndex: $variable['ordinal'], ); } @@ -179,12 +185,12 @@ private function variables(string $datasetName): array return $variables; } - /** @return array{measure: Measure, columns: int, alignment: Alignment} */ - private function display(string $datasetName, int $ordinal): array + /** + * @param array|null $row + * @return array{measure: Measure, columns: int, alignment: Alignment} + */ + private function display(?array $row): array { - $statement = $this->statement('SELECT measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $row = $statement->fetch(PDO::FETCH_ASSOC); if (!is_array($row)) { return ['measure' => Measure::UNKNOWN, 'columns' => 8, 'alignment' => Alignment::LEFT]; } @@ -200,22 +206,22 @@ private function display(string $datasetName, int $ordinal): array ]; } - /** @return array{labels: list, missing: MissingValues} */ - private function dictionary(string $datasetName, int $ordinal): array + /** + * @param list> $labels + * @param list> $missingValues + * @return array{labels: list, missing: MissingValues} + */ + private function dictionary(array $labels, mixed $format, array $missingValues): array { - $labels = $this->statement('SELECT value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $labels->execute([$datasetName, $ordinal]); $typedLabels = []; - while (($row = $labels->fetch(PDO::FETCH_ASSOC)) !== false) { - if (!is_array($row) || !is_string($row['label'] ?? null)) { + foreach ($labels as $row) { + if (!is_string($row['label'] ?? null)) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family value-label catalogue is malformed.'); } $typedLabels[] = new ValueLabel($this->dictionaryValue($row), $row['label']); } - $rule = $this->statement('SELECT missing_format FROM missing_rules WHERE dataset_name = ? AND variable_ordinal = ?'); - $rule->execute([$datasetName, $ordinal]); - $format = $this->integer($rule->fetchColumn()); + $format = $this->integer($format); if ($format === null || $format === 0) { return ['labels' => $typedLabels, 'missing' => MissingValues::none()]; } @@ -223,13 +229,8 @@ private function dictionary(string $datasetName, int $ordinal): array throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing rule has an unsupported SPSS missing format.'); } - $valuesStatement = $this->statement('SELECT value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $valuesStatement->execute([$datasetName, $ordinal]); $values = []; - while (($row = $valuesStatement->fetch(PDO::FETCH_ASSOC)) !== false) { - if (!is_array($row)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family user-missing value catalogue is malformed.'); - } + foreach ($missingValues as $row) { $values[] = $this->dictionaryValue($row); } @@ -294,11 +295,9 @@ private function numeric(int|float|string $value): int|float throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS missing-value ranges require numeric endpoints.'); } - private function role(string $datasetName, int $ordinal): VariableRole + private function role(mixed $role): VariableRole { - $statement = $this->statement('SELECT role FROM variable_roles WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $role = $this->integer($statement->fetchColumn()); + $role = $this->integer($role); if ($role === null || ($typed = VariableRole::tryFrom($role)) === null) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family variable role catalogue is malformed.'); } @@ -306,14 +305,14 @@ private function role(string $datasetName, int $ordinal): VariableRole return $typed; } - /** @return list */ - private function variableAttributes(string $datasetName, int $ordinal, string $variableName): array + /** + * @param list> $rows + * @return list + */ + private function variableAttributes(array $rows, string $variableName): array { - $statement = $this->statement('SELECT attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY attribute_name, ordinal'); - $statement->execute([$datasetName, $ordinal]); - $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues($rows) as $name => $values) { $attributes[] = new VariableAttribute($variableName, $name, $values); } @@ -327,19 +326,22 @@ private function fileAttributes(string $datasetName): array $statement->execute([$datasetName]); $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues(array_values($statement->fetchAll(PDO::FETCH_ASSOC))) as $name => $values) { $attributes[] = new FileAttribute($name, $values); } return $attributes; } - /** @return array> */ - private function attributeValues(PDOStatement $statement): array + /** + * @param list> $rows + * @return array> + */ + private function attributeValues(array $rows): array { /** @var array> $grouped */ $grouped = []; - while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($rows as $row) { $name = $row['attribute_name'] ?? null; $value = $row['value'] ?? null; if (!is_string($name) || $name === '' || !is_string($value)) { @@ -356,7 +358,7 @@ private function variableSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name FROM variable_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $this->integer($set['set_ordinal'] ?? null); @@ -364,9 +366,8 @@ private function variableSets(string $datasetName): array if ($ordinal === null || !is_string($name) || $name === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family variable-set catalogue is malformed.'); } - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL-family variable set references an unknown variable.'); @@ -384,7 +385,7 @@ private function multipleResponseSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name, set_type, label, counted_value_kind, counted_numeric_value, counted_text_value, category_labels, label_source FROM multiple_response_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $this->integer($set['set_ordinal'] ?? null); @@ -397,9 +398,8 @@ private function multipleResponseSets(string $datasetName): array throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The MySQL-family multiple-response-set catalogue is malformed.'); } $countedValue = $this->countedValue($set); - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A MySQL-family multiple-response set references an unknown variable.'); @@ -516,6 +516,18 @@ private function integer(mixed $value): ?int return is_int($value) ? $value : (is_string($value) && preg_match('/^-?[0-9]+$/D', $value) === 1 ? (int) $value : null); } + /** @return array>> */ + private function grouped(string $sql, string $datasetName, string $key = 'variable_ordinal'): array + { + $statement = $this->statement($sql); + $statement->execute([$datasetName]); + $grouped = []; + while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + $grouped[$row[$key]][] = $row; + } + return $grouped; + } + private function statement(string $sql): PDOStatement { $statement = $this->pdo->prepare($sql); diff --git a/src/Sql/PostgreSqlWideTableExporter.php b/src/Sql/PostgreSqlWideTableExporter.php index ddb17e5..f4b4032 100644 --- a/src/Sql/PostgreSqlWideTableExporter.php +++ b/src/Sql/PostgreSqlWideTableExporter.php @@ -56,11 +56,17 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array $rows[] = $values; } + $labels = $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $rules = $this->grouped('SELECT variable_ordinal, missing_format FROM missing_rules WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $missingValues = $rules === [] ? [] : $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $displays = $this->grouped('SELECT variable_ordinal, measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $roles = $this->grouped('SELECT variable_ordinal, role FROM variable_roles WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $attributes = $this->grouped('SELECT variable_ordinal, attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? ORDER BY variable_ordinal, attribute_name, ordinal', $datasetName); $typedVariables = []; foreach ($variables as $variable) { $isString = $variable['storage_kind'] === 'string'; - $dictionary = $this->dictionary($datasetName, $variable['ordinal']); - $display = $this->display($datasetName, $variable['ordinal']); + $dictionary = $this->dictionary($labels[$variable['ordinal']] ?? [], $rules[$variable['ordinal']][0]['missing_format'] ?? null, $missingValues[$variable['ordinal']] ?? []); + $display = $this->display($displays[$variable['ordinal']][0] ?? null); $typedVariables[] = new VariableMetadata( name: $variable['source_name'], type: $isString ? VariableType::STRING : VariableType::NUMERIC, @@ -73,8 +79,8 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array measure: $display['measure'], alignment: $display['alignment'], columns: $display['columns'], - role: $this->role($datasetName, $variable['ordinal']), - attributes: $this->variableAttributes($datasetName, $variable['ordinal'], $variable['source_name']), + role: $this->role($roles[$variable['ordinal']][0]['role'] ?? null), + attributes: $this->variableAttributes($attributes[$variable['ordinal']] ?? [], $variable['source_name']), dictionaryIndex: $variable['ordinal'], ); } @@ -162,12 +168,12 @@ private function variables(string $datasetName): array return $variables; } - /** @return array{measure: Measure, columns: int, alignment: Alignment} */ - private function display(string $datasetName, int $ordinal): array + /** + * @param array|null $row + * @return array{measure: Measure, columns: int, alignment: Alignment} + */ + private function display(?array $row): array { - $statement = $this->statement('SELECT measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $row = $statement->fetch(PDO::FETCH_ASSOC); if (!is_array($row)) { return ['measure' => Measure::UNKNOWN, 'columns' => 8, 'alignment' => Alignment::LEFT]; } @@ -183,22 +189,22 @@ private function display(string $datasetName, int $ordinal): array ]; } - /** @return array{labels: list, missing: MissingValues} */ - private function dictionary(string $datasetName, int $ordinal): array + /** + * @param list> $labels + * @param list> $missingValues + * @return array{labels: list, missing: MissingValues} + */ + private function dictionary(array $labels, mixed $format, array $missingValues): array { - $labels = $this->statement('SELECT value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $labels->execute([$datasetName, $ordinal]); $typedLabels = []; - while (($row = $labels->fetch(PDO::FETCH_ASSOC)) !== false) { - if (!is_array($row) || !is_string($row['label'] ?? null)) { + foreach ($labels as $row) { + if (!is_string($row['label'] ?? null)) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL value-label catalogue is malformed.'); } $typedLabels[] = new ValueLabel($this->dictionaryValue($row), $row['label']); } - $rule = $this->statement('SELECT missing_format FROM missing_rules WHERE dataset_name = ? AND variable_ordinal = ?'); - $rule->execute([$datasetName, $ordinal]); - $format = $this->integer($rule->fetchColumn()); + $format = $this->integer($format); if ($format === null || $format === 0) { return ['labels' => $typedLabels, 'missing' => MissingValues::none()]; } @@ -206,13 +212,8 @@ private function dictionary(string $datasetName, int $ordinal): array throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL user-missing rule has an unsupported SPSS missing format.'); } - $valuesStatement = $this->statement('SELECT value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $valuesStatement->execute([$datasetName, $ordinal]); $values = []; - while (($row = $valuesStatement->fetch(PDO::FETCH_ASSOC)) !== false) { - if (!is_array($row)) { - throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL user-missing value catalogue is malformed.'); - } + foreach ($missingValues as $row) { $values[] = $this->dictionaryValue($row); } @@ -277,11 +278,9 @@ private function numeric(int|float|string $value): int|float throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'SPSS missing-value ranges require numeric endpoints.'); } - private function role(string $datasetName, int $ordinal): VariableRole + private function role(mixed $role): VariableRole { - $statement = $this->statement('SELECT role FROM variable_roles WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $role = $this->integer($statement->fetchColumn()); + $role = $this->integer($role); if ($role === null || ($typed = VariableRole::tryFrom($role)) === null) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL variable role catalogue is malformed.'); } @@ -289,14 +288,14 @@ private function role(string $datasetName, int $ordinal): VariableRole return $typed; } - /** @return list */ - private function variableAttributes(string $datasetName, int $ordinal, string $variableName): array + /** + * @param list> $rows + * @return list + */ + private function variableAttributes(array $rows, string $variableName): array { - $statement = $this->statement('SELECT attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY attribute_name, ordinal'); - $statement->execute([$datasetName, $ordinal]); - $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues($rows) as $name => $values) { $attributes[] = new VariableAttribute($variableName, $name, $values); } @@ -310,19 +309,22 @@ private function fileAttributes(string $datasetName): array $statement->execute([$datasetName]); $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues(array_values($statement->fetchAll(PDO::FETCH_ASSOC))) as $name => $values) { $attributes[] = new FileAttribute($name, $values); } return $attributes; } - /** @return array> */ - private function attributeValues(PDOStatement $statement): array + /** + * @param list> $rows + * @return array> + */ + private function attributeValues(array $rows): array { /** @var array> $grouped */ $grouped = []; - while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($rows as $row) { $name = $row['attribute_name'] ?? null; $value = $row['value'] ?? null; if (!is_string($name) || $name === '' || !is_string($value)) { @@ -339,7 +341,7 @@ private function variableSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name FROM variable_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $this->integer($set['set_ordinal'] ?? null); @@ -347,9 +349,8 @@ private function variableSets(string $datasetName): array if ($ordinal === null || !is_string($name) || $name === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL variable-set catalogue is malformed.'); } - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A PostgreSQL variable set references an unknown variable.'); @@ -367,7 +368,7 @@ private function multipleResponseSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name, set_type, label, counted_value_kind, counted_numeric_value, counted_text_value, category_labels, label_source FROM multiple_response_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $this->integer($set['set_ordinal'] ?? null); @@ -380,9 +381,8 @@ private function multipleResponseSets(string $datasetName): array throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The PostgreSQL multiple-response-set catalogue is malformed.'); } $countedValue = $this->countedValue($set); - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A PostgreSQL multiple-response set references an unknown variable.'); @@ -499,6 +499,18 @@ private function integer(mixed $value): ?int return is_int($value) ? $value : (is_string($value) && preg_match('/^-?[0-9]+$/D', $value) === 1 ? (int) $value : null); } + /** @return array>> */ + private function grouped(string $sql, string $datasetName, string $key = 'variable_ordinal'): array + { + $statement = $this->statement($sql); + $statement->execute([$datasetName]); + $grouped = []; + while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + $grouped[$row[$key]][] = $row; + } + return $grouped; + } + private function statement(string $sql): PDOStatement { $statement = $this->pdo->prepare($sql); diff --git a/src/Sql/SqliteWideTableExporter.php b/src/Sql/SqliteWideTableExporter.php index 3556887..4d1bf19 100644 --- a/src/Sql/SqliteWideTableExporter.php +++ b/src/Sql/SqliteWideTableExporter.php @@ -74,10 +74,16 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array multipleResponseSets: $this->multipleResponseSets($datasetName), ); + $labels = $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $rules = $this->grouped('SELECT variable_ordinal, missing_format FROM missing_rules WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $missingValues = $rules === [] ? [] : $this->grouped('SELECT variable_ordinal, value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? ORDER BY variable_ordinal, ordinal', $datasetName); + $displays = $this->grouped('SELECT variable_ordinal, measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $roles = $this->grouped('SELECT variable_ordinal, role FROM variable_roles WHERE dataset_name = ? ORDER BY variable_ordinal', $datasetName); + $attributes = $this->grouped('SELECT variable_ordinal, attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? ORDER BY variable_ordinal, attribute_name, ordinal', $datasetName); $typedVariables = []; foreach ($variables as $variable) { - $dictionary = $this->dictionary($datasetName, $variable['ordinal']); - $display = $this->display($datasetName, $variable['ordinal']); + $dictionary = $this->dictionary($labels[$variable['ordinal']] ?? [], $rules[$variable['ordinal']][0]['missing_format'] ?? null, $missingValues[$variable['ordinal']] ?? []); + $display = $this->display($displays[$variable['ordinal']][0] ?? null); $isString = $variable['storage_kind'] === 'string'; $type = $isString ? VariableType::STRING : VariableType::NUMERIC; $width = $isString ? $variable['source_width'] : 0; @@ -98,8 +104,8 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array measure: $display['measure'], alignment: $display['alignment'], columns: $display['columns'], - role: $this->role($datasetName, $variable['ordinal']), - attributes: $this->variableAttributes($datasetName, $variable['ordinal'], $variable['source_name']), + role: $this->role($roles[$variable['ordinal']][0]['role'] ?? null), + attributes: $this->variableAttributes($attributes[$variable['ordinal']] ?? [], $variable['source_name']), dictionaryIndex: $variable['ordinal'], ); } @@ -167,30 +173,27 @@ static function (array $variable): array { )); } - /** @return array{labels: list, missing: MissingValues} */ - private function dictionary(string $datasetName, int $ordinal): array + /** + * @param list> $labels + * @param list> $missingValues + * @return array{labels: list, missing: MissingValues} + */ + private function dictionary(array $labels, mixed $format, array $missingValues): array { - $labels = $this->statement('SELECT value_kind, numeric_value, text_value, label FROM value_labels WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $labels->execute([$datasetName, $ordinal]); $typedLabels = []; - while (($row = $labels->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($labels as $row) { $value = $this->typedValue($row); if ($value !== null && is_string($row['label'] ?? null)) { $typedLabels[] = new ValueLabel($value, $row['label']); } } - $rule = $this->statement('SELECT missing_format FROM missing_rules WHERE dataset_name = ? AND variable_ordinal = ?'); - $rule->execute([$datasetName, $ordinal]); - $format = $rule->fetchColumn(); if (!is_int($format) || $format === 0) { return ['labels' => $typedLabels, 'missing' => MissingValues::none()]; } - $valuesStatement = $this->statement('SELECT value_kind, numeric_value, text_value FROM missing_rule_values WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY ordinal'); - $valuesStatement->execute([$datasetName, $ordinal]); $values = []; - while (($row = $valuesStatement->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($missingValues as $row) { $value = $this->typedValue($row); if ($value !== null) { $values[] = $value; @@ -206,12 +209,12 @@ private function dictionary(string $datasetName, int $ordinal): array return ['labels' => $typedLabels, 'missing' => $missing]; } - /** @return array{measure: Measure, columns: int, alignment: Alignment} */ - private function display(string $datasetName, int $ordinal): array + /** + * @param array|null $row + * @return array{measure: Measure, columns: int, alignment: Alignment} + */ + private function display(?array $row): array { - $statement = $this->statement('SELECT measurement_level, display_width, alignment FROM variable_display_metadata WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $row = $statement->fetch(PDO::FETCH_ASSOC); if (!is_array($row)) { return ['measure' => Measure::UNKNOWN, 'columns' => 8, 'alignment' => Alignment::LEFT]; } @@ -302,6 +305,18 @@ private function numeric(mixed $value): int|float return $value; } + /** @return array>> */ + private function grouped(string $sql, string $datasetName, string $key = 'variable_ordinal'): array + { + $statement = $this->statement($sql); + $statement->execute([$datasetName]); + $grouped = []; + while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + $grouped[$row[$key]][] = $row; + } + return $grouped; + } + private function statement(string $sql): PDOStatement { $statement = $this->pdo->prepare($sql); @@ -316,11 +331,8 @@ private function quote(string $identifier): string { return '"' . str_replace('"', '""', $identifier) . '"'; } - private function role(string $datasetName, int $ordinal): VariableRole + private function role(mixed $role): VariableRole { - $statement = $this->statement('SELECT role FROM variable_roles WHERE dataset_name = ? AND variable_ordinal = ?'); - $statement->execute([$datasetName, $ordinal]); - $role = $statement->fetchColumn(); if (!is_int($role) || ($typed = VariableRole::tryFrom($role)) === null) { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The SQLite variable role catalogue is malformed.'); } @@ -328,14 +340,14 @@ private function role(string $datasetName, int $ordinal): VariableRole return $typed; } - /** @return list */ - private function variableAttributes(string $datasetName, int $ordinal, string $variableName): array + /** + * @param list> $rows + * @return list + */ + private function variableAttributes(array $rows, string $variableName): array { - $statement = $this->statement('SELECT attribute_name, ordinal, value FROM variable_attributes WHERE dataset_name = ? AND variable_ordinal = ? ORDER BY attribute_name, ordinal'); - $statement->execute([$datasetName, $ordinal]); - $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues($rows) as $name => $values) { $attributes[] = new VariableAttribute($variableName, $name, $values); } @@ -349,19 +361,22 @@ private function fileAttributes(string $datasetName): array $statement->execute([$datasetName]); $attributes = []; - foreach ($this->attributeValues($statement) as $name => $values) { + foreach ($this->attributeValues(array_values($statement->fetchAll(PDO::FETCH_ASSOC))) as $name => $values) { $attributes[] = new FileAttribute($name, $values); } return $attributes; } - /** @return array> */ - private function attributeValues(PDOStatement $statement): array + /** + * @param list> $rows + * @return array> + */ + private function attributeValues(array $rows): array { /** @var array> $grouped */ $grouped = []; - while (($row = $statement->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($rows as $row) { $name = $row['attribute_name'] ?? null; $value = $row['value'] ?? null; if (!is_string($name) || $name === '' || !is_string($value)) { @@ -378,7 +393,7 @@ private function variableSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name FROM variable_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM variable_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $set['set_ordinal'] ?? null; @@ -386,9 +401,8 @@ private function variableSets(string $datasetName): array if (!is_int($ordinal) || !is_string($name) || $name === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The SQLite variable-set catalogue is malformed.'); } - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A SQLite variable set references an unknown variable.'); @@ -406,7 +420,7 @@ private function multipleResponseSets(string $datasetName): array { $sets = $this->statement('SELECT set_ordinal, name, set_type, label, counted_value_kind, counted_numeric_value, counted_text_value, category_labels, label_source FROM multiple_response_sets WHERE dataset_name = ? ORDER BY set_ordinal'); $sets->execute([$datasetName]); - $members = $this->statement('SELECT member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? AND member.set_ordinal = ? ORDER BY member.member_ordinal'); + $members = $this->grouped('SELECT member.set_ordinal, member.member_ordinal, variable.source_name FROM multiple_response_set_members member LEFT JOIN variables variable ON variable.dataset_name = member.dataset_name AND variable.ordinal = member.variable_ordinal WHERE member.dataset_name = ? ORDER BY member.set_ordinal, member.member_ordinal', $datasetName, 'set_ordinal'); $result = []; while (($set = $sets->fetch(PDO::FETCH_ASSOC)) !== false) { $ordinal = $set['set_ordinal'] ?? null; @@ -419,9 +433,8 @@ private function multipleResponseSets(string $datasetName): array throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'The SQLite multiple-response-set catalogue is malformed.'); } $countedValue = $this->countedValue($set); - $members->execute([$datasetName, $ordinal]); $names = []; - while (($member = $members->fetch(PDO::FETCH_ASSOC)) !== false) { + foreach ($members[$ordinal] ?? [] as $member) { $sourceName = $member['source_name'] ?? null; if (!is_string($sourceName) || $sourceName === '') { throw new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, 'A SQLite multiple-response set references an unknown variable.'); From 67534e8af73d6a4125876be79df3cc3828d28c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 8 Sep 2026 22:22:45 +0300 Subject: [PATCH 3/3] fix: avoid reserved attribute alias in Dolt export query --- src/Sql/CanonicalWideTableExporter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sql/CanonicalWideTableExporter.php b/src/Sql/CanonicalWideTableExporter.php index f4a37b6..23eebf3 100644 --- a/src/Sql/CanonicalWideTableExporter.php +++ b/src/Sql/CanonicalWideTableExporter.php @@ -51,7 +51,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array throw $this->invalid('The canonical dataset has no variables.'); } $labelsByVariable = $this->all('SELECT link.variable_id, label.* FROM variable_value_label_set link JOIN variable ON variable.variable_id = link.variable_id JOIN value_label label ON label.value_label_set_id = link.value_label_set_id WHERE variable.dataset_id = ? ORDER BY link.variable_id, label.ordinal', [$id], 'variable_id'); - $attributesByVariable = $this->all('SELECT attribute.* FROM variable_attribute attribute JOIN variable ON variable.variable_id = attribute.variable_id WHERE variable.dataset_id = ? ORDER BY attribute.variable_id, attribute.attribute_name, attribute.array_ordinal', [$id], 'variable_id'); + $attributesByVariable = $this->all('SELECT attr.* FROM variable_attribute attr JOIN variable ON variable.variable_id = attr.variable_id WHERE variable.dataset_id = ? ORDER BY attr.variable_id, attr.attribute_name, attr.array_ordinal', [$id], 'variable_id'); $missingByVariable = $this->all('SELECT rule.* FROM missing_rule rule JOIN variable ON variable.variable_id = rule.variable_id WHERE variable.dataset_id = ? ORDER BY rule.variable_id, rule.ordinal', [$id], 'variable_id'); $typedVariables = []; $columns = [];