Skip to content

perf: group export metadata reads by dataset #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TonisOrmisson merged 3 commits into from
Sep 8, 2026
Merged

perf: group export metadata reads by dataset #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3222e63
test: bound export metadata reads independently of dictionary size
TonisOrmisson Sep 8, 2026
5e00c74
perf: load export dictionary children in dataset-scoped groups
TonisOrmisson Sep 8, 2026
67534e8
fix: avoid reserved attribute alias in Dolt export query
TonisOrmisson Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 30 additions & 14 deletions src/Sql/CanonicalWideTableExporter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array
if ($variables === []) {
throw $this->invalid('The canonical dataset has no variables.');
}
$labelsByVariable = $this->all('SELECT link.variable_id, label.* FROM variable_value_label_set link JOIN variable ON variable.variable_id = link.variable_id JOIN value_label label ON label.value_label_set_id = link.value_label_set_id WHERE variable.dataset_id = ? ORDER BY link.variable_id, label.ordinal', [$id], 'variable_id');
$attributesByVariable = $this->all('SELECT attr.* FROM variable_attribute attr JOIN variable ON variable.variable_id = attr.variable_id WHERE variable.dataset_id = ? ORDER BY attr.variable_id, attr.attribute_name, attr.array_ordinal', [$id], 'variable_id');
$missingByVariable = $this->all('SELECT rule.* FROM missing_rule rule JOIN variable ON variable.variable_id = rule.variable_id WHERE variable.dataset_id = ? ORDER BY rule.variable_id, rule.ordinal', [$id], 'variable_id');
$typedVariables = [];
$columns = [];
foreach ($variables as $variable) {
Expand All @@ -62,11 +65,11 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array
default => throw $this->invalid('Invalid variable storage kind.'),
};
$labels = [];
foreach ($this->all('SELECT label.* FROM variable_value_label_set link JOIN value_label label ON label.value_label_set_id = link.value_label_set_id WHERE link.variable_id = ? ORDER BY label.ordinal', [$variableId]) as $label) {
foreach ($labelsByVariable[$variableId] ?? [] as $label) {
$labels[] = new ValueLabel($this->value($label['code_kind'], $label['numeric_code'], $label['string_code']), $this->text($label['label']));
}
$attributes = [];
foreach ($this->attributes('SELECT attribute_name, attribute_value FROM variable_attribute WHERE variable_id = ? ORDER BY attribute_name, array_ordinal', $variableId) as $attributeName => $values) {
foreach ($this->attributes($attributesByVariable[$variableId] ?? []) as $attributeName => $values) {
$attributes[] = new VariableAttribute($name, $attributeName, $values);
}
$typedVariables[] = new VariableMetadata(
Expand All @@ -77,7 +80,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array
writeFormat: $this->format($variable, 'write', $type),
label: $variable['variable_label'] === null ? null : $this->text($variable['variable_label']),
valueLabels: new ValueLabelSet($labels, [$name]),
missingValues: $this->missingValues($variableId),
missingValues: $this->missingValues($missingByVariable[$variableId] ?? []),
measure: match ($variable['measurement_level']) {
'nominal' => Measure::NOMINAL,
'ordinal' => Measure::ORDINAL,
Expand Down Expand Up @@ -111,7 +114,7 @@ public function export(string $datasetName, string $targetFormat = 'sav'): array
$rows[] = $values;
}
$attributes = [];
foreach ($this->attributes('SELECT attribute_name, attribute_value FROM dataset_attribute WHERE dataset_id = ? ORDER BY attribute_name, array_ordinal', $id) as $name => $values) {
foreach ($this->attributes($this->all('SELECT attribute_name, attribute_value FROM dataset_attribute WHERE dataset_id = ? ORDER BY attribute_name, array_ordinal', [$id])) as $name => $values) {
$attributes[] = new FileAttribute($name, $values);
}
$weight = $this->all('SELECT variable.source_name FROM dataset_weight_variable weight LEFT JOIN variable ON variable.variable_id = weight.variable_id AND variable.dataset_id = weight.dataset_id WHERE weight.dataset_id = ?', [$id]);
Expand Down Expand Up @@ -167,9 +170,9 @@ private function format(array $variable, string $prefix, VariableType $type): Va
);
}

private function missingValues(string $variableId): MissingValues
/** @param list<array<string, mixed>> $rules */
private function missingValues(array $rules): MissingValues
{
$rules = $this->all('SELECT * FROM missing_rule WHERE variable_id = ? ORDER BY ordinal', [$variableId]);
if ($rules === []) {
return MissingValues::none();
}
Expand Down Expand Up @@ -197,8 +200,9 @@ private function missingValues(string $variableId): MissingValues
private function variableSets(string $datasetId): array
{
$sets = [];
$membersBySet = $this->all('SELECT member.variable_set_id, variable.source_name FROM variable_set_member member JOIN variable_set owner ON owner.variable_set_id = member.variable_set_id LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = owner.dataset_id WHERE owner.dataset_id = ? ORDER BY member.variable_set_id, member.source_ordinal', [$datasetId], 'variable_set_id');
foreach ($this->all('SELECT variable_set_id, set_name FROM variable_set WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]) as $set) {
$members = $this->all('SELECT variable.source_name FROM variable_set_member member LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = ? WHERE member.variable_set_id = ? ORDER BY member.source_ordinal', [$datasetId, $set['variable_set_id']]);
$members = $membersBySet[$set['variable_set_id']] ?? [];
$sets[] = new VariableSet($this->string($set['set_name']), array_map(fn(array $member): string => $this->string($member['source_name']), $members));
}
return $sets;
Expand All @@ -208,8 +212,9 @@ private function variableSets(string $datasetId): array
private function multipleResponseSets(string $datasetId): array
{
$sets = [];
$membersBySet = $this->all('SELECT member.multiple_response_set_id, variable.source_name FROM multiple_response_member member JOIN multiple_response_set owner ON owner.multiple_response_set_id = member.multiple_response_set_id LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = owner.dataset_id WHERE owner.dataset_id = ? ORDER BY member.multiple_response_set_id, member.source_ordinal', [$datasetId], 'multiple_response_set_id');
foreach ($this->all('SELECT * FROM multiple_response_set WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]) as $set) {
$members = $this->all('SELECT variable.source_name FROM multiple_response_member member LEFT JOIN variable ON variable.variable_id = member.variable_id AND variable.dataset_id = ? WHERE member.multiple_response_set_id = ? ORDER BY member.source_ordinal', [$datasetId, $set['multiple_response_set_id']]);
$members = $membersBySet[$set['multiple_response_set_id']] ?? [];
$counted = $set['counted_value_kind'] === null ? null : $this->value($set['counted_value_kind'], $set['counted_numeric_value'], $set['counted_string_value']);
if (is_float($counted)) {
if (floor($counted) !== $counted || $counted >= (float) PHP_INT_MAX || $counted < PHP_INT_MIN) {
Expand Down Expand Up @@ -238,11 +243,14 @@ private function multipleResponseSets(string $datasetId): array
return $sets;
}

/** @return array<string, list<string>> */
private function attributes(string $sql, string $id): array
/**
* @param list<array<string, mixed>> $rows
* @return array<string, list<string>>
*/
private function attributes(array $rows): array
{
$attributes = [];
foreach ($this->all($sql, [$id]) as $row) {
foreach ($rows as $row) {
$attributes[$this->string($row['attribute_name'])][] = $this->text($row['attribute_value']);
}
return $attributes;
Expand Down Expand Up @@ -297,15 +305,23 @@ private function optionalText(mixed $value): ?string

/**
* @param list<mixed> $parameters
* @return list<array<string, mixed>>
* @return ($groupBy is null ? list<array<string, mixed>> : array<array-key, list<array<string, mixed>>>)
*/
private function all(string $sql, array $parameters): array
private function all(string $sql, array $parameters, ?string $groupBy = null): array
{
$statement = $this->pdo->prepare($sql);
if ($statement === false || !$statement->execute($parameters)) {
throw $this->invalid('Could not read the canonical dataset.');
}
return array_values($statement->fetchAll(PDO::FETCH_ASSOC));
$rows = $statement->fetchAll(PDO::FETCH_ASSOC);
if ($groupBy === null) {
return array_values($rows);
}
$grouped = [];
foreach ($rows as $row) {
$grouped[$row[$groupBy]][] = $row;
}
return $grouped;
}

private function quote(string $identifier): string
Expand Down
Loading