From 802517f0f29051b197dccf2a51bde0a67e3e3001 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 1 Jul 2024 09:35:36 +0900 Subject: [PATCH 01/50] Remove PgStat_KindInfo.named_on_disk This field is used to track if a stats kind can use a custom format representation on disk when reading or writing its stats case. On HEAD, this exists for replication slots stats, that need a mapping between an internal index ID and the slot names. named_on_disk is currently used nowhere and the callbacks to_serialized_name and from_serialized_name are in charge of checking if the serialization of the stats data should apply, so let's remove it. Reviewed-by: Andres Freund Discussion: https://postgr.es/m/ZmKVlSX_T5YvIOsd@paquier.xyz (cherry picked from commit b19db55bd687af243dd1edf021ffb166e5270a06) --- src/backend/utils/activity/pgstat.c | 1 - src/include/utils/pgstat_internal.h | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index beaeb669266..28cc7f6d69f 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -314,7 +314,6 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = false, .accessed_across_databases = true, - .named_on_disk = true, .shared_size = sizeof(PgStatShared_ReplSlot), .shared_data_off = offsetof(PgStatShared_ReplSlot, stats), diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 861dd7d7184..a3ac8d0344c 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -212,12 +212,6 @@ typedef struct PgStat_KindInfo */ bool accessed_across_databases:1; - /* - * For variable-numbered stats: Identified on-disk using a name, rather - * than PgStat_HashKey. Probably only needed for replication slot stats. - */ - bool named_on_disk:1; - /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). @@ -258,7 +252,7 @@ typedef struct PgStat_KindInfo void (*reset_timestamp_cb) (PgStatShared_Common *header, TimestampTz ts); /* - * For variable-numbered stats with named_on_disk. Optional. + * For variable-numbered stats. Optional. */ void (*to_serialized_name) (const PgStat_HashKey *key, const PgStatShared_Common *header, NameData *name); From 96046c58dcc4f2e5f374e4207fa7ecbbaea9845a Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 1 Jul 2024 14:26:25 +0900 Subject: [PATCH 02/50] Use pgstat_kind_infos to read fixed shared statistics Shared statistics with a fixed number of objects are read from the stats file in pgstat_read_statsfile() using members of PgStat_ShmemControl and following an order based on their PgStat_Kind value. Instead of being explicit, this commit changes the stats read to iterate over the pgstat_kind_infos array to find the memory locations to read into, based on a new shared_ctl_off in PgStat_KindInfo that can be used to define the position of this stats kind in shared memory. This makes the read logic simpler, and eases the introduction of future improvements aimed at making this area more pluggable for external modules. Original idea suggested by Andres Freund. Author: Tristan Partin Reviewed-by: Andres Freund, Michael Paquier Discussion: https://postgr.es/m/D12SQ7OYCD85.20BUVF3DWU5K7@neon.tech (cherry picked from commit 9004abf6206e815d25b7b763c756cf97b457f3cd) --- src/backend/utils/activity/pgstat.c | 74 ++++++++++++++--------------- src/include/utils/pgstat_internal.h | 6 +++ 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 28cc7f6d69f..711082beee7 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -364,6 +364,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, archiver), + .shared_data_off = offsetof(PgStatShared_Archiver, stats), + .shared_data_len = sizeof(((PgStatShared_Archiver *) 0)->stats), + .reset_all_cb = pgstat_archiver_reset_all_cb, .snapshot_cb = pgstat_archiver_snapshot_cb, }, @@ -373,6 +377,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, bgwriter), + .shared_data_off = offsetof(PgStatShared_BgWriter, stats), + .shared_data_len = sizeof(((PgStatShared_BgWriter *) 0)->stats), + .reset_all_cb = pgstat_bgwriter_reset_all_cb, .snapshot_cb = pgstat_bgwriter_snapshot_cb, }, @@ -382,6 +390,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, checkpointer), + .shared_data_off = offsetof(PgStatShared_Checkpointer, stats), + .shared_data_len = sizeof(((PgStatShared_Checkpointer *) 0)->stats), + .reset_all_cb = pgstat_checkpointer_reset_all_cb, .snapshot_cb = pgstat_checkpointer_snapshot_cb, }, @@ -391,6 +403,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, io), + .shared_data_off = offsetof(PgStatShared_IO, stats), + .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + .reset_all_cb = pgstat_io_reset_all_cb, .snapshot_cb = pgstat_io_snapshot_cb, }, @@ -400,6 +416,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), + .shared_data_off = offsetof(PgStatShared_SLRU, stats), + .shared_data_len = sizeof(((PgStatShared_SLRU *) 0)->stats), + .reset_all_cb = pgstat_slru_reset_all_cb, .snapshot_cb = pgstat_slru_snapshot_cb, }, @@ -409,6 +429,10 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), + .shared_data_off = offsetof(PgStatShared_Wal, stats), + .shared_data_len = sizeof(((PgStatShared_Wal *) 0)->stats), + .reset_all_cb = pgstat_wal_reset_all_cb, .snapshot_cb = pgstat_wal_snapshot_cb, }, @@ -1552,47 +1576,21 @@ pgstat_read_statsfile(void) format_id != PGSTAT_FILE_FORMAT_ID) goto error; - /* - * XXX: The following could now be generalized to just iterate over - * pgstat_kind_infos instead of knowing about the different kinds of - * stats. - */ - - /* - * Read archiver stats struct - */ - if (!read_chunk_s(fpin, &shmem->archiver.stats)) - goto error; - - /* - * Read bgwriter stats struct - */ - if (!read_chunk_s(fpin, &shmem->bgwriter.stats)) - goto error; - - /* - * Read checkpointer stats struct - */ - if (!read_chunk_s(fpin, &shmem->checkpointer.stats)) - goto error; + /* Read various stats structs with fixed number of objects */ + for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + { + char *ptr; + const PgStat_KindInfo *info = pgstat_get_kind_info(kind); - /* - * Read IO stats struct - */ - if (!read_chunk_s(fpin, &shmem->io.stats)) - goto error; + if (!info->fixed_amount) + continue; - /* - * Read SLRU stats struct - */ - if (!read_chunk_s(fpin, &shmem->slru.stats)) - goto error; + Assert(info->shared_ctl_off != 0); - /* - * Read WAL stats struct - */ - if (!read_chunk_s(fpin, &shmem->wal.stats)) - goto error; + ptr = ((char *) shmem) + info->shared_ctl_off + info->shared_data_off; + if (!read_chunk(fpin, ptr, info->shared_data_len)) + goto error; + } /* * We found an existing statistics file. Read it and put all the hash diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index a3ac8d0344c..e6e9151d9d2 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -218,6 +218,12 @@ typedef struct PgStat_KindInfo */ uint32 shared_size; + /* + * The offset of the statistics struct in the containing shared memory + * control structure PgStat_ShmemControl, for fixed-numbered statistics. + */ + uint32 shared_ctl_off; + /* * The offset/size of statistics inside the shared stats entry. Used when * [de-]serializing statistics to / from disk respectively. Separate from From cf5ef0710de8b1ea52c223e02b1170052d6a2cc6 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 3 Jul 2024 13:09:20 +0900 Subject: [PATCH 03/50] Replace hardcoded identifiers of pgstats file by #defines This changes pgstat.c so as the three types of entries that can exist in a pgstats file are not hardcoded anymore, replacing them with descriptively-named macros, when reading and writing stats files: - 'N' for named entries, like replication slot stats. - 'S' for entries identified by a hash. - 'E' for the end-of-file This has come up while working on making this area of the code more pluggable. The format of the stats file is unchanged, hence there is no need to bump PGSTAT_FILE_FORMAT_ID. Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Zmqm9j5EO0I4W8dx@paquier.xyz (cherry picked from commit 9fd02525793f3e06c72ee0da45303c21f7067ccf) --- src/backend/utils/activity/pgstat.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 711082beee7..0ac1efb4756 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -134,6 +134,14 @@ #define PGSTAT_SNAPSHOT_HASH_SIZE 512 +/* --------- + * Identifiers in stats file. + * --------- + */ +#define PGSTAT_FILE_ENTRY_END 'E' /* end of file */ +#define PGSTAT_FILE_ENTRY_NAME 'N' /* stats entry identified by name */ +#define PGSTAT_FILE_ENTRY_HASH 'S' /* stats entry identified by + * PgStat_HashKey */ /* hash table for statistics snapshots entry */ typedef struct PgStat_SnapshotEntry @@ -1464,7 +1472,7 @@ pgstat_write_statsfile(void) if (!kind_info->to_serialized_name) { /* normal stats entry, identified by PgStat_HashKey */ - fputc('S', fpout); + fputc(PGSTAT_FILE_ENTRY_HASH, fpout); write_chunk_s(fpout, &ps->key); } else @@ -1474,7 +1482,7 @@ pgstat_write_statsfile(void) kind_info->to_serialized_name(&ps->key, shstats, &name); - fputc('N', fpout); + fputc(PGSTAT_FILE_ENTRY_NAME, fpout); write_chunk_s(fpout, &ps->key.kind); write_chunk_s(fpout, &name); } @@ -1491,7 +1499,7 @@ pgstat_write_statsfile(void) * pgstat.stat with it. The ferror() check replaces testing for error * after each individual fputc or fwrite (in write_chunk()) above. */ - fputc('E', fpout); + fputc(PGSTAT_FILE_ENTRY_END, fpout); if (ferror(fpout)) { @@ -1602,8 +1610,8 @@ pgstat_read_statsfile(void) switch (t) { - case 'S': - case 'N': + case PGSTAT_FILE_ENTRY_HASH: + case PGSTAT_FILE_ENTRY_NAME: { PgStat_HashKey key; PgStatShared_HashEntry *p; @@ -1611,7 +1619,7 @@ pgstat_read_statsfile(void) CHECK_FOR_INTERRUPTS(); - if (t == 'S') + if (t == PGSTAT_FILE_ENTRY_HASH) { /* normal stats entry, identified by PgStat_HashKey */ if (!read_chunk_s(fpin, &key)) @@ -1677,8 +1685,12 @@ pgstat_read_statsfile(void) break; } - case 'E': - /* check that 'E' actually signals end of file */ + case PGSTAT_FILE_ENTRY_END: + + /* + * check that PGSTAT_FILE_ENTRY_END actually signals end of + * file + */ if (fgetc(fpin) != EOF) goto error; From d1268d1c3233cf7581ec3f59e37356d98f3f4542 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 9 Jul 2024 10:27:12 +0900 Subject: [PATCH 04/50] Use pgstat_kind_infos to write fixed shared statistics This is similar to 9004abf6206e, but this time for the write part of the stats file. The code is changed so as, rather than referring to individual members of PgStat_Snapshot in an order based on their PgStat_Kind value, a loop based on pgstat_kind_infos is used to retrieve the contents to write from the snapshot structure, for a size of PgStat_KindInfo's shared_data_len. This requires the addition to PgStat_KindInfo of an offset to track the location of each fixed-numbered stats in PgStat_Snapshot. This change is useful to make this area of the code more easily pluggable, and reduces the knowledge of specific fixed-numbered kinds in pgstat.c. Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Zot5bxoPYdS7yaoy@paquier.xyz (cherry picked from commit b68b29bc8feca0eb340cb857ff1d28d4099c7878) --- src/backend/utils/activity/pgstat.c | 56 ++++++++++------------------- src/include/utils/pgstat_internal.h | 6 ++++ 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 0ac1efb4756..39e08bd2507 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -372,6 +372,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, archiver), .shared_ctl_off = offsetof(PgStat_ShmemControl, archiver), .shared_data_off = offsetof(PgStatShared_Archiver, stats), .shared_data_len = sizeof(((PgStatShared_Archiver *) 0)->stats), @@ -385,6 +386,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, bgwriter), .shared_ctl_off = offsetof(PgStat_ShmemControl, bgwriter), .shared_data_off = offsetof(PgStatShared_BgWriter, stats), .shared_data_len = sizeof(((PgStatShared_BgWriter *) 0)->stats), @@ -398,6 +400,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, checkpointer), .shared_ctl_off = offsetof(PgStat_ShmemControl, checkpointer), .shared_data_off = offsetof(PgStatShared_Checkpointer, stats), .shared_data_len = sizeof(((PgStatShared_Checkpointer *) 0)->stats), @@ -411,6 +414,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), .shared_data_off = offsetof(PgStatShared_IO, stats), .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), @@ -424,6 +428,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, slru), .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), .shared_data_off = offsetof(PgStatShared_SLRU, stats), .shared_data_len = sizeof(((PgStatShared_SLRU *) 0)->stats), @@ -437,6 +442,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .fixed_amount = true, + .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), .shared_data_off = offsetof(PgStatShared_Wal, stats), .shared_data_len = sizeof(((PgStatShared_Wal *) 0)->stats), @@ -1396,47 +1402,21 @@ pgstat_write_statsfile(void) format_id = PGSTAT_FILE_FORMAT_ID; write_chunk_s(fpout, &format_id); - /* - * XXX: The following could now be generalized to just iterate over - * pgstat_kind_infos instead of knowing about the different kinds of - * stats. - */ - - /* - * Write archiver stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_ARCHIVER); - write_chunk_s(fpout, &pgStatLocal.snapshot.archiver); - - /* - * Write bgwriter stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_BGWRITER); - write_chunk_s(fpout, &pgStatLocal.snapshot.bgwriter); - - /* - * Write checkpointer stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_CHECKPOINTER); - write_chunk_s(fpout, &pgStatLocal.snapshot.checkpointer); + /* Write various stats structs for fixed number of objects */ + for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + { + char *ptr; + const PgStat_KindInfo *info = pgstat_get_kind_info(kind); - /* - * Write IO stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_IO); - write_chunk_s(fpout, &pgStatLocal.snapshot.io); + if (!info->fixed_amount) + continue; - /* - * Write SLRU stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_SLRU); - write_chunk_s(fpout, &pgStatLocal.snapshot.slru); + Assert(info->snapshot_ctl_off != 0); - /* - * Write WAL stats struct - */ - pgstat_build_snapshot_fixed(PGSTAT_KIND_WAL); - write_chunk_s(fpout, &pgStatLocal.snapshot.wal); + pgstat_build_snapshot_fixed(kind); + ptr = ((char *) &pgStatLocal.snapshot) + info->snapshot_ctl_off; + write_chunk(fpout, ptr, info->shared_data_len); + } /* * Walk through the stats entries diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index e6e9151d9d2..38caca48e9b 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -218,6 +218,12 @@ typedef struct PgStat_KindInfo */ uint32 shared_size; + /* + * The offset of the statistics struct in the cached statistics snapshot + * PgStat_Snapshot, for fixed-numbered statistics. + */ + uint32 snapshot_ctl_off; + /* * The offset of the statistics struct in the containing shared memory * control structure PgStat_ShmemControl, for fixed-numbered statistics. From bc0f54f165c41446a3fb0e03ba600824ec4c8d2b Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 11 Jul 2024 09:21:40 +0900 Subject: [PATCH 05/50] Add PgStat_KindInfo.init_shmem_cb This new callback gives fixed-numbered stats the possibility to take actions based on the area of shared memory allocated for them. This removes from pgstat_shmem.c any knowledge specific to the types of fixed-numbered stats, and the initializations happen in their own files. Like b68b29bc8fec, this change is useful to make this area of the code more pluggable, so as custom fixed-numbered stats can take actions after their shared memory area is initialized. Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Zot5bxoPYdS7yaoy@paquier.xyz (cherry picked from commit 21471f18e9d9aa3383314ed8bc9163bc369109ae) --- src/backend/utils/activity/pgstat.c | 6 ++++++ src/backend/utils/activity/pgstat_archiver.c | 8 +++++++ src/backend/utils/activity/pgstat_bgwriter.c | 8 +++++++ .../utils/activity/pgstat_checkpointer.c | 8 +++++++ src/backend/utils/activity/pgstat_io.c | 9 ++++++++ src/backend/utils/activity/pgstat_shmem.c | 21 ++++++++++--------- src/backend/utils/activity/pgstat_slru.c | 8 +++++++ src/backend/utils/activity/pgstat_wal.c | 8 +++++++ src/include/utils/pgstat_internal.h | 13 ++++++++++++ 9 files changed, 79 insertions(+), 10 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 39e08bd2507..fb464909abb 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -377,6 +377,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_Archiver, stats), .shared_data_len = sizeof(((PgStatShared_Archiver *) 0)->stats), + .init_shmem_cb = pgstat_archiver_init_shmem_cb, .reset_all_cb = pgstat_archiver_reset_all_cb, .snapshot_cb = pgstat_archiver_snapshot_cb, }, @@ -391,6 +392,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_BgWriter, stats), .shared_data_len = sizeof(((PgStatShared_BgWriter *) 0)->stats), + .init_shmem_cb = pgstat_bgwriter_init_shmem_cb, .reset_all_cb = pgstat_bgwriter_reset_all_cb, .snapshot_cb = pgstat_bgwriter_snapshot_cb, }, @@ -405,6 +407,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_Checkpointer, stats), .shared_data_len = sizeof(((PgStatShared_Checkpointer *) 0)->stats), + .init_shmem_cb = pgstat_checkpointer_init_shmem_cb, .reset_all_cb = pgstat_checkpointer_reset_all_cb, .snapshot_cb = pgstat_checkpointer_snapshot_cb, }, @@ -419,6 +422,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_IO, stats), .shared_data_len = sizeof(((PgStatShared_IO *) 0)->stats), + .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, .snapshot_cb = pgstat_io_snapshot_cb, }, @@ -433,6 +437,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_SLRU, stats), .shared_data_len = sizeof(((PgStatShared_SLRU *) 0)->stats), + .init_shmem_cb = pgstat_slru_init_shmem_cb, .reset_all_cb = pgstat_slru_reset_all_cb, .snapshot_cb = pgstat_slru_snapshot_cb, }, @@ -447,6 +452,7 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { .shared_data_off = offsetof(PgStatShared_Wal, stats), .shared_data_len = sizeof(((PgStatShared_Wal *) 0)->stats), + .init_shmem_cb = pgstat_wal_init_shmem_cb, .reset_all_cb = pgstat_wal_reset_all_cb, .snapshot_cb = pgstat_wal_snapshot_cb, }, diff --git a/src/backend/utils/activity/pgstat_archiver.c b/src/backend/utils/activity/pgstat_archiver.c index f6af36e60cd..e972b3aeb8b 100644 --- a/src/backend/utils/activity/pgstat_archiver.c +++ b/src/backend/utils/activity/pgstat_archiver.c @@ -62,6 +62,14 @@ pgstat_fetch_stat_archiver(void) return &pgStatLocal.snapshot.archiver; } +void +pgstat_archiver_init_shmem_cb(void *stats) +{ + PgStatShared_Archiver *stats_shmem = (PgStatShared_Archiver *) stats; + + LWLockInitialize(&stats_shmem->lock, LWTRANCHE_PGSTATS_DATA); +} + void pgstat_archiver_reset_all_cb(TimestampTz ts) { diff --git a/src/backend/utils/activity/pgstat_bgwriter.c b/src/backend/utils/activity/pgstat_bgwriter.c index 92be384b0d0..76225097f7b 100644 --- a/src/backend/utils/activity/pgstat_bgwriter.c +++ b/src/backend/utils/activity/pgstat_bgwriter.c @@ -75,6 +75,14 @@ pgstat_fetch_stat_bgwriter(void) return &pgStatLocal.snapshot.bgwriter; } +void +pgstat_bgwriter_init_shmem_cb(void *stats) +{ + PgStatShared_BgWriter *stats_shmem = (PgStatShared_BgWriter *) stats; + + LWLockInitialize(&stats_shmem->lock, LWTRANCHE_PGSTATS_DATA); +} + void pgstat_bgwriter_reset_all_cb(TimestampTz ts) { diff --git a/src/backend/utils/activity/pgstat_checkpointer.c b/src/backend/utils/activity/pgstat_checkpointer.c index 26dec112f6c..6628bcce307 100644 --- a/src/backend/utils/activity/pgstat_checkpointer.c +++ b/src/backend/utils/activity/pgstat_checkpointer.c @@ -83,6 +83,14 @@ pgstat_fetch_stat_checkpointer(void) return &pgStatLocal.snapshot.checkpointer; } +void +pgstat_checkpointer_init_shmem_cb(void *stats) +{ + PgStatShared_Checkpointer *stats_shmem = (PgStatShared_Checkpointer *) stats; + + LWLockInitialize(&stats_shmem->lock, LWTRANCHE_PGSTATS_DATA); +} + void pgstat_checkpointer_reset_all_cb(TimestampTz ts) { diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 4fd4d95291b..6ffcf4771b2 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -237,6 +237,15 @@ pgstat_get_io_object_name(IOObject io_object) pg_unreachable(); } +void +pgstat_io_init_shmem_cb(void *stats) +{ + PgStatShared_IO *stat_shmem = (PgStatShared_IO *) stats; + + for (int i = 0; i < BACKEND_NUM_TYPES; i++) + LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); +} + void pgstat_io_reset_all_cb(TimestampTz ts) { diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index 0225c794774..d1b85ae9a4b 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -195,17 +195,18 @@ StatsShmemInit(void) pg_atomic_init_u64(&ctl->gc_request_count, 1); - /* initialize fixed-numbered stats */ - LWLockInitialize(&ctl->archiver.lock, LWTRANCHE_PGSTATS_DATA); - LWLockInitialize(&ctl->bgwriter.lock, LWTRANCHE_PGSTATS_DATA); - LWLockInitialize(&ctl->checkpointer.lock, LWTRANCHE_PGSTATS_DATA); - LWLockInitialize(&ctl->slru.lock, LWTRANCHE_PGSTATS_DATA); - LWLockInitialize(&ctl->wal.lock, LWTRANCHE_PGSTATS_DATA); - - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - LWLockInitialize(&ctl->io.locks[i], - LWTRANCHE_PGSTATS_DATA); + for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + { + const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + char *ptr; + + if (!kind_info->fixed_amount) + continue; + + ptr = ((char *) ctl) + kind_info->shared_ctl_off; + kind_info->init_shmem_cb(ptr); + } } else { diff --git a/src/backend/utils/activity/pgstat_slru.c b/src/backend/utils/activity/pgstat_slru.c index 1b16740f396..ffe2c168b48 100644 --- a/src/backend/utils/activity/pgstat_slru.c +++ b/src/backend/utils/activity/pgstat_slru.c @@ -192,6 +192,14 @@ pgstat_slru_flush(bool nowait) return false; } +void +pgstat_slru_init_shmem_cb(void *stats) +{ + PgStatShared_SLRU *stats_shmem = (PgStatShared_SLRU *) stats; + + LWLockInitialize(&stats_shmem->lock, LWTRANCHE_PGSTATS_DATA); +} + void pgstat_slru_reset_all_cb(TimestampTz ts) { diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c index 6a81b781357..7d440419c3f 100644 --- a/src/backend/utils/activity/pgstat_wal.c +++ b/src/backend/utils/activity/pgstat_wal.c @@ -163,6 +163,14 @@ pgstat_have_pending_wal(void) PendingWalStats.wal_sync != 0; } +void +pgstat_wal_init_shmem_cb(void *stats) +{ + PgStatShared_Wal *stats_shmem = (PgStatShared_Wal *) stats; + + LWLockInitialize(&stats_shmem->lock, LWTRANCHE_PGSTATS_DATA); +} + void pgstat_wal_reset_all_cb(TimestampTz ts) { diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 38caca48e9b..d6ef47b4e96 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -270,6 +270,13 @@ typedef struct PgStat_KindInfo const PgStatShared_Common *header, NameData *name); bool (*from_serialized_name) (const NameData *name, PgStat_HashKey *key); + /* + * For fixed-numbered statistics: Initialize shared memory state. + * + * "stats" is the pointer to the allocated shared memory area. + */ + void (*init_shmem_cb) (void *stats); + /* * For fixed-numbered statistics: Reset All. */ @@ -578,6 +585,7 @@ extern void pgstat_snapshot_fixed(PgStat_Kind kind); * Functions in pgstat_archiver.c */ +extern void pgstat_archiver_init_shmem_cb(void *stats); extern void pgstat_archiver_reset_all_cb(TimestampTz ts); extern void pgstat_archiver_snapshot_cb(void); @@ -586,6 +594,7 @@ extern void pgstat_archiver_snapshot_cb(void); * Functions in pgstat_bgwriter.c */ +extern void pgstat_bgwriter_init_shmem_cb(void *stats); extern void pgstat_bgwriter_reset_all_cb(TimestampTz ts); extern void pgstat_bgwriter_snapshot_cb(void); @@ -594,6 +603,7 @@ extern void pgstat_bgwriter_snapshot_cb(void); * Functions in pgstat_checkpointer.c */ +extern void pgstat_checkpointer_init_shmem_cb(void *stats); extern void pgstat_checkpointer_reset_all_cb(TimestampTz ts); extern void pgstat_checkpointer_snapshot_cb(void); @@ -624,6 +634,7 @@ extern bool pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); */ extern bool pgstat_flush_io(bool nowait); +extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); @@ -682,6 +693,7 @@ extern PgStatShared_Common *pgstat_init_entry(PgStat_Kind kind, */ extern bool pgstat_slru_flush(bool nowait); +extern void pgstat_slru_init_shmem_cb(void *stats); extern void pgstat_slru_reset_all_cb(TimestampTz ts); extern void pgstat_slru_snapshot_cb(void); @@ -694,6 +706,7 @@ extern bool pgstat_flush_wal(bool nowait); extern void pgstat_init_wal(void); extern bool pgstat_have_pending_wal(void); +extern void pgstat_wal_init_shmem_cb(void *stats); extern void pgstat_wal_reset_all_cb(TimestampTz ts); extern void pgstat_wal_snapshot_cb(void); From 268cb1563df4cc92a9b5cb83f217d884480d327f Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 11 Jul 2024 16:12:04 +0900 Subject: [PATCH 06/50] Add a new 'F' entry type for fixed-numbered stats in pgstats file This new entry type is used for all the fixed-numbered statistics, making possible support for custom pluggable stats. In short, we need to be able to detect more easily if a stats kind exists or not when reading back its data from the pgstats file without a dependency on the order of the entries read. The kind ID of the stats is added to the data written. The data is written in the same fashion as previously, with the fixed-numbered stats first and the dshash entries next. The read part becomes more flexible, loading fixed-numbered stats into shared memory based on the new entry type found. Bump PGSTAT_FILE_FORMAT_ID. Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Zot5bxoPYdS7yaoy@paquier.xyz (cherry picked from commit 9e4664d950c125a1ff7bb29cd1593ca37f8b0c01) --- src/backend/utils/activity/pgstat.c | 51 +++++++++++++++++++---------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index fb464909abb..da685dd4b20 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -139,6 +139,7 @@ * --------- */ #define PGSTAT_FILE_ENTRY_END 'E' /* end of file */ +#define PGSTAT_FILE_ENTRY_FIXED 'F' /* fixed-numbered stats entry */ #define PGSTAT_FILE_ENTRY_NAME 'N' /* stats entry identified by name */ #define PGSTAT_FILE_ENTRY_HASH 'S' /* stats entry identified by * PgStat_HashKey */ @@ -1421,6 +1422,9 @@ pgstat_write_statsfile(void) pgstat_build_snapshot_fixed(kind); ptr = ((char *) &pgStatLocal.snapshot) + info->snapshot_ctl_off; + + fputc(PGSTAT_FILE_ENTRY_FIXED, fpout); + write_chunk_s(fpout, &kind); write_chunk(fpout, ptr, info->shared_data_len); } @@ -1570,25 +1574,9 @@ pgstat_read_statsfile(void) format_id != PGSTAT_FILE_FORMAT_ID) goto error; - /* Read various stats structs with fixed number of objects */ - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) - { - char *ptr; - const PgStat_KindInfo *info = pgstat_get_kind_info(kind); - - if (!info->fixed_amount) - continue; - - Assert(info->shared_ctl_off != 0); - - ptr = ((char *) shmem) + info->shared_ctl_off + info->shared_data_off; - if (!read_chunk(fpin, ptr, info->shared_data_len)) - goto error; - } - /* - * We found an existing statistics file. Read it and put all the hash - * table entries into place. + * We found an existing statistics file. Read it and put all the stats + * data into place. */ for (;;) { @@ -1596,6 +1584,33 @@ pgstat_read_statsfile(void) switch (t) { + case PGSTAT_FILE_ENTRY_FIXED: + { + PgStat_Kind kind; + const PgStat_KindInfo *info; + char *ptr; + + /* entry for fixed-numbered stats */ + if (!read_chunk_s(fpin, &kind)) + goto error; + + if (!pgstat_is_kind_valid(kind)) + goto error; + + info = pgstat_get_kind_info(kind); + + if (!info->fixed_amount) + goto error; + + /* Load back stats into shared memory */ + ptr = ((char *) shmem) + info->shared_ctl_off + + info->shared_data_off; + + if (!read_chunk(fpin, ptr, info->shared_data_len)) + goto error; + + break; + } case PGSTAT_FILE_ENTRY_HASH: case PGSTAT_FILE_ENTRY_NAME: { From 6a7f4fe2181b205c72699bd1d79332a8010fa74b Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 2 Aug 2024 04:49:34 +0900 Subject: [PATCH 07/50] Switch PgStat_Kind from an enum to a uint32 type A follow-up patch is planned to make cumulative statistics pluggable, and using a type is useful in the internal routines used by pgstats as PgStat_Kind may have a value that was not originally in the enum removed here, once made pluggable. While on it, this commit switches pgstat_is_kind_valid() to use PgStat_Kind rather than an int, to be more consistent with its existing callers. Some loops based on the stats kind IDs are switched to use PgStat_Kind rather than int, for consistency with the new time. Author: Michael Paquier Reviewed-by: Dmitry Dolgov, Bertrand Drouvot Discussion: https://postgr.es/m/Zmqm9j5EO0I4W8dx@paquier.xyz Cloudberry: PGSTAT_KIND_RESQUEUE is kept as the sixth variable-numbered kind, so the IDs of the fixed-numbered kinds are shifted by one. (cherry picked from commit 3188a4582a8ce8223fba6e08636d8aaa4cbfd29f) --- src/backend/utils/activity/pgstat.c | 14 ++++---- src/backend/utils/activity/pgstat_shmem.c | 2 +- src/include/pgstat.h | 41 +++++++++++------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index da685dd4b20..73bd2423082 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -189,7 +189,7 @@ static void pgstat_prep_snapshot(void); static void pgstat_build_snapshot(void); static void pgstat_build_snapshot_fixed(PgStat_Kind kind); -static inline bool pgstat_is_kind_valid(int ikind); +static inline bool pgstat_is_kind_valid(PgStat_Kind kind); /* ---------- @@ -1113,7 +1113,7 @@ pgstat_build_snapshot(void) /* * Build snapshot of all fixed-numbered stats. */ - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); @@ -1310,7 +1310,7 @@ pgstat_flush_pending_entries(bool nowait) PgStat_Kind pgstat_get_kind_from_str(char *kind_str) { - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) { if (pg_strcasecmp(kind_str, pgstat_kind_infos[kind].name) == 0) return kind; @@ -1323,9 +1323,9 @@ pgstat_get_kind_from_str(char *kind_str) } static inline bool -pgstat_is_kind_valid(int ikind) +pgstat_is_kind_valid(PgStat_Kind kind) { - return ikind >= PGSTAT_KIND_FIRST_VALID && ikind <= PGSTAT_KIND_LAST; + return kind >= PGSTAT_KIND_FIRST_VALID && kind <= PGSTAT_KIND_LAST; } const PgStat_KindInfo * @@ -1410,7 +1410,7 @@ pgstat_write_statsfile(void) write_chunk_s(fpout, &format_id); /* Write various stats structs for fixed number of objects */ - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) { char *ptr; const PgStat_KindInfo *info = pgstat_get_kind_info(kind); @@ -1729,7 +1729,7 @@ pgstat_reset_after_failure(void) TimestampTz ts = GetCurrentTimestamp(); /* reset fixed-numbered stats */ - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index d1b85ae9a4b..314b99511a6 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -196,7 +196,7 @@ StatsShmemInit(void) pg_atomic_init_u64(&ctl->gc_request_count, 1); /* initialize fixed-numbered stats */ - for (int kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); char *ptr; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..020a3d71cb8 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -34,27 +34,26 @@ #define PG_STAT_TMP_DIR "pg_stat_tmp" /* The types of statistics entries */ -typedef enum PgStat_Kind -{ - /* use 0 for INVALID, to catch zero-initialized data */ - PGSTAT_KIND_INVALID = 0, - - /* stats for variable-numbered objects */ - PGSTAT_KIND_DATABASE, /* database-wide statistics */ - PGSTAT_KIND_RELATION, /* per-table statistics */ - PGSTAT_KIND_FUNCTION, /* per-function statistics */ - PGSTAT_KIND_REPLSLOT, /* per-slot statistics */ - PGSTAT_KIND_SUBSCRIPTION, /* per-subscription statistics */ - PGSTAT_KIND_RESQUEUE, /* per-resource-queue statistics */ - - /* stats for fixed-numbered objects */ - PGSTAT_KIND_ARCHIVER, - PGSTAT_KIND_BGWRITER, - PGSTAT_KIND_CHECKPOINTER, - PGSTAT_KIND_IO, - PGSTAT_KIND_SLRU, - PGSTAT_KIND_WAL, -} PgStat_Kind; +#define PgStat_Kind uint32 + +/* use 0 for INVALID, to catch zero-initialized data */ +#define PGSTAT_KIND_INVALID 0 + +/* stats for variable-numbered objects */ +#define PGSTAT_KIND_DATABASE 1 /* database-wide statistics */ +#define PGSTAT_KIND_RELATION 2 /* per-table statistics */ +#define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ +#define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ +#define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_RESQUEUE 6 /* per-resource-queue statistics */ + +/* stats for fixed-numbered objects */ +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_FIRST_VALID PGSTAT_KIND_DATABASE #define PGSTAT_KIND_LAST PGSTAT_KIND_WAL From d35ad4d5a57bfd880b05e0ecd823a5e7a32264d0 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 30 Jul 2024 15:08:21 +0900 Subject: [PATCH 08/50] Add more debugging information when failing to read pgstats files This is useful to know which part of a stats file is corrupted when reading it, adding to the server logs a WARNING with details about what could not be read before giving up with the remaining data in the file. Author: Michael Paquier Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Zp8o6_cl0KSgsnvS@paquier.xyz (cherry picked from commit ca1ba50fcb6f85ac603aecbc184c7c59886c2a7c) --- src/backend/utils/activity/pgstat.c | 66 +++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 73bd2423082..e219c69cc9c 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1570,9 +1570,18 @@ pgstat_read_statsfile(void) /* * Verify it's of the expected format. */ - if (!read_chunk_s(fpin, &format_id) || - format_id != PGSTAT_FILE_FORMAT_ID) + if (!read_chunk_s(fpin, &format_id)) + { + elog(WARNING, "could not read format ID"); + goto error; + } + + if (format_id != PGSTAT_FILE_FORMAT_ID) + { + elog(WARNING, "found incorrect format ID %d (expected %d)", + format_id, PGSTAT_FILE_FORMAT_ID); goto error; + } /* * We found an existing statistics file. Read it and put all the stats @@ -1592,22 +1601,37 @@ pgstat_read_statsfile(void) /* entry for fixed-numbered stats */ if (!read_chunk_s(fpin, &kind)) + { + elog(WARNING, "could not read stats kind for entry of type %c", t); goto error; + } if (!pgstat_is_kind_valid(kind)) + { + elog(WARNING, "invalid stats kind %d for entry of type %c", + kind, t); goto error; + } info = pgstat_get_kind_info(kind); if (!info->fixed_amount) + { + elog(WARNING, "invalid fixed_amount in stats kind %d for entry of type %c", + kind, t); goto error; + } /* Load back stats into shared memory */ ptr = ((char *) shmem) + info->shared_ctl_off + info->shared_data_off; if (!read_chunk(fpin, ptr, info->shared_data_len)) + { + elog(WARNING, "could not read data of stats kind %d for entry of type %c with size %u", + kind, t, info->shared_data_len); goto error; + } break; } @@ -1624,10 +1648,17 @@ pgstat_read_statsfile(void) { /* normal stats entry, identified by PgStat_HashKey */ if (!read_chunk_s(fpin, &key)) + { + elog(WARNING, "could not read key for entry of type %c", t); goto error; + } if (!pgstat_is_kind_valid(key.kind)) + { + elog(WARNING, "invalid stats kind for entry %d/%u/%u of type %c", + key.kind, key.dboid, key.objoid, t); goto error; + } } else { @@ -1637,22 +1668,41 @@ pgstat_read_statsfile(void) NameData name; if (!read_chunk_s(fpin, &kind)) + { + elog(WARNING, "could not read stats kind for entry of type %c", t); goto error; + } if (!read_chunk_s(fpin, &name)) + { + elog(WARNING, "could not read name of stats kind %d for entry of type %c", + kind, t); goto error; + } if (!pgstat_is_kind_valid(kind)) + { + elog(WARNING, "invalid stats kind %d for entry of type %c", + kind, t); goto error; + } kind_info = pgstat_get_kind_info(kind); if (!kind_info->from_serialized_name) + { + elog(WARNING, "invalid from_serialized_name in stats kind %d for entry of type %c", + kind, t); goto error; + } if (!kind_info->from_serialized_name(&name, &key)) { /* skip over data for entry we don't care about */ if (fseek(fpin, pgstat_get_entry_len(kind), SEEK_CUR) != 0) + { + elog(WARNING, "could not seek \"%s\" of stats kind %d for entry of type %c", + NameStr(name), kind, t); goto error; + } continue; } @@ -1671,8 +1721,8 @@ pgstat_read_statsfile(void) if (found) { dshash_release_lock(pgStatLocal.shared_hash, p); - elog(WARNING, "found duplicate stats entry %d/%u/%u", - key.kind, key.dboid, key.objoid); + elog(WARNING, "found duplicate stats entry %d/%u/%u of type %c", + key.kind, key.dboid, key.objoid, t); goto error; } @@ -1682,7 +1732,11 @@ pgstat_read_statsfile(void) if (!read_chunk(fpin, pgstat_get_entry_data(key.kind, header), pgstat_get_entry_len(key.kind))) + { + elog(WARNING, "could not read data for entry %d/%u/%u of type %c", + key.kind, key.dboid, key.objoid, t); goto error; + } break; } @@ -1693,11 +1747,15 @@ pgstat_read_statsfile(void) * file */ if (fgetc(fpin) != EOF) + { + elog(WARNING, "could not read end-of-file"); goto error; + } goto done; default: + elog(WARNING, "could not read entry of type %c", t); goto error; } } From 575d5f3598f2419ba3a26237e641198c85e586b1 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 4 Aug 2024 03:07:20 +0900 Subject: [PATCH 09/50] Fix incorrect format placeholders in pgstat.c These should have been switched from %d to %u in 3188a4582a8c in the debugging elogs added in ca1ba50fcb6f. PgStat_Kind should never be higher than INT32_MAX, but let's be clean. Issue noticed while hacking more on this area. (cherry picked from commit 028b4b21df26fee67b3ce75c6f14fcfd3c7cf2ee) --- src/backend/utils/activity/pgstat.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index e219c69cc9c..a36287bb9cc 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1608,7 +1608,7 @@ pgstat_read_statsfile(void) if (!pgstat_is_kind_valid(kind)) { - elog(WARNING, "invalid stats kind %d for entry of type %c", + elog(WARNING, "invalid stats kind %u for entry of type %c", kind, t); goto error; } @@ -1617,7 +1617,7 @@ pgstat_read_statsfile(void) if (!info->fixed_amount) { - elog(WARNING, "invalid fixed_amount in stats kind %d for entry of type %c", + elog(WARNING, "invalid fixed_amount in stats kind %u for entry of type %c", kind, t); goto error; } @@ -1628,7 +1628,7 @@ pgstat_read_statsfile(void) if (!read_chunk(fpin, ptr, info->shared_data_len)) { - elog(WARNING, "could not read data of stats kind %d for entry of type %c with size %u", + elog(WARNING, "could not read data of stats kind %u for entry of type %c with size %u", kind, t, info->shared_data_len); goto error; } @@ -1655,7 +1655,7 @@ pgstat_read_statsfile(void) if (!pgstat_is_kind_valid(key.kind)) { - elog(WARNING, "invalid stats kind for entry %d/%u/%u of type %c", + elog(WARNING, "invalid stats kind for entry %u/%u/%u of type %c", key.kind, key.dboid, key.objoid, t); goto error; } @@ -1674,13 +1674,13 @@ pgstat_read_statsfile(void) } if (!read_chunk_s(fpin, &name)) { - elog(WARNING, "could not read name of stats kind %d for entry of type %c", + elog(WARNING, "could not read name of stats kind %u for entry of type %c", kind, t); goto error; } if (!pgstat_is_kind_valid(kind)) { - elog(WARNING, "invalid stats kind %d for entry of type %c", + elog(WARNING, "invalid stats kind %u for entry of type %c", kind, t); goto error; } @@ -1689,7 +1689,7 @@ pgstat_read_statsfile(void) if (!kind_info->from_serialized_name) { - elog(WARNING, "invalid from_serialized_name in stats kind %d for entry of type %c", + elog(WARNING, "invalid from_serialized_name in stats kind %u for entry of type %c", kind, t); goto error; } @@ -1699,7 +1699,7 @@ pgstat_read_statsfile(void) /* skip over data for entry we don't care about */ if (fseek(fpin, pgstat_get_entry_len(kind), SEEK_CUR) != 0) { - elog(WARNING, "could not seek \"%s\" of stats kind %d for entry of type %c", + elog(WARNING, "could not seek \"%s\" of stats kind %u for entry of type %c", NameStr(name), kind, t); goto error; } @@ -1721,7 +1721,7 @@ pgstat_read_statsfile(void) if (found) { dshash_release_lock(pgStatLocal.shared_hash, p); - elog(WARNING, "found duplicate stats entry %d/%u/%u of type %c", + elog(WARNING, "found duplicate stats entry %u/%u/%u of type %c", key.kind, key.dboid, key.objoid, t); goto error; } @@ -1733,7 +1733,7 @@ pgstat_read_statsfile(void) pgstat_get_entry_data(key.kind, header), pgstat_get_entry_len(key.kind))) { - elog(WARNING, "could not read data for entry %d/%u/%u of type %c", + elog(WARNING, "could not read data for entry %u/%u/%u of type %c", key.kind, key.dboid, key.objoid, t); goto error; } From 717ecca9f0f4824aab20c1b2872d4303ed26187e Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sun, 4 Aug 2024 19:41:24 +0900 Subject: [PATCH 10/50] Introduce pluggable APIs for Cumulative Statistics This commit adds support in the backend for $subject, allowing out-of-core extensions to plug their own custom kinds of cumulative statistics. This feature has come up a few times into the lists, and the first, original, suggestion came from Andres Freund, about pg_stat_statements to use the cumulative statistics APIs in shared memory rather than its own less efficient internals. The advantage of this implementation is that this can be extended to any kind of statistics. The stats kinds are divided into two parts: - The in-core "builtin" stats kinds, with designated initializers, able to use IDs up to 128. - The "custom" stats kinds, able to use a range of IDs from 128 to 256 (128 slots available as of this patch), with information saved in TopMemoryContext. This can be made larger, if necessary. There are two types of cumulative statistics in the backend: - For fixed-numbered objects (like WAL, archiver, etc.). These are attached to the snapshot and pgstats shmem control structures for efficiency, and built-in stats kinds still do that to avoid any redirection penalty. The data of custom kinds is stored in a first array in snapshot structure and a second array in the shmem control structure, both indexed by their ID, acting as an equivalent of the builtin stats. - For variable-numbered objects (like tables, functions, etc.). These are stored in a dshash using the stats kind ID in the hash lookup key. Internally, the handling of the builtin stats is unchanged, and both fixed and variabled-numbered objects are supported. Structure definitions for builtin stats kinds are renamed to reflect better the differences with custom kinds. Like custom RMGRs, custom cumulative statistics can only be loaded with shared_preload_libraries at startup, and must allocate a unique ID shared across all the PostgreSQL extension ecosystem with the following wiki page to avoid conflicts: https://wiki.postgresql.org/wiki/CustomCumulativeStats This makes the detection of the stats kinds and their handling when reading and writing stats much easier than, say, allocating IDs for stats kinds from a shared memory counter, that may change the ID used by a stats kind across restarts. When under development, extensions can use PGSTAT_KIND_EXPERIMENTAL. Two examples that can be used as templates for fixed-numbered and variable-numbered stats kinds will be added in some follow-up commits, with tests to provide coverage. Some documentation is added to explain how to use this plugin facility. Author: Michael Paquier Reviewed-by: Dmitry Dolgov, Bertrand Drouvot Discussion: https://postgr.es/m/Zmqm9j5EO0I4W8dx@paquier.xyz (cherry picked from commit 7949d9594582ab49dee221e1db1aa5401ace49d4) --- doc/src/sgml/xfunc.sgml | 57 ++++++ src/backend/utils/activity/pgstat.c | 233 +++++++++++++++++++--- src/backend/utils/activity/pgstat_shmem.c | 31 ++- src/backend/utils/adt/pgstatfuncs.c | 2 +- src/include/pgstat.h | 36 +++- src/include/utils/pgstat_internal.h | 22 +- 6 files changed, 348 insertions(+), 33 deletions(-) diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 9620ea9ae38..f2c97afa1a2 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -3453,6 +3453,63 @@ if (!ptr) + + Custom Cumulative Statistics + + + Is is possible for add-ins written in C-language to use custom types + of cumulative statistics registered in the + Cumulative Statistics System. + + + + First, define a PgStat_KindInfo that includes all + the information related to the custom type registered. For example: + +static const PgStat_KindInfo custom_stats = { + .name = "custom_stats", + .fixed_amount = false, + .shared_size = sizeof(PgStatShared_Custom), + .shared_data_off = offsetof(PgStatShared_Custom, stats), + .shared_data_len = sizeof(((PgStatShared_Custom *) 0)->stats), + .pending_size = sizeof(PgStat_StatCustomEntry), +} + + + Then, each backend that needs to use this custom type needs to register + it with pgstat_register_kind and a unique ID used to + store the entries related to this type of statistics: + +extern PgStat_Kind pgstat_add_kind(PgStat_Kind kind, + const PgStat_KindInfo *kind_info); + + While developing a new extension, use + PGSTAT_KIND_EXPERIMENTAL for + kind. When you are ready to release the extension + to users, reserve a kind ID at the + + Custom Cumulative Statistics page. + + + + The details of the API for PgStat_KindInfo can + be found in src/include/utils/pgstat_internal.h. + + + + The type of statistics registered is associated with a name and a unique + ID shared across the server in shared memory. Each backend using a + custom type of statistics maintains a local cache storing the information + of each custom PgStat_KindInfo. + + + + Place the extension module implementing the custom cumulative statistics + type in so that it will + be loaded early during PostgreSQL startup. + + + Using C++ for Extensibility diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index a36287bb9cc..2588becd644 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -49,8 +49,16 @@ * pgStatPending list. Pending statistics updates are flushed out by * pgstat_report_stat(). * + * It is possible for external modules to define custom statistics kinds, + * that can use the same properties as any built-in stats kinds. Each custom + * stats kind needs to assign a unique ID to ensure that it does not overlap + * with other extensions. In order to reserve a unique stats kind ID, refer + * to https://wiki.postgresql.org/wiki/CustomCumulativeStats. + * * The behavior of different kinds of statistics is determined by the kind's - * entry in pgstat_kind_infos, see PgStat_KindInfo for details. + * entry in pgstat_kind_builtin_infos for all the built-in statistics kinds + * defined, and pgstat_kind_custom_infos for custom kinds registered at + * startup by pgstat_register_kind(). See PgStat_KindInfo for details. * * The consistency of read accesses to statistics can be configured using the * stats_fetch_consistency GUC (see config.sgml and monitoring.sgml for the @@ -181,6 +189,8 @@ typedef struct PgStat_SnapshotEntry static void pgstat_write_statsfile(void); static void pgstat_read_statsfile(void); +static void pgstat_init_snapshot_fixed(void); + static void pgstat_reset_after_failure(void); static bool pgstat_flush_pending_entries(bool nowait); @@ -258,7 +268,7 @@ static bool pgstat_is_shutdown = false; /* - * The different kinds of statistics. + * The different kinds of built-in statistics. * * If reasonably possible, handling specific to one kind of stats should go * through this abstraction, rather than making more of pgstat.c aware. @@ -270,7 +280,7 @@ static bool pgstat_is_shutdown = false; * seem to be a great way of doing that, given the split across multiple * files. */ -static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { +static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] = { /* stats kinds for variable-numbered objects */ @@ -459,6 +469,15 @@ static const PgStat_KindInfo pgstat_kind_infos[PGSTAT_NUM_KINDS] = { }, }; +/* + * Information about custom statistics kinds. + * + * These are saved in a different array than the built-in kinds to save + * in clarity with the initializations. + * + * Indexed by PGSTAT_KIND_CUSTOM_MIN, of size PGSTAT_KIND_CUSTOM_SIZE. + */ +static const PgStat_KindInfo **pgstat_kind_custom_infos = NULL; /* ------------------------------------------------------------ * Functions managing the state of the stats system for all backends. @@ -609,6 +628,8 @@ pgstat_initialize(void) pgstat_init_wal(); + pgstat_init_snapshot_fixed(); + /* Set up a process-exit hook to clean up */ before_shmem_exit(pgstat_shutdown_hook, 0); @@ -851,6 +872,8 @@ pgstat_clear_snapshot(void) memset(&pgStatLocal.snapshot.fixed_valid, 0, sizeof(pgStatLocal.snapshot.fixed_valid)); + memset(&pgStatLocal.snapshot.custom_valid, 0, + sizeof(pgStatLocal.snapshot.custom_valid)); pgStatLocal.snapshot.stats = NULL; pgStatLocal.snapshot.mode = PGSTAT_FETCH_CONSISTENCY_NONE; @@ -1017,7 +1040,29 @@ pgstat_snapshot_fixed(PgStat_Kind kind) else pgstat_build_snapshot_fixed(kind); - Assert(pgStatLocal.snapshot.fixed_valid[kind]); + if (pgstat_is_kind_builtin(kind)) + Assert(pgStatLocal.snapshot.fixed_valid[kind]); + else if (pgstat_is_kind_custom(kind)) + Assert(pgStatLocal.snapshot.custom_valid[kind - PGSTAT_KIND_CUSTOM_MIN]); +} + +static void +pgstat_init_snapshot_fixed(void) +{ + /* + * Initialize fixed-numbered statistics data in snapshots, only for custom + * stats kinds. + */ + for (PgStat_Kind kind = PGSTAT_KIND_CUSTOM_MIN; kind <= PGSTAT_KIND_CUSTOM_MAX; kind++) + { + const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + + if (!kind_info || !kind_info->fixed_amount) + continue; + + pgStatLocal.snapshot.custom_data[kind - PGSTAT_KIND_CUSTOM_MIN] = + MemoryContextAlloc(TopMemoryContext, kind_info->shared_data_len); + } } static void @@ -1113,10 +1158,12 @@ pgstat_build_snapshot(void) /* * Build snapshot of all fixed-numbered stats. */ - for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + if (!kind_info) + continue; if (!kind_info->fixed_amount) { Assert(kind_info->snapshot_cb == NULL); @@ -1133,6 +1180,20 @@ static void pgstat_build_snapshot_fixed(PgStat_Kind kind) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + int idx; + bool *valid; + + /* Position in fixed_valid or custom_valid */ + if (pgstat_is_kind_builtin(kind)) + { + idx = kind; + valid = pgStatLocal.snapshot.fixed_valid; + } + else + { + idx = kind - PGSTAT_KIND_CUSTOM_MIN; + valid = pgStatLocal.snapshot.custom_valid; + } Assert(kind_info->fixed_amount); Assert(kind_info->snapshot_cb != NULL); @@ -1140,21 +1201,21 @@ pgstat_build_snapshot_fixed(PgStat_Kind kind) if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_NONE) { /* rebuild every time */ - pgStatLocal.snapshot.fixed_valid[kind] = false; + valid[idx] = false; } - else if (pgStatLocal.snapshot.fixed_valid[kind]) + else if (valid[idx]) { /* in snapshot mode we shouldn't get called again */ Assert(pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_CACHE); return; } - Assert(!pgStatLocal.snapshot.fixed_valid[kind]); + Assert(!valid[idx]); kind_info->snapshot_cb(); - Assert(!pgStatLocal.snapshot.fixed_valid[kind]); - pgStatLocal.snapshot.fixed_valid[kind] = true; + Assert(!valid[idx]); + valid[idx] = true; } @@ -1310,30 +1371,131 @@ pgstat_flush_pending_entries(bool nowait) PgStat_Kind pgstat_get_kind_from_str(char *kind_str) { - for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_BUILTIN_MIN; kind <= PGSTAT_KIND_BUILTIN_MAX; kind++) { - if (pg_strcasecmp(kind_str, pgstat_kind_infos[kind].name) == 0) + if (pg_strcasecmp(kind_str, pgstat_kind_builtin_infos[kind].name) == 0) return kind; } + /* Check the custom set of cumulative stats */ + if (pgstat_kind_custom_infos) + { + for (PgStat_Kind kind = PGSTAT_KIND_CUSTOM_MIN; kind <= PGSTAT_KIND_CUSTOM_MAX; kind++) + { + uint32 idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + if (pgstat_kind_custom_infos[idx] && + pg_strcasecmp(kind_str, pgstat_kind_custom_infos[idx]->name) == 0) + return kind; + } + } + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid statistics kind: \"%s\"", kind_str))); - return PGSTAT_KIND_DATABASE; /* avoid compiler warnings */ + return PGSTAT_KIND_INVALID; /* avoid compiler warnings */ } static inline bool pgstat_is_kind_valid(PgStat_Kind kind) { - return kind >= PGSTAT_KIND_FIRST_VALID && kind <= PGSTAT_KIND_LAST; + return pgstat_is_kind_builtin(kind) || pgstat_is_kind_custom(kind); } const PgStat_KindInfo * pgstat_get_kind_info(PgStat_Kind kind) { - Assert(pgstat_is_kind_valid(kind)); + if (pgstat_is_kind_builtin(kind)) + return &pgstat_kind_builtin_infos[kind]; + + if (pgstat_is_kind_custom(kind)) + { + uint32 idx = kind - PGSTAT_KIND_CUSTOM_MIN; - return &pgstat_kind_infos[kind]; + if (pgstat_kind_custom_infos == NULL || + pgstat_kind_custom_infos[idx] == NULL) + return NULL; + return pgstat_kind_custom_infos[idx]; + } + + return NULL; +} + +/* + * Register a new stats kind. + * + * PgStat_Kinds must be globally unique across all extensions. Refer + * to https://wiki.postgresql.org/wiki/CustomCumulativeStats to reserve a + * unique ID for your extension, to avoid conflicts with other extension + * developers. During development, use PGSTAT_KIND_EXPERIMENTAL to avoid + * needlessly reserving a new ID. + */ +void +pgstat_register_kind(PgStat_Kind kind, const PgStat_KindInfo *kind_info) +{ + uint32 idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + if (kind_info->name == NULL || strlen(kind_info->name) == 0) + ereport(ERROR, + (errmsg("custom cumulative statistics name is invalid"), + errhint("Provide a non-empty name for the custom cumulative statistics."))); + + if (!pgstat_is_kind_custom(kind)) + ereport(ERROR, (errmsg("custom cumulative statistics ID %u is out of range", kind), + errhint("Provide a custom cumulative statistics ID between %u and %u.", + PGSTAT_KIND_CUSTOM_MIN, PGSTAT_KIND_CUSTOM_MAX))); + + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), + errdetail("Custom cumulative statistics must be registered while initializing modules in \"shared_preload_libraries\"."))); + + /* + * Check some data for fixed-numbered stats. + */ + if (kind_info->fixed_amount) + { + if (kind_info->shared_size == 0) + ereport(ERROR, + (errmsg("custom cumulative statistics property is invalid"), + errhint("Custom cumulative statistics require a shared memory size for fixed-numbered objects."))); + } + + /* + * If pgstat_kind_custom_infos is not available yet, allocate it. + */ + if (pgstat_kind_custom_infos == NULL) + { + pgstat_kind_custom_infos = (const PgStat_KindInfo **) + MemoryContextAllocZero(TopMemoryContext, + sizeof(PgStat_KindInfo *) * PGSTAT_KIND_CUSTOM_SIZE); + } + + if (pgstat_kind_custom_infos[idx] != NULL && + pgstat_kind_custom_infos[idx]->name != NULL) + ereport(ERROR, + (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), + errdetail("Custom cumulative statistics \"%s\" already registered with the same ID.", + pgstat_kind_custom_infos[idx]->name))); + + /* check for existing custom stats with the same name */ + for (PgStat_Kind existing_kind = PGSTAT_KIND_CUSTOM_MIN; existing_kind <= PGSTAT_KIND_CUSTOM_MAX; existing_kind++) + { + uint32 existing_idx = existing_kind - PGSTAT_KIND_CUSTOM_MIN; + + if (pgstat_kind_custom_infos[existing_idx] == NULL) + continue; + if (!pg_strcasecmp(pgstat_kind_custom_infos[existing_idx]->name, kind_info->name)) + ereport(ERROR, + (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), + errdetail("Existing cumulative statistics with ID %u has the same name.", existing_kind))); + } + + /* Register it */ + pgstat_kind_custom_infos[idx] = kind_info; + ereport(LOG, + (errmsg("registered custom cumulative statistics \"%s\" with ID %u", + kind_info->name, kind))); } /* @@ -1410,18 +1572,22 @@ pgstat_write_statsfile(void) write_chunk_s(fpout, &format_id); /* Write various stats structs for fixed number of objects */ - for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) { char *ptr; const PgStat_KindInfo *info = pgstat_get_kind_info(kind); - if (!info->fixed_amount) + if (!info || !info->fixed_amount) continue; - Assert(info->snapshot_ctl_off != 0); + if (pgstat_is_kind_builtin(kind)) + Assert(info->snapshot_ctl_off != 0); pgstat_build_snapshot_fixed(kind); - ptr = ((char *) &pgStatLocal.snapshot) + info->snapshot_ctl_off; + if (pgstat_is_kind_builtin(kind)) + ptr = ((char *) &pgStatLocal.snapshot) + info->snapshot_ctl_off; + else + ptr = pgStatLocal.snapshot.custom_data[kind - PGSTAT_KIND_CUSTOM_MIN]; fputc(PGSTAT_FILE_ENTRY_FIXED, fpout); write_chunk_s(fpout, &kind); @@ -1452,6 +1618,17 @@ pgstat_write_statsfile(void) if (ps->dropped) continue; + /* + * This discards data related to custom stats kinds that are unknown + * to this process. + */ + if (!pgstat_is_kind_valid(ps->key.kind)) + { + elog(WARNING, "found unknown stats entry %u/%u/%u", + ps->key.kind, ps->key.dboid, ps->key.objoid); + continue; + } + shstats = (PgStatShared_Common *) dsa_get_address(pgStatLocal.dsa, ps->body); kind_info = pgstat_get_kind_info(ps->key.kind); @@ -1623,8 +1800,16 @@ pgstat_read_statsfile(void) } /* Load back stats into shared memory */ - ptr = ((char *) shmem) + info->shared_ctl_off + - info->shared_data_off; + if (pgstat_is_kind_builtin(kind)) + ptr = ((char *) shmem) + info->shared_ctl_off + + info->shared_data_off; + else + { + int idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + ptr = ((char *) shmem->custom_data[idx]) + + info->shared_data_off; + } if (!read_chunk(fpin, ptr, info->shared_data_len)) { @@ -1787,11 +1972,11 @@ pgstat_reset_after_failure(void) TimestampTz ts = GetCurrentTimestamp(); /* reset fixed-numbered stats */ - for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); - if (!kind_info->fixed_amount) + if (!kind_info || !kind_info->fixed_amount) continue; kind_info->reset_all_cb(ts); diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index 314b99511a6..659e5f0fdb1 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -130,6 +130,21 @@ StatsShmemSize(void) sz = MAXALIGN(sizeof(PgStat_ShmemControl)); sz = add_size(sz, pgstat_dsa_init_size()); + /* Add shared memory for all the custom fixed-numbered statistics */ + for (PgStat_Kind kind = PGSTAT_KIND_CUSTOM_MIN; kind <= PGSTAT_KIND_CUSTOM_MAX; kind++) + { + const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); + + if (!kind_info) + continue; + if (!kind_info->fixed_amount) + continue; + + Assert(kind_info->shared_size != 0); + + sz += MAXALIGN(kind_info->shared_size); + } + return sz; } @@ -196,15 +211,25 @@ StatsShmemInit(void) pg_atomic_init_u64(&ctl->gc_request_count, 1); /* initialize fixed-numbered stats */ - for (PgStat_Kind kind = PGSTAT_KIND_FIRST_VALID; kind <= PGSTAT_KIND_LAST; kind++) + for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); char *ptr; - if (!kind_info->fixed_amount) + if (!kind_info || !kind_info->fixed_amount) continue; - ptr = ((char *) ctl) + kind_info->shared_ctl_off; + if (pgstat_is_kind_builtin(kind)) + ptr = ((char *) ctl) + kind_info->shared_ctl_off; + else + { + int idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + Assert(kind_info->shared_size != 0); + ctl->custom_data[idx] = ShmemAlloc(kind_info->shared_size); + ptr = ctl->custom_data[idx]; + } + kind_info->init_shmem_cb(ptr); } } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 61c69b3ca4e..777ac00a340 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1810,7 +1810,7 @@ pg_stat_reset(PG_FUNCTION_ARGS) * Reset some shared cluster-wide counters * * When adding a new reset target, ideally the name should match that in - * pgstat_kind_infos, if relevant. + * pgstat_kind_builtin_infos, if relevant. */ Datum pg_stat_reset_shared(PG_FUNCTION_ARGS) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 020a3d71cb8..37bb4c41f54 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -36,6 +36,10 @@ /* The types of statistics entries */ #define PgStat_Kind uint32 +/* Range of IDs allowed, for built-in and custom kinds */ +#define PGSTAT_KIND_MIN 1 /* Minimum ID allowed */ +#define PGSTAT_KIND_MAX 256 /* Maximum ID allowed */ + /* use 0 for INVALID, to catch zero-initialized data */ #define PGSTAT_KIND_INVALID 0 @@ -55,9 +59,35 @@ #define PGSTAT_KIND_SLRU 11 #define PGSTAT_KIND_WAL 12 -#define PGSTAT_KIND_FIRST_VALID PGSTAT_KIND_DATABASE -#define PGSTAT_KIND_LAST PGSTAT_KIND_WAL -#define PGSTAT_NUM_KINDS (PGSTAT_KIND_LAST + 1) +#define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE +#define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL +#define PGSTAT_KIND_BUILTIN_SIZE (PGSTAT_KIND_BUILTIN_MAX + 1) + +/* Custom stats kinds */ + +/* Range of IDs allowed for custom stats kinds */ +#define PGSTAT_KIND_CUSTOM_MIN 128 +#define PGSTAT_KIND_CUSTOM_MAX PGSTAT_KIND_MAX +#define PGSTAT_KIND_CUSTOM_SIZE (PGSTAT_KIND_CUSTOM_MAX - PGSTAT_KIND_CUSTOM_MIN + 1) + +/* + * PgStat_Kind to use for extensions that require an ID, but are still in + * development and have not reserved their own unique kind ID yet. See: + * https://wiki.postgresql.org/wiki/CustomCumulativeStats + */ +#define PGSTAT_KIND_EXPERIMENTAL 128 + +static inline bool +pgstat_is_kind_builtin(PgStat_Kind kind) +{ + return kind >= PGSTAT_KIND_BUILTIN_MIN && kind <= PGSTAT_KIND_BUILTIN_MAX; +} + +static inline bool +pgstat_is_kind_custom(PgStat_Kind kind) +{ + return kind >= PGSTAT_KIND_CUSTOM_MIN && kind <= PGSTAT_KIND_CUSTOM_MAX; +} /* Values for track_functions GUC variable --- order is significant! */ typedef enum TrackFunctionsLevel diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index d6ef47b4e96..4bb4d052366 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -214,7 +214,8 @@ typedef struct PgStat_KindInfo /* * The size of an entry in the shared stats hash table (pointed to by - * PgStatShared_HashEntry->body). + * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is + * the size of an entry in PgStat_ShmemControl->custom_data. */ uint32 shared_size; @@ -496,6 +497,13 @@ typedef struct PgStat_ShmemControl PgStatShared_IO io; PgStatShared_SLRU slru; PgStatShared_Wal wal; + + /* + * Custom stats data with fixed-numbered objects, indexed by (PgStat_Kind + * - PGSTAT_KIND_CUSTOM_MIN). + */ + void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + } PgStat_ShmemControl; @@ -509,7 +517,7 @@ typedef struct PgStat_Snapshot /* time at which snapshot was taken */ TimestampTz snapshot_timestamp; - bool fixed_valid[PGSTAT_NUM_KINDS]; + bool fixed_valid[PGSTAT_KIND_BUILTIN_SIZE]; PgStat_ArchiverStats archiver; @@ -523,6 +531,14 @@ typedef struct PgStat_Snapshot PgStat_WalStats wal; + /* + * Data in snapshot for custom fixed-numbered statistics, indexed by + * (PgStat_Kind - PGSTAT_KIND_CUSTOM_MIN). Each entry is allocated in + * TopMemoryContext, for a size of PgStat_KindInfo->shared_data_len. + */ + bool custom_valid[PGSTAT_KIND_CUSTOM_SIZE]; + void *custom_data[PGSTAT_KIND_CUSTOM_SIZE]; + /* to free snapshot in bulk */ MemoryContext context; struct pgstat_snapshot_hash *stats; @@ -566,6 +582,8 @@ static inline void *pgstat_get_entry_data(PgStat_Kind kind, PgStatShared_Common */ extern const PgStat_KindInfo *pgstat_get_kind_info(PgStat_Kind kind); +extern void pgstat_register_kind(PgStat_Kind kind, + const PgStat_KindInfo *kind_info); #ifdef USE_ASSERT_CHECKING extern void pgstat_assert_is_up(void); From 61cacda26474cd1776d30b4d7dd54847c5761512 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 5 Aug 2024 11:43:33 +0900 Subject: [PATCH 11/50] Add helper routines to retrieve data for custom fixed-numbered pgstats This is useful for extensions to get snapshot and shmem data for custom cumulative statistics when these have a fixed number of objects, so as these do not need to know about the snapshot internals, aka pgStatLocal. An upcoming commit introducing an example template for custom cumulative stats with fixed-numbered objects will make use of these. I have noticed that this is useful for extension developers while hacking my own example, actually. Author: Michael Paquier Reviewed-by: Dmitry Dolgov, Bertrand Drouvot Discussion: https://postgr.es/m/Zmqm9j5EO0I4W8dx@paquier.xyz (cherry picked from commit 2eff9e678d350882f7a4b6ec6361927c9093051c) --- src/include/utils/pgstat_internal.h | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 4bb4d052366..74a50b74190 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -575,6 +575,8 @@ static inline int pgstat_cmp_hash_key(const void *a, const void *b, size_t size, static inline uint32 pgstat_hash_hash_key(const void *d, size_t size, void *arg); static inline size_t pgstat_get_entry_len(PgStat_Kind kind); static inline void *pgstat_get_entry_data(PgStat_Kind kind, PgStatShared_Common *entry); +static inline void *pgstat_get_custom_shmem_data(PgStat_Kind kind); +static inline void *pgstat_get_custom_snapshot_data(PgStat_Kind kind); /* @@ -899,4 +901,34 @@ pgstat_get_entry_data(PgStat_Kind kind, PgStatShared_Common *entry) return ((char *) (entry)) + off; } +/* + * Returns a pointer to the shared memory area of custom stats for + * fixed-numbered statistics. + */ +static inline void * +pgstat_get_custom_shmem_data(PgStat_Kind kind) +{ + int idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + Assert(pgstat_is_kind_custom(kind)); + Assert(pgstat_get_kind_info(kind)->fixed_amount); + + return pgStatLocal.shmem->custom_data[idx]; +} + +/* + * Returns a pointer to the portion of custom data for fixed-numbered + * statistics in the current snapshot. + */ +static inline void * +pgstat_get_custom_snapshot_data(PgStat_Kind kind) +{ + int idx = kind - PGSTAT_KIND_CUSTOM_MIN; + + Assert(pgstat_is_kind_custom(kind)); + Assert(pgstat_get_kind_info(kind)->fixed_amount); + + return pgStatLocal.snapshot.custom_data[idx]; +} + #endif /* PGSTAT_INTERNAL_H */ From b899a87b336f9f4382a06d3777be220cb9953723 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 5 Sep 2024 14:36:57 +0900 Subject: [PATCH 12/50] Fix two NULL pointer dereferences when reading custom pgstats from file There were two spots in pgstat_read_statsfile() where is was possible to finish with a null-pointer-dereference crash for custom pgstats kinds: - When reading stats for a fixed-numbered stats entry. - When reading a variable stats entry with name serialization. For both cases, these issues were reachable by starting a server after changing shared_preload_libraries so as the stats written previously could not be loaded. The code is changed so as the stats are ignored in this case, like the other code paths doing similar sanity checks. Two WARNINGs are added to be able to debug these issues. A test is added for the case of fixed-numbered stats with the module injection_points. Oversights in 7949d9594582, spotted while looking at a different report. Discussion: https://postgr.es/m/Ztj0Jftsn4xXuXtl@paquier.xyz Cloudberry: the injection_points test module does not exist here (it appeared in PostgreSQL 17), so its part of the change is left out. (cherry picked from commit 341e9a05e7b4c4fb2a1e539a5073dfd0e5b46735) --- src/backend/utils/activity/pgstat.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 2588becd644..de65c662d3f 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -1791,6 +1791,12 @@ pgstat_read_statsfile(void) } info = pgstat_get_kind_info(kind); + if (!info) + { + elog(WARNING, "could not find information of kind %u for entry of type %c", + kind, t); + goto error; + } if (!info->fixed_amount) { @@ -1871,6 +1877,12 @@ pgstat_read_statsfile(void) } kind_info = pgstat_get_kind_info(kind); + if (!kind_info) + { + elog(WARNING, "could not find information of kind %u for entry of type %c", + kind, t); + goto error; + } if (!kind_info->from_serialized_name) { From 64d6b8f84fa63817588a6bbf559ee15a9598f6bc Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 22 Nov 2024 10:12:26 +0900 Subject: [PATCH 13/50] Add write_to_file to PgStat_KindInfo for pgstats kinds This new field controls if entries of a stats kind should be written or not to the on-disk pgstats file when shutting down an instance. This affects both fixed and variable-numbered kinds. This is useful for custom statistics by itself, and a patch is under discussion to add a new builtin stats kind where the write of the stats is not necessary. All the built-in stats kinds, as well as the two custom stats kinds in the test module injection_points, set this flag to "true" for now, so as stats entries are written to the on-disk pgstats file. Author: Bertrand Drouvot Reviewed-by: Nazir Bilal Yavuz Discussion: https://postgr.es/m/Zz7T47nHwYgeYwOe@ip-10-97-1-34.eu-west-3.compute.internal Cloudberry: the injection_points test module does not exist here (it appeared in PostgreSQL 17), so its part of the change is left out. (cherry picked from commit c06e71d1acaf490660ed4ae5c28b3079628746b0) --- src/backend/utils/activity/pgstat.c | 22 +++++++++++++++++++++- src/include/utils/pgstat_internal.h | 3 +++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index de65c662d3f..6db2f524ded 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -12,7 +12,8 @@ * Statistics are loaded from the filesystem during startup (by the startup * process), unless preceded by a crash, in which case all stats are * discarded. They are written out by the checkpointer process just before - * shutting down, except when shutting down in immediate mode. + * shutting down (if the stats kind allows it), except when shutting down in + * immediate mode. * * Fixed-numbered stats are stored in plain (non-dynamic) shared memory. * @@ -288,6 +289,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "database", .fixed_amount = false, + .write_to_file = true, /* so pg_stat_database entries can be seen in all databases */ .accessed_across_databases = true, @@ -304,6 +306,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "relation", .fixed_amount = false, + .write_to_file = true, .shared_size = sizeof(PgStatShared_Relation), .shared_data_off = offsetof(PgStatShared_Relation, stats), @@ -318,6 +321,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "function", .fixed_amount = false, + .write_to_file = true, .shared_size = sizeof(PgStatShared_Function), .shared_data_off = offsetof(PgStatShared_Function, stats), @@ -331,6 +335,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "replslot", .fixed_amount = false, + .write_to_file = true, .accessed_across_databases = true, @@ -347,6 +352,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "subscription", .fixed_amount = false, + .write_to_file = true, /* so pg_stat_subscription_stats entries can be seen in all databases */ .accessed_across_databases = true, @@ -382,6 +388,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "archiver", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, archiver), .shared_ctl_off = offsetof(PgStat_ShmemControl, archiver), @@ -397,6 +404,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "bgwriter", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, bgwriter), .shared_ctl_off = offsetof(PgStat_ShmemControl, bgwriter), @@ -412,6 +420,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "checkpointer", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, checkpointer), .shared_ctl_off = offsetof(PgStat_ShmemControl, checkpointer), @@ -427,6 +436,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "io", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, io), .shared_ctl_off = offsetof(PgStat_ShmemControl, io), @@ -442,6 +452,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "slru", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, slru), .shared_ctl_off = offsetof(PgStat_ShmemControl, slru), @@ -457,6 +468,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "wal", .fixed_amount = true, + .write_to_file = true, .snapshot_ctl_off = offsetof(PgStat_Snapshot, wal), .shared_ctl_off = offsetof(PgStat_ShmemControl, wal), @@ -1583,6 +1595,10 @@ pgstat_write_statsfile(void) if (pgstat_is_kind_builtin(kind)) Assert(info->snapshot_ctl_off != 0); + /* skip if no need to write to file */ + if (!info->write_to_file) + continue; + pgstat_build_snapshot_fixed(kind); if (pgstat_is_kind_builtin(kind)) ptr = ((char *) &pgStatLocal.snapshot) + info->snapshot_ctl_off; @@ -1636,6 +1652,10 @@ pgstat_write_statsfile(void) /* if not dropped the valid-entry refcount should exist */ Assert(pg_atomic_read_u32(&ps->refcount) > 0); + /* skip if no need to write to file */ + if (!kind_info->write_to_file) + continue; + if (!kind_info->to_serialized_name) { /* normal stats entry, identified by PgStat_HashKey */ diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 74a50b74190..3f1fae0b7c2 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -212,6 +212,9 @@ typedef struct PgStat_KindInfo */ bool accessed_across_databases:1; + /* Should stats be written to the on-disk stats file? */ + bool write_to_file:1; + /* * The size of an entry in the shared stats hash table (pointed to by * PgStatShared_HashEntry->body). For fixed-numbered statistics, this is From a062ad8ba6e2e80ef06c22db4458eb4a40863bd2 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 14 Jan 2025 12:43:07 +0900 Subject: [PATCH 14/50] Move information about pgstats kinds into its own header pgstat_kind.h This includes all the definitions for the various PGSTAT_KIND_* values, the range allowed for custom stats kinds and some macros related all that. One use-case behind this split is the possibility to use this information for frontend tools, without having to rely on pgstat.h and a backend footprint. Author: Michael Paquier Reviewed-by: Bertrand Drouvot Discussion: https://postgr.es/m/Z24fyb3ipXKR38oS@paquier.xyz Cloudberry: there are no backend statistics here (PostgreSQL 18), so the sixth variable-numbered kind is PGSTAT_KIND_RESQUEUE instead of PGSTAT_KIND_BACKEND. (cherry picked from commit d35ea27e51c05cbe3575d50a6b99d64f20a3a742) --- src/include/pgstat.h | 57 +------------------------- src/include/utils/pgstat_kind.h | 72 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 56 deletions(-) create mode 100644 src/include/utils/pgstat_kind.h diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 37bb4c41f54..34262d89257 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -16,6 +16,7 @@ #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ +#include "utils/pgstat_kind.h" #include "utils/relcache.h" #include "utils/wait_event.h" /* for backward compatibility */ @@ -33,62 +34,6 @@ /* Default directory to store temporary statistics data in */ #define PG_STAT_TMP_DIR "pg_stat_tmp" -/* The types of statistics entries */ -#define PgStat_Kind uint32 - -/* Range of IDs allowed, for built-in and custom kinds */ -#define PGSTAT_KIND_MIN 1 /* Minimum ID allowed */ -#define PGSTAT_KIND_MAX 256 /* Maximum ID allowed */ - -/* use 0 for INVALID, to catch zero-initialized data */ -#define PGSTAT_KIND_INVALID 0 - -/* stats for variable-numbered objects */ -#define PGSTAT_KIND_DATABASE 1 /* database-wide statistics */ -#define PGSTAT_KIND_RELATION 2 /* per-table statistics */ -#define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ -#define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ -#define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ -#define PGSTAT_KIND_RESQUEUE 6 /* per-resource-queue statistics */ - -/* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 7 -#define PGSTAT_KIND_BGWRITER 8 -#define PGSTAT_KIND_CHECKPOINTER 9 -#define PGSTAT_KIND_IO 10 -#define PGSTAT_KIND_SLRU 11 -#define PGSTAT_KIND_WAL 12 - -#define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE -#define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL -#define PGSTAT_KIND_BUILTIN_SIZE (PGSTAT_KIND_BUILTIN_MAX + 1) - -/* Custom stats kinds */ - -/* Range of IDs allowed for custom stats kinds */ -#define PGSTAT_KIND_CUSTOM_MIN 128 -#define PGSTAT_KIND_CUSTOM_MAX PGSTAT_KIND_MAX -#define PGSTAT_KIND_CUSTOM_SIZE (PGSTAT_KIND_CUSTOM_MAX - PGSTAT_KIND_CUSTOM_MIN + 1) - -/* - * PgStat_Kind to use for extensions that require an ID, but are still in - * development and have not reserved their own unique kind ID yet. See: - * https://wiki.postgresql.org/wiki/CustomCumulativeStats - */ -#define PGSTAT_KIND_EXPERIMENTAL 128 - -static inline bool -pgstat_is_kind_builtin(PgStat_Kind kind) -{ - return kind >= PGSTAT_KIND_BUILTIN_MIN && kind <= PGSTAT_KIND_BUILTIN_MAX; -} - -static inline bool -pgstat_is_kind_custom(PgStat_Kind kind) -{ - return kind >= PGSTAT_KIND_CUSTOM_MIN && kind <= PGSTAT_KIND_CUSTOM_MAX; -} - /* Values for track_functions GUC variable --- order is significant! */ typedef enum TrackFunctionsLevel { diff --git a/src/include/utils/pgstat_kind.h b/src/include/utils/pgstat_kind.h new file mode 100644 index 00000000000..d77639c7002 --- /dev/null +++ b/src/include/utils/pgstat_kind.h @@ -0,0 +1,72 @@ +/* ---------- + * pgstat_kind.h + * + * Definitions related to the statistics kinds for the PostgreSQL + * cumulative statistics system. Can be included in backend or + * frontend code. + * + * Copyright (c) 2001-2025, PostgreSQL Global Development Group + * + * src/include/utils/pgstat_kind.h + * ---------- + */ +#ifndef PGSTAT_KIND_H +#define PGSTAT_KIND_H + +/* The types of statistics entries */ +#define PgStat_Kind uint32 + +/* Range of IDs allowed, for built-in and custom kinds */ +#define PGSTAT_KIND_MIN 1 /* Minimum ID allowed */ +#define PGSTAT_KIND_MAX 256 /* Maximum ID allowed */ + +/* use 0 for INVALID, to catch zero-initialized data */ +#define PGSTAT_KIND_INVALID 0 + +/* stats for variable-numbered objects */ +#define PGSTAT_KIND_DATABASE 1 /* database-wide statistics */ +#define PGSTAT_KIND_RELATION 2 /* per-table statistics */ +#define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ +#define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ +#define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_RESQUEUE 6 /* per-resource-queue statistics */ + +/* stats for fixed-numbered objects */ +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 + +#define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE +#define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL +#define PGSTAT_KIND_BUILTIN_SIZE (PGSTAT_KIND_BUILTIN_MAX + 1) + +/* Custom stats kinds */ + +/* Range of IDs allowed for custom stats kinds */ +#define PGSTAT_KIND_CUSTOM_MIN 128 +#define PGSTAT_KIND_CUSTOM_MAX PGSTAT_KIND_MAX +#define PGSTAT_KIND_CUSTOM_SIZE (PGSTAT_KIND_CUSTOM_MAX - PGSTAT_KIND_CUSTOM_MIN + 1) + +/* + * PgStat_Kind to use for extensions that require an ID, but are still in + * development and have not reserved their own unique kind ID yet. See: + * https://wiki.postgresql.org/wiki/CustomCumulativeStats + */ +#define PGSTAT_KIND_EXPERIMENTAL 128 + +static inline bool +pgstat_is_kind_builtin(PgStat_Kind kind) +{ + return kind >= PGSTAT_KIND_BUILTIN_MIN && kind <= PGSTAT_KIND_BUILTIN_MAX; +} + +static inline bool +pgstat_is_kind_custom(PgStat_Kind kind) +{ + return kind >= PGSTAT_KIND_CUSTOM_MIN && kind <= PGSTAT_KIND_CUSTOM_MAX; +} + +#endif /* PGSTAT_KIND_H */ From 4d4381423ccdd47ba1ed3d95eaf5675ac093390e Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 25 Jul 2025 11:17:48 +0900 Subject: [PATCH 15/50] Lower bounds related to pgstats kinds This commit changes stats kinds to have the following bounds, making their handling in core cheaper by default: - PGSTAT_KIND_CUSTOM_MIN 128 -> 24 - PGSTAT_KIND_MAX 256 -> 32 The original numbers were rather high, and showed an impact on performance in pgstat_report_stat() for the case of simple queries with its early-exit path if there are no pending statistics to flush. This logic will be improved more in a follow-up commit to bring the performance of pgstat_report_stat() on par with v17 and older versions. Lowering the bounds is a change worth doing on its own, independently of the other improvement. These new numbers should be enough to leave some room for the following years for built-in and custom stats kinds, with stable ID numbers. At least that should be enough to start with this facility for extension developers. It can be always increased in the tree depending on the requirements wanted. Per discussion with Andres Freund and Bertrand Drouvot. Discussion: https://postgr.es/m/eb224uegsga2hgq7dfq3ps5cduhpqej7ir2hjxzzozjthrekx5@dysei6buqthe Backpatch-through: 18 Cloudberry: the injection_points test module does not exist here, so its part of the change is left out. (cherry picked from commit ac000fca743eff923d1feb4bc722d905901ae540) --- src/include/utils/pgstat_kind.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/utils/pgstat_kind.h b/src/include/utils/pgstat_kind.h index d77639c7002..dd1f4022bf0 100644 --- a/src/include/utils/pgstat_kind.h +++ b/src/include/utils/pgstat_kind.h @@ -18,7 +18,7 @@ /* Range of IDs allowed, for built-in and custom kinds */ #define PGSTAT_KIND_MIN 1 /* Minimum ID allowed */ -#define PGSTAT_KIND_MAX 256 /* Maximum ID allowed */ +#define PGSTAT_KIND_MAX 32 /* Maximum ID allowed */ /* use 0 for INVALID, to catch zero-initialized data */ #define PGSTAT_KIND_INVALID 0 @@ -46,7 +46,7 @@ /* Custom stats kinds */ /* Range of IDs allowed for custom stats kinds */ -#define PGSTAT_KIND_CUSTOM_MIN 128 +#define PGSTAT_KIND_CUSTOM_MIN 24 #define PGSTAT_KIND_CUSTOM_MAX PGSTAT_KIND_MAX #define PGSTAT_KIND_CUSTOM_SIZE (PGSTAT_KIND_CUSTOM_MAX - PGSTAT_KIND_CUSTOM_MIN + 1) @@ -55,7 +55,7 @@ * development and have not reserved their own unique kind ID yet. See: * https://wiki.postgresql.org/wiki/CustomCumulativeStats */ -#define PGSTAT_KIND_EXPERIMENTAL 128 +#define PGSTAT_KIND_EXPERIMENTAL 24 static inline bool pgstat_is_kind_builtin(PgStat_Kind kind) From 83ee8ed38961e4d4878059bf7d8302738411c569 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 17:18:19 +0300 Subject: [PATCH 16/50] Preserve Cloudberry statistics persistence with pluggable kinds Keep resource queue statistics across clean restarts by enabling write_to_file for the Cloudberry resqueue kind, and cover persistence with a restart TAP test. Advance the statistics file format identifier for the pluggable statistics layout. --- src/backend/utils/activity/pgstat.c | 1 + src/include/pgstat.h | 2 +- src/test/modules/test_misc/meson.build | 1 + .../test_misc/t/019_resqueue_stats_restart.pl | 76 +++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/test/modules/test_misc/t/019_resqueue_stats_restart.pl diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 6db2f524ded..cea1f90a99c 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -369,6 +369,7 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .name = "resqueue", .fixed_amount = false, + .write_to_file = true, /* resource queues are cluster-wide objects, visible across databases */ .accessed_across_databases = true, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 34262d89257..aed08dba825 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -212,7 +212,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB3 typedef struct PgStat_ArchiverStats { diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..c6c367ef683 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,7 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/019_resqueue_stats_restart.pl', ], }, } diff --git a/src/test/modules/test_misc/t/019_resqueue_stats_restart.pl b/src/test/modules/test_misc/t/019_resqueue_stats_restart.pl new file mode 100644 index 00000000000..b8435641a71 --- /dev/null +++ b/src/test/modules/test_misc/t/019_resqueue_stats_restart.pl @@ -0,0 +1,76 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Resource queue statistics must survive a clean restart. Queue admission +# needs a dispatcher, so use a coordinator and one primary segment rather +# than the standalone utility-mode instances normally used by TAP. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'temporary cluster requires internal FTS' + unless check_pg_config('#define USE_INTERNAL_FTS 1'); + +my $node = PostgreSQL::Test::Cluster->new('resqueue_coordinator'); +my $segment = PostgreSQL::Test::Cluster->new('resqueue_segment'); +for my $instance ($node, $segment) +{ + $instance->init; + $instance->append_conf('postgresql.conf', q{ +autovacuum = off +gp_resource_manager = 'queue' +listen_addresses = '127.0.0.1' +}); +} + +sub start_instance +{ + my ($instance, $role, $content) = @_; + command_ok( + ['pg_ctl', '-D', $instance->data_dir, '-l', $instance->logfile, + '-o', "-c gp_role=$role --gp_dbid=$instance->{_dbid} --gp_contentid=$content", + '-w', 'start'], + "started content $content in $role mode"); + $instance->_update_pid(1); +} + +# Populate the cluster configuration in utility mode before starting the +# dispatcher, as gpinitsystem does. +start_instance($node, 'utility', -1); +for my $entry ([$node, -1], [$segment, 0]) +{ + my ($instance, $content) = @$entry; + my $port = $instance->port; + my $datadir = $instance->data_dir; + $node->safe_psql('postgres', qq{ +SELECT gp_add_segment($instance->{_dbid}::int2, ${content}::int2, + 'p', 'p', 's', 'u', $port, + 'localhost', '127.0.0.1', '$datadir'); +}); +} +$node->stop; +start_instance($segment, 'execute', 0); +start_instance($node, 'dispatch', -1); + +# Explicit connection options override the utility-mode default of psql(). +my $connstr = $node->connstr('postgres') . " options=''"; +$node->safe_psql('postgres', q{ +CREATE RESOURCE QUEUE restart_queue WITH (active_statements = 1); +CREATE ROLE restart_user RESOURCE QUEUE restart_queue; +SET ROLE restart_user; +SELECT 1; +RESET ROLE; +}, connstr => $connstr); +my $stats_sql = q{ +SELECT queries_submitted, queries_admitted, queries_completed + FROM pg_stat_resqueues WHERE queuename = 'restart_queue'}; +my $before = $node->safe_psql('postgres', $stats_sql, connstr => $connstr); +like($before, qr/^[1-9][0-9]*[|][1-9][0-9]*[|][1-9][0-9]*$/, + 'query populated the resource queue statistics'); +$node->restart; +is($node->safe_psql('postgres', $stats_sql, connstr => $connstr), $before, + 'resource queue statistics survive a clean restart'); +$node->stop; +$segment->stop; +done_testing(); From c6187aa8b439d031c2daa8eaf725cc4a493f7b6b Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 8 Oct 2026 00:00:00 +0300 Subject: [PATCH 17/50] Allow custom statistics resets to preserve entry metadata Add an optional reset_data_cb for variable-numbered statistics kinds. Call it under the existing entry lock before resetting the timestamp. Kinds without the callback retain the existing zero-fill behavior. --- src/backend/utils/activity/pgstat_shmem.c | 7 +++++-- src/include/utils/pgstat_internal.h | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index 659e5f0fdb1..45b7e19389a 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -1019,8 +1019,11 @@ shared_stat_reset_contents(PgStat_Kind kind, PgStatShared_Common *header, { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); - memset(pgstat_get_entry_data(kind, header), 0, - pgstat_get_entry_len(kind)); + if (kind_info->reset_data_cb) + kind_info->reset_data_cb(header); + else + memset(pgstat_get_entry_data(kind, header), 0, + pgstat_get_entry_len(kind)); if (kind_info->reset_timestamp_cb) kind_info->reset_timestamp_cb(header, ts); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 3f1fae0b7c2..5f316f98a1b 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -262,6 +262,13 @@ typedef struct PgStat_KindInfo */ void (*delete_pending_cb) (PgStat_EntryRef *sr); + /* + * For variable-numbered stats: reset counters while preserving identifying + * metadata in the payload. Optional; otherwise the whole payload is zeroed. + * Called with the entry's lock held exclusively, before reset_timestamp_cb. + */ + void (*reset_data_cb) (PgStatShared_Common *header); + /* * For variable-numbered stats: reset the reset timestamp. Optional. */ From b251aefdf040916423be18c5119eb0a8f52131eb Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 28 Jan 2025 09:57:32 +0900 Subject: [PATCH 18/50] Track per-relation cumulative time spent in [auto]vacuum and [auto]analyze This commit adds four fields to the statistics of relations, aggregating the amount of time spent for each operation on a relation: - total_vacuum_time, for manual vacuum. - total_autovacuum_time, for vacuum done by the autovacuum daemon. - total_analyze_time, for manual analyze. - total_autoanalyze_time, for analyze done by the autovacuum daemon. This gives users the option to derive the average time spent for these operations with the help of the related "count" fields. Bump PGSTAT_FILE_FORMAT_ID for the additions in PgStat_StatTabEntry. Catalog functions, views and their version bump follow separately. Author: Sami Imseih Reviewed-by: Bertrand Drouvot, Michael Paquier Discussion: https://postgr.es/m/CAA5RZ0uVOGBYmPEeGF2d1B_67tgNjKx_bKDuL+oUftuoz+=Y1g@mail.gmail.com (cherry picked from commit 30a6ed0ce4bb18212ec38cdb537ea4b43bc99b83) Cloudberry adaptation: time AO vacuum from its first phase and pass the start timestamp to the collector at final cleanup, so the changed API is usable by heap and AO immediately. Co-authored-by: Alena Rybakina --- src/backend/access/heap/vacuumlazy.c | 7 +++++-- src/backend/commands/analyze.c | 13 ++++++++---- src/backend/commands/vacuum_ao.c | 9 ++++++++- src/backend/utils/activity/pgstat_relation.c | 21 ++++++++++++++++---- src/include/access/appendonly_compaction.h | 3 +++ src/include/pgstat.h | 12 ++++++++--- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..fbf3b1245ae 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -351,7 +351,6 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, if (instrument) { pg_rusage_init(&ru0); - starttime = GetCurrentTimestamp(); if (track_io_timing) { startreadtime = pgStatBlockReadTime; @@ -359,6 +358,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } + /* Used for instrumentation and stats report */ + starttime = GetCurrentTimestamp(); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -619,7 +621,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples); + vacrel->missed_dead_tuples, + starttime); pgstat_progress_end_command(); if (instrument) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index e7c5e68082c..b4269a23ccd 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -541,7 +541,10 @@ do_analyze_rel(Relation onerel, VacuumParams *params, save_sec_context | SECURITY_RESTRICTED_OPERATION); save_nestlevel = NewGUCNestLevel(); - /* measure elapsed time iff autovacuum logging requires it */ + /* + * When autovacuum logging is used, initialize a resource usage snapshot + * and optionally track I/O timing. + */ if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0) { if (track_io_timing) @@ -551,9 +554,11 @@ do_analyze_rel(Relation onerel, VacuumParams *params, } pg_rusage_init(&ru0); - starttime = GetCurrentTimestamp(); } + /* Used for instrumentation and stats report */ + starttime = GetCurrentTimestamp(); + /* * Determine which columns to analyze * @@ -1222,9 +1227,9 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ if (!inh) pgstat_report_analyze(onerel, totalrows, totaldeadrows, - (va_cols == NIL)); + (va_cols == NIL), starttime); else if (onerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - pgstat_report_analyze(onerel, 0, 0, (va_cols == NIL)); + pgstat_report_analyze(onerel, 0, 0, (va_cols == NIL), starttime); /* * If this isn't part of VACUUM ANALYZE, let index AMs do cleanup. diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..4b1137787d9 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -306,7 +306,8 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt pgstat_report_vacuum(RelationGetRelid(onerel), onerel->rd_rel->relisshared, reltuples, - deadtuples); + deadtuples, + vacrelstats->starttime); SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); } @@ -416,6 +417,12 @@ init_vacrelstats() vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); MemoryContextSwitchTo(old_context); + /* + * The phases of an AO vacuum run as separate steps, all of them sharing + * these stats; time the vacuum from the first one. + */ + vacrelstats->starttime = GetCurrentTimestamp(); + return vacrelstats; } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 45bbf9abcb7..0a47a18e638 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -215,19 +215,22 @@ pgstat_drop_relation(Relation rel) */ void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + TimestampTz starttime) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = (shared ? InvalidOid : MyDatabaseId); TimestampTz ts; + PgStat_Counter elapsedtime; if (!pgstat_track_counts) return; /* Store the data in the table's hash table entry. */ ts = GetCurrentTimestamp(); + elapsedtime = TimestampDifferenceMilliseconds(starttime, ts); /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, @@ -255,11 +258,13 @@ pgstat_report_vacuum(Oid tableoid, bool shared, { tabentry->last_autovacuum_time = ts; tabentry->autovacuum_count++; + tabentry->total_autovacuum_time += elapsedtime; } else { tabentry->last_vacuum_time = ts; tabentry->vacuum_count++; + tabentry->total_vacuum_time += elapsedtime; } pgstat_unlock_entry(entry_ref); @@ -282,12 +287,14 @@ pgstat_report_vacuum(Oid tableoid, bool shared, void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, - bool resetcounter) + bool resetcounter, TimestampTz starttime) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = (rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId); + TimestampTz ts; + PgStat_Counter elapsedtime; if (!pgstat_track_counts) return; @@ -321,6 +328,10 @@ pgstat_report_analyze(Relation rel, deadtuples = Max(deadtuples, 0); } + /* Store the data in the table's hash table entry. */ + ts = GetCurrentTimestamp(); + elapsedtime = TimestampDifferenceMilliseconds(starttime, ts); + /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, RelationGetRelid(rel), @@ -344,13 +355,15 @@ pgstat_report_analyze(Relation rel, if (IsAutoVacuumWorkerProcess()) { - tabentry->last_autoanalyze_time = GetCurrentTimestamp(); + tabentry->last_autoanalyze_time = ts; tabentry->autoanalyze_count++; + tabentry->total_autoanalyze_time += elapsedtime; } else { - tabentry->last_analyze_time = GetCurrentTimestamp(); + tabentry->last_analyze_time = ts; tabentry->analyze_count++; + tabentry->total_analyze_time += elapsedtime; } pgstat_unlock_entry(entry_ref); diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..71f954865b2 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -13,6 +13,7 @@ #ifndef APPENDONLY_COMPACTION_H #define APPENDONLY_COMPACTION_H +#include "datatype/timestamp.h" #include "nodes/pg_list.h" #include "access/appendonly_visimap.h" #include "utils/rel.h" @@ -31,6 +32,8 @@ typedef struct AOVacuumRelStats int nbytes_truncated; /* current # of bytes truncated from segment file */ int num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + /* when the first phase started, for the cumulative vacuum time */ + TimestampTz starttime; } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..1193694b330 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -238,7 +238,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAE typedef struct PgStat_ArchiverStats { @@ -427,6 +427,11 @@ typedef struct PgStat_StatTabEntry PgStat_Counter analyze_count; TimestampTz last_autoanalyze_time; /* autovacuum initiated */ PgStat_Counter autoanalyze_count; + + PgStat_Counter total_vacuum_time; /* times in milliseconds */ + PgStat_Counter total_autovacuum_time; + PgStat_Counter total_analyze_time; + PgStat_Counter total_autoanalyze_time; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -651,10 +656,11 @@ extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + TimestampTz starttime); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, - bool resetcounter); + bool resetcounter, TimestampTz starttime); /* * If stats are enabled, but pending data hasn't been prepared yet, call From 463bf6e6affc045dd21fad602de22adb08dce3c4 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 11 Feb 2025 16:38:14 -0600 Subject: [PATCH 19/50] Add is_analyze parameter to vacuum_delay_point(). This function is used in both vacuum and analyze code paths, and a follow-up commit will require distinguishing between the two. This commit forces callers to specify whether they are in a vacuum or analyze path, but it does not use that information for anything yet. Author: Nathan Bossart Co-authored-by: Bertrand Drouvot Discussion: https://postgr.es/m/ZmaXmWDL829fzAVX%40ip-10-97-1-34.eu-west-3.compute.internal Backported from PostgreSQL 18 (commit e5b0b0ce150972bf162a059430d84e5f8e07cf30) as a prerequisite of the vacuum statistics series. Cloudberry adaptation: the append-optimized and AOCS sample-row acquisition and compaction call vacuum_delay_point() too; the former pass is_analyze = true, the latter false. --- contrib/bloom/blvacuum.c | 4 ++-- contrib/file_fdw/file_fdw.c | 2 +- src/backend/access/aocs/aocs_compaction.c | 2 +- src/backend/access/aocs/aocsam_handler.c | 2 +- src/backend/access/appendonly/appendonly_compaction.c | 2 +- src/backend/access/appendonly/appendonlyam_handler.c | 2 +- src/backend/access/gin/ginfast.c | 6 +++--- src/backend/access/gin/ginvacuum.c | 6 +++--- src/backend/access/gist/gistvacuum.c | 2 +- src/backend/access/hash/hash.c | 2 +- src/backend/access/heap/vacuumlazy.c | 6 +++--- src/backend/access/nbtree/nbtree.c | 2 +- src/backend/access/spgist/spgvacuum.c | 4 ++-- src/backend/commands/analyze.c | 10 +++++----- src/backend/commands/vacuum.c | 2 +- src/backend/tsearch/ts_typanalyze.c | 2 +- src/backend/utils/adt/array_typanalyze.c | 2 +- src/backend/utils/adt/rangetypes_typanalyze.c | 2 +- src/include/commands/vacuum.h | 2 +- 19 files changed, 31 insertions(+), 31 deletions(-) diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c index 2340d49e00a..6a6609be16f 100644 --- a/contrib/bloom/blvacuum.c +++ b/contrib/bloom/blvacuum.c @@ -61,7 +61,7 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, *itupPtr, *itupEnd; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); @@ -191,7 +191,7 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) Buffer buffer; Page page; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index b5ffde4131b..59fcc8ae729 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1197,7 +1197,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, for (;;) { /* Check for user-requested abort or sleep */ - vacuum_delay_point(); + vacuum_delay_point(true); /* Fetch next row */ MemoryContextReset(tupcontext); diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index 12df08f3d67..17ddd8e060d 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -323,7 +323,7 @@ AOCSSegmentFileFullCompaction(Relation aorel, tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { - vacuum_delay_point(); + vacuum_delay_point(false); } /* diff --git a/src/backend/access/aocs/aocsam_handler.c b/src/backend/access/aocs/aocsam_handler.c index 4dc7d541c5f..0256c1f4e00 100644 --- a/src/backend/access/aocs/aocsam_handler.c +++ b/src/backend/access/aocs/aocsam_handler.c @@ -1661,7 +1661,7 @@ aoco_acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, { aocoscan->targrow = RowSampler_Next(&rs); - vacuum_delay_point(); + vacuum_delay_point(true); if (aocs_get_target_tuple(aocoscan, aocoscan->targrow, slot)) { diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index b8cae6ebf6c..c37120be0c3 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -506,7 +506,7 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { - vacuum_delay_point(); + vacuum_delay_point(false); } } diff --git a/src/backend/access/appendonly/appendonlyam_handler.c b/src/backend/access/appendonly/appendonlyam_handler.c index cae12e110ff..7a7942c5e24 100644 --- a/src/backend/access/appendonly/appendonlyam_handler.c +++ b/src/backend/access/appendonly/appendonlyam_handler.c @@ -1544,7 +1544,7 @@ appendonly_acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, { aoscan->targrow = RowSampler_Next(&rs); - vacuum_delay_point(); + vacuum_delay_point(true); if (appendonly_get_target_tuple(aoscan, aoscan->targrow, slot)) { diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c index 2c23afe3a03..65b7f7ef3b7 100644 --- a/src/backend/access/gin/ginfast.c +++ b/src/backend/access/gin/ginfast.c @@ -891,7 +891,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, */ processPendingPage(&accum, &datums, page, FirstOffsetNumber); - vacuum_delay_point(); + vacuum_delay_point(false); /* * Is it time to flush memory to disk? Flush if we are at the end of @@ -928,7 +928,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, { ginEntryInsert(ginstate, attnum, key, category, list, nlist, NULL); - vacuum_delay_point(); + vacuum_delay_point(false); } /* @@ -1001,7 +1001,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, /* * Read next page in pending list */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBuffer(index, blkno); LockBuffer(buffer, GIN_SHARE); page = BufferGetPage(buffer); diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index e5d310d8362..067224ab475 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -663,12 +663,12 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, UnlockReleaseBuffer(buffer); } - vacuum_delay_point(); + vacuum_delay_point(false); for (i = 0; i < nRoot; i++) { ginVacuumPostingTree(&gvs, rootOfPostingTree[i]); - vacuum_delay_point(); + vacuum_delay_point(false); } if (blkno == InvalidBlockNumber) /* rightmost page */ @@ -749,7 +749,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) Buffer buffer; Page page; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 3f60d3274d2..f6608b9c3cf 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -283,7 +283,7 @@ gistvacuumpage(GistVacState *vstate, BlockNumber blkno, BlockNumber orig_blkno) recurse_to = InvalidBlockNumber; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index e50f117bcb9..bb5fd890d76 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -732,7 +732,7 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, bool retain_pin = false; bool clear_dead_marking = false; - vacuum_delay_point(); + vacuum_delay_point(false); page = BufferGetPage(buf); opaque = HashPageGetOpaque(page); diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index fbf3b1245ae..9b65941352b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -914,7 +914,7 @@ lazy_scan_heap(LVRelState *vacrel) update_vacuum_error_info(vacrel, NULL, VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, InvalidOffsetNumber); - vacuum_delay_point(); + vacuum_delay_point(false); /* * Regularly check if wraparound failsafe should trigger. @@ -1377,7 +1377,7 @@ lazy_scan_skip(LVRelState *vacrel, Buffer *vmbuffer, BlockNumber next_block, skipsallvis = true; } - vacuum_delay_point(); + vacuum_delay_point(false); next_unskippable_block++; nskippable_blocks++; } @@ -2463,7 +2463,7 @@ lazy_vacuum_heap_rel(LVRelState *vacrel) Page page; Size freespace; - vacuum_delay_point(); + vacuum_delay_point(false); blkno = ItemPointerGetBlockNumber(&vacrel->dead_items->items[index]); vacrel->blkno = blkno; diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 3ad1c7ea0d8..2345a5bcb02 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -1171,7 +1171,7 @@ btvacuumpage(BTVacState *vstate, BlockNumber scanblkno) backtrack_to = P_NONE; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); /* * We can't use _bt_getbuf() here because it always applies diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index 85bc02b92c8..1f0e98648b8 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -626,7 +626,7 @@ spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) Page page; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bds->info->strategy); @@ -705,7 +705,7 @@ spgprocesspending(spgBulkDeleteState *bds) continue; /* ignore already-done items */ /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); /* examine the referenced page */ blkno = ItemPointerGetBlockNumber(&pitem->tid); diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index b4269a23ccd..465ca5b19ed 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -1428,7 +1428,7 @@ compute_index_stats(Relation onerel, double totalrows, { HeapTuple heapTuple = rows[rowno]; - vacuum_delay_point(); + vacuum_delay_point(true); /* * Reset the per-tuple context each time, to reclaim any cruft @@ -1848,7 +1848,7 @@ acquire_sample_rows(Relation onerel, int elevel, prefetch_targblock = BlockSampler_Next(&prefetch_bs); #endif - vacuum_delay_point(); + vacuum_delay_point(true); block_accepted = table_scan_analyze_next_block(scan, targblock, vac_strategy); @@ -3476,7 +3476,7 @@ compute_trivial_stats(VacAttrStatsP stats, Datum value; bool isnull; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); @@ -3598,7 +3598,7 @@ compute_distinct_stats(VacAttrStatsP stats, int firstcount1, j; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); @@ -3958,7 +3958,7 @@ compute_scalar_stats(VacAttrStatsP stats, Datum value; bool isnull; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index f4ba5615d5a..5ed4a436d11 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -3179,7 +3179,7 @@ vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode) * typically once per page processed. */ void -vacuum_delay_point(void) +vacuum_delay_point(bool is_analyze) { double msec = 0; diff --git a/src/backend/tsearch/ts_typanalyze.c b/src/backend/tsearch/ts_typanalyze.c index c24243b11b4..6aa685ed55f 100644 --- a/src/backend/tsearch/ts_typanalyze.c +++ b/src/backend/tsearch/ts_typanalyze.c @@ -206,7 +206,7 @@ compute_tsvector_stats(VacAttrStats *stats, char *lexemesptr; int j; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, vector_no, &isnull); diff --git a/src/backend/utils/adt/array_typanalyze.c b/src/backend/utils/adt/array_typanalyze.c index 52e160d6bbb..2db192d7544 100644 --- a/src/backend/utils/adt/array_typanalyze.c +++ b/src/backend/utils/adt/array_typanalyze.c @@ -314,7 +314,7 @@ compute_array_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int distinct_count; bool count_item_found; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, array_no, &isnull); if (isnull) diff --git a/src/backend/utils/adt/rangetypes_typanalyze.c b/src/backend/utils/adt/rangetypes_typanalyze.c index ff8700e8e43..a3e98fcf766 100644 --- a/src/backend/utils/adt/rangetypes_typanalyze.c +++ b/src/backend/utils/adt/rangetypes_typanalyze.c @@ -169,7 +169,7 @@ compute_range_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, upper; float8 length; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, range_no, &isnull); if (isnull) diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..1ac8919a5a1 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -475,7 +475,7 @@ extern bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs); extern bool vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs); extern void vac_update_datfrozenxid(void); -extern void vacuum_delay_point(void); +extern void vacuum_delay_point(bool is_analyze); extern bool vacuum_is_relation_owner(Oid relid, Form_pg_class reltuple, bits32 options); extern Relation vacuum_open_relation(Oid relid, RangeVar *relation, From 1439570a2690e92d123da1084df0637cc4ce9020 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 11 Feb 2025 16:38:14 -0600 Subject: [PATCH 20/50] Measure cost-based vacuum delay for progress reporting. This commit adds the amount of time spent sleeping due to cost-based delay to the pg_stat_progress_vacuum and pg_stat_progress_analyze system views. A new configuration parameter named track_cost_delay_timing, which is off by default, controls whether this information is gathered. For vacuum, the reported value includes the sleep time of any associated parallel workers. However, parallel workers only report their sleep time once per second to avoid overloading the leader process. Bumps catversion. Author: Bertrand Drouvot Co-authored-by: Nathan Bossart Reviewed-by: Sami Imseih Reviewed-by: Robert Haas Reviewed-by: Masahiko Sawada Reviewed-by: Masahiro Ikeda Reviewed-by: Dilip Kumar Reviewed-by: Sergei Kornilov Discussion: https://postgr.es/m/ZmaXmWDL829fzAVX%40ip-10-97-1-34.eu-west-3.compute.internal Backported from PostgreSQL 18 (commit bb8dff9995f2cf501376772898bcbcf58aa05cde) as a prerequisite of the vacuum statistics series. Cloudberry adaptation: the vacuum progress parameters of this branch end at num_dead_tuples, so the delay time is PROGRESS_VACUUM_DELAY_TIME 7 (param8 of the view). A parallel vacuum worker only accumulates its sleep time: reporting it to the leader needs pgstat_progress_parallel_incr_param() of PostgreSQL 17, and Cloudberry does not run parallel vacuum at all (see dead_items_alloc()). The new GUC is listed in unsync_guc_names_array, next to track_io_timing, as every GUC has to be in one of the two lists here. SQL progress-view changes are separated into the native SQL reporting commit; measurement, the GUC and progress counters are introduced here. --- src/backend/commands/vacuum.c | 44 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 9 ++++ src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/commands/progress.h | 2 + src/include/commands/vacuum.h | 3 ++ src/include/utils/unsync_guc_name.h | 1 + 6 files changed, 60 insertions(+) diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 5ed4a436d11..f68b1fd274e 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -47,6 +47,7 @@ #include "catalog/pg_namespace.h" #include "commands/cluster.h" #include "commands/defrem.h" +#include "commands/progress.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" #include "miscadmin.h" @@ -106,6 +107,7 @@ int vacuum_multixact_freeze_min_age; int vacuum_multixact_freeze_table_age; int vacuum_failsafe_age; int vacuum_multixact_failsafe_age; +bool track_cost_delay_timing; /* * Variables for cost-based vacuum delay. The defaults differ between @@ -116,6 +118,9 @@ int vacuum_multixact_failsafe_age; double vacuum_cost_delay = 0; int vacuum_cost_limit = 200; +/* Variable for reporting cost-based vacuum delay from parallel workers. */ +int64 parallel_vacuum_worker_delay_ns = 0; + /* * VacuumFailsafeActive is a defined as a global so that we can determine * whether or not to re-enable cost-based vacuum delay when vacuuming a table. @@ -3222,13 +3227,52 @@ vacuum_delay_point(bool is_analyze) /* Nap if appropriate */ if (msec > 0) { + instr_time delay_start; + if (msec > vacuum_cost_delay * 4) msec = vacuum_cost_delay * 4; + if (track_cost_delay_timing) + INSTR_TIME_SET_CURRENT(delay_start); + pgstat_report_wait_start(WAIT_EVENT_VACUUM_DELAY); pg_usleep(msec * 1000); pgstat_report_wait_end(); + if (track_cost_delay_timing) + { + instr_time delay_end; + instr_time delay; + + INSTR_TIME_SET_CURRENT(delay_end); + INSTR_TIME_SET_ZERO(delay); + INSTR_TIME_ACCUM_DIFF(delay, delay_end, delay_start); + + /* + * For parallel workers, we only report the delay time every once + * in a while to avoid overloading the leader with messages and + * interrupts. + */ + if (IsParallelWorker()) + { + /* + * Upstream sends the sleep time of a parallel worker to the + * leader's progress entry now and then, which needs + * pgstat_progress_parallel_incr_param() of PostgreSQL 17. + * Cloudberry does not run parallel vacuum, so just + * accumulate it. + */ + Assert(!is_analyze); + parallel_vacuum_worker_delay_ns += INSTR_TIME_GET_NANOSEC(delay); + } + else if (is_analyze) + pgstat_progress_incr_param(PROGRESS_ANALYZE_DELAY_TIME, + INSTR_TIME_GET_NANOSEC(delay)); + else + pgstat_progress_incr_param(PROGRESS_VACUUM_DELAY_TIME, + INSTR_TIME_GET_NANOSEC(delay)); + } + /* * We don't want to ignore postmaster death during very long vacuums * with vacuum_cost_delay configured. We can't use the usual diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1ff92994527..6e9313b393b 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1480,6 +1480,15 @@ struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"track_cost_delay_timing", PGC_SUSET, STATS_CUMULATIVE, + gettext_noop("Collects timing statistics for cost-based vacuum delay."), + NULL + }, + &track_cost_delay_timing, + false, + NULL, NULL, NULL + }, { {"track_io_timing", PGC_SUSET, STATS_CUMULATIVE, gettext_noop("Collects timing statistics for database I/O activity."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index f1848a3d072..b8f3e6586ab 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -646,6 +646,7 @@ optimizer_analyze_root_partition = on # stats collection on root partitions #track_activities = on #track_activity_query_size = 1024 # (change requires restart) #track_counts = off +#track_cost_delay_timing = off #track_io_timing = off #track_wal_io_timing = off #track_functions = none # none, pl, all diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index fc9ce7c1a66..01e0b766cd3 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -25,6 +25,7 @@ #define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4 #define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5 #define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6 +#define PROGRESS_VACUUM_DELAY_TIME 7 /* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */ #define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1 @@ -47,6 +48,7 @@ #define PROGRESS_ANALYZE_CHILD_TABLES_TOTAL 5 #define PROGRESS_ANALYZE_CHILD_TABLES_DONE 6 #define PROGRESS_ANALYZE_CURRENT_CHILD_TABLE_RELID 7 +#define PROGRESS_ANALYZE_DELAY_TIME 8 /* Phases of analyze (as advertised via PROGRESS_ANALYZE_PHASE) */ #define PROGRESS_ANALYZE_PHASE_ACQUIRE_SAMPLE_ROWS 1 diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 1ac8919a5a1..ad5f980d9b1 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -434,6 +434,7 @@ extern PGDLLIMPORT int vacuum_multixact_freeze_min_age; extern PGDLLIMPORT int vacuum_multixact_freeze_table_age; extern PGDLLIMPORT int vacuum_failsafe_age; extern PGDLLIMPORT int vacuum_multixact_failsafe_age; +extern PGDLLIMPORT bool track_cost_delay_timing; /* Variables for cost-based parallel vacuum */ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; @@ -444,6 +445,8 @@ extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; +extern PGDLLIMPORT int64 parallel_vacuum_worker_delay_ns; + /* in commands/vacuum.c */ extern void ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_stats); extern void vacuum(List *relations, VacuumParams *params, diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 02b04fdc5c4..235732f7266 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -616,6 +616,7 @@ "track_activities", "track_activity_query_size", "track_commit_timestamp", + "track_cost_delay_timing", "track_counts", "track_functions", "track_io_timing", From 4919bf929ef559254f3629ceb764544968e2ed3d Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 11 Feb 2025 16:38:14 -0600 Subject: [PATCH 21/50] Measure cost-based vacuum delay Port the process-local VacuumDelayTime accumulator from the REL_2_STABLE commit. Accumulate measured cost-based VACUUM sleep time only while track_cost_delay_timing is enabled, excluding ANALYZE. Consumers take before/after differences to attribute delays to their own operations. The preceding PostgreSQL 18 progress-view backport already provides the GUC, its documentation, sample setting, unsynchronized registration and sleep timing. Reuse that measurement and preserve both VACUUM and ANALYZE progress reporting. Keep the accumulator in milliseconds as double to match this branch's cumulative timing consumers, rather than the source branch's int64 microseconds. Extract the accumulator from the index/database timing commit and fold in the ANALYZE exclusion previously included in the combined heap/index/AO measurements commit. The final branch contents are unchanged. Author: Bertrand Drouvot Discussion: https://postgr.es/m/ZmaXmWDL829fzAVX%40ip-10-97-1-34.eu-west-3.compute.internal Backported from PostgreSQL 18 (commit bb8dff9995f2cf501376772898bcbcf58aa05cde). (cherry picked from commit 01aaacbebf47548bde0f3521762fb643996d6094) Co-authored-by: Nathan Bossart Co-authored-by: Alena Rybakina --- src/backend/commands/vacuum.c | 9 +++++++++ src/include/commands/vacuum.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index f68b1fd274e..cae73685cca 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -121,6 +121,13 @@ int vacuum_cost_limit = 200; /* Variable for reporting cost-based vacuum delay from parallel workers. */ int64 parallel_vacuum_worker_delay_ns = 0; +/* + * Cumulative cost-based VACUUM delay time (ms) for this process, excluding + * ANALYZE. + * Callers save the starting value and report the delta. + */ +double VacuumDelayTime = 0; + /* * VacuumFailsafeActive is a defined as a global so that we can determine * whether or not to re-enable cost-based vacuum delay when vacuuming a table. @@ -3247,6 +3254,8 @@ vacuum_delay_point(bool is_analyze) INSTR_TIME_SET_CURRENT(delay_end); INSTR_TIME_SET_ZERO(delay); INSTR_TIME_ACCUM_DIFF(delay, delay_end, delay_start); + if (!is_analyze) + VacuumDelayTime += INSTR_TIME_GET_MILLISEC(delay); /* * For parallel workers, we only report the delay time every once diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index ad5f980d9b1..54b897a528c 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -446,6 +446,7 @@ extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; extern PGDLLIMPORT int64 parallel_vacuum_worker_delay_ns; +extern PGDLLIMPORT double VacuumDelayTime; /* in commands/vacuum.c */ extern void ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_stats); From 7fd7bafb4970b3ca57ff1369290c9c8763920e78 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 21 Sep 2026 01:40:05 +0300 Subject: [PATCH 22/50] Measure append-optimized vacuum active-phase time Accumulate elapsed time and cost-based delays over active AO vacuum phases, excluding the gaps between phases. Discard state left over from an interrupted operation. Prepare per-index timing measurements for the subsequent cumulative-reporting change. Adapted-from: 6f38fb242db39b4df922687265f9651bb115b35a --- src/backend/commands/vacuum_ao.c | 90 ++++++++++++++++++---- src/include/access/appendonly_compaction.h | 14 +++- 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 4b1137787d9..53e920958fe 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -114,6 +114,8 @@ */ #include "postgres.h" +#include + #include "access/table.h" #include "access/aocs_compaction.h" #include "access/aomd.h" @@ -156,13 +158,15 @@ static bool appendonly_tid_reaped(ItemPointer itemptr, void *state); static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static AOVacuumRelStats *init_vacrelstats(void); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); +static void ao_measure_index_vacuum_time(Relation indrel, TimestampTz starttime, + double startdelaytime); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -237,7 +241,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt { BlockNumber relpages; double reltuples; - double deadtuples; + int64 deadtuples; bool relhasindex; /* AO/AOCO total file segment number, use type BlockNumber to * represent same type with num_all_visible_pages in libpq. @@ -302,6 +306,10 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt false, true /* isvacuum */); + /* Report once all phase timers have been stopped. */ + vacrelstats->live_tuples = reltuples; + vacrelstats->dead_tuples = deadtuples; + /* report results to the stats collector, too */ pgstat_report_vacuum(RelationGetRelid(onerel), onerel->rd_rel->relisshared, @@ -309,6 +317,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt deadtuples, vacrelstats->starttime); + SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); } @@ -415,13 +424,8 @@ init_vacrelstats() old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); - MemoryContextSwitchTo(old_context); - - /* - * The phases of an AO vacuum run as separate steps, all of them sharing - * these stats; time the vacuum from the first one. - */ vacrelstats->starttime = GetCurrentTimestamp(); + MemoryContextSwitchTo(old_context); return vacrelstats; } @@ -435,11 +439,27 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + instr_time phase_start; + instr_time phase_end; + double phase_start_delay; Assert(RelationStorageIsAO(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -457,8 +477,12 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); vacrelstats = init_vacrelstats(); + vacrelstats->relid = RelationGetRelid(rel); } + INSTR_TIME_SET_CURRENT(phase_start); + phase_start_delay = VacuumDelayTime; + /* * Do the actual work --- either FULL or "lazy" vacuum */ @@ -467,14 +491,42 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); - pgstat_progress_end_command(); - cleanup_vacrelstats(&vacrelstats); - } else /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); + + if (ao_vacuum_phase != 0) + { + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + vacrelstats->vacuum_time += INSTR_TIME_GET_MILLISEC(phase_end); + vacrelstats->delay_time += VacuumDelayTime - phase_start_delay; + } + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { + pgstat_progress_end_command(); + cleanup_vacrelstats(&vacrelstats); + } +} + +/* + * Accumulate one vacuum pass over an index of an append-optimized table + * into the index's cumulative vacuum and delay times, as the heap does in + * lazy_vacuum_one_index() and lazy_cleanup_one_index(). + */ +static void +ao_measure_index_vacuum_time(Relation indrel, TimestampTz starttime, + double startdelaytime) +{ + PgStat_Counter elapsedtime = + TimestampDifferenceMilliseconds(starttime, GetCurrentTimestamp()); + PgStat_Counter delaytime = (PgStat_Counter) rint(VacuumDelayTime - startdelaytime); + + /* Backend-local measurements; cumulative reporting follows separately. */ + (void) elapsedtime; + (void) delaytime; } /* @@ -551,6 +603,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs int i; Relation *Irel; int nindexes; + bool final_cleanup = + (options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE; Assert(RelationStorageIsAO(aoRelation)); @@ -581,29 +635,37 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; + scan_index(Irel[i], aoRelation, elevel, bstrategy); + ao_measure_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } else { for (i = 0; i < nindexes; i++) { + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; + vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); + ao_measure_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } } vac_close_indexes(nindexes, Irel, NoLock); pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, - ((options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE) ? + final_cleanup ? PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP : PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP); return nindexes; } @@ -721,7 +783,7 @@ appendonly_tid_reaped(ItemPointer itemptr, void *state) static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) { FileSegTotals *fstotal; BlockNumber nblocks; diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 71f954865b2..40f6d937cd5 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -29,11 +29,19 @@ */ typedef struct AOVacuumRelStats { - int nbytes_truncated; /* current # of bytes truncated from segment file */ - int num_dead_tuples; /* current # of dead tuples */ + int64 nbytes_truncated; /* current # of bytes truncated from segment file */ + int64 num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ - /* when the first phase started, for the cumulative vacuum time */ + /* Legacy timing-report baseline, retained until elapsed-time reporting. */ TimestampTz starttime; + /* Active phase durations and delays in milliseconds, excluding phase gaps. */ + double vacuum_time; + double delay_time; + int64 live_tuples; + int64 dead_tuples; + + /* the relation these stats were started for */ + Oid relid; } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); From 0737b53f79cc0e925d8e2fe9b6323ee4e3b2bf88 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:21:13 +0300 Subject: [PATCH 23/50] Accumulate vacuum timing for indexes and databases Record index-pass elapsed and cost-delay time, including parallel workers with the leader's autovacuum classification. Add cumulative table delay time and database totals without adding index time to database totals a second time. Include removed tuple counts in verbose index reports. Catalog getters, views, documentation and SQL-dependent tests follow in the separate native SQL reporting commit. --- src/backend/access/heap/vacuumlazy.c | 34 +++++++++- src/backend/commands/vacuum_ao.c | 3 +- src/backend/commands/vacuumparallel.c | 21 ++++++ src/backend/utils/activity/pgstat_database.c | 5 ++ src/backend/utils/activity/pgstat_relation.c | 68 +++++++++++++++++++- src/include/pgstat.h | 27 +++++++- 6 files changed, 152 insertions(+), 6 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 9b65941352b..4c3917f2759 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -340,6 +340,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, startwritetime = 0; + double startdelaytime; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -360,6 +361,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, /* Used for instrumentation and stats report */ starttime = GetCurrentTimestamp(); + startdelaytime = VacuumDelayTime; pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -617,12 +619,19 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + /* + * The delay counter covers the whole heap_vacuum_rel() run, matching the + * scope of total_vacuum_time. In a parallel vacuum it covers the + * leader's sleeps only; parallel workers account their own sleeps to the + * indexes they process. + */ pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + vacrel->missed_dead_tuples, - starttime); + starttime, + (PgStat_Counter) rint(VacuumDelayTime - startdelaytime)); pgstat_progress_end_command(); if (instrument) @@ -757,8 +766,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, continue; appendStringInfo(&buf, - _("index \"%s\": pages: %u in total, %u newly deleted, %u currently deleted, %u reusable\n"), + _("index \"%s\": tuples: %.0f removed; pages: %u in total, %u newly deleted, %u currently deleted, %u reusable\n"), indnames[i], + istat->tuples_removed, istat->num_pages, istat->pages_newly_deleted, istat->pages_deleted, @@ -2737,6 +2747,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2762,6 +2774,14 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + /* Accumulate this pass into the index's cumulative vacuum times */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2786,6 +2806,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2811,6 +2833,14 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_cleanup_one_index(&ivinfo, istat); + /* Accumulate this pass into the index's cumulative vacuum times */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 53e920958fe..58ee8162322 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -315,7 +315,8 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt onerel->rd_rel->relisshared, reltuples, deadtuples, - vacrelstats->starttime); + vacrelstats->starttime, + 0); /* AO delay time tracking is added in a later commit. */ SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..b95c58aef3f 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -26,6 +26,8 @@ */ #include "postgres.h" +#include + #include "access/amapi.h" #include "access/table.h" #include "access/xact.h" @@ -33,6 +35,7 @@ #include "commands/vacuum.h" #include "optimizer/paths.h" #include "pgstat.h" +#include "postmaster/autovacuum.h" #include "storage/bufmgr.h" #include "tcop/tcopprot.h" #include "utils/lsyscache.h" @@ -64,6 +67,9 @@ typedef struct PVShared Oid relid; int elevel; + /* Leader's autovacuum flag, used for worker timing statistics. */ + bool is_autovacuum; + /* * Fields for both index vacuum and cleanup. * @@ -365,6 +371,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, shared = (PVShared *) shm_toc_allocate(pcxt->toc, est_shared_len); MemSet(shared, 0, est_shared_len); shared->relid = RelationGetRelid(rel); + shared->is_autovacuum = IsAutoVacuumWorkerProcess(); shared->elevel = elevel; shared->maintenance_work_mem_worker = (nindexes_mwm > 0) ? @@ -836,6 +843,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -871,6 +880,18 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + /* + * Accumulate this pass into the index's cumulative vacuum times. Use the + * leader's DSM flag to classify autovacuum: a parallel worker of an + * autovacuum leader is a regular background worker. + */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + pvs->shared->is_autovacuum); + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 813aa359fa0..0f6b9f7f1c4 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -37,6 +37,7 @@ static int pgStatXactRollback = 0; static PgStat_Counter pgLastSessionReportTime = 0; + /* * Remove entry for the database being dropped. */ @@ -417,6 +418,10 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(blk_read_time); PGSTAT_ACCUM_DBCOUNT(blk_write_time); + PGSTAT_ACCUM_DBCOUNT(total_vacuum_time); + PGSTAT_ACCUM_DBCOUNT(total_autovacuum_time); + PGSTAT_ACCUM_DBCOUNT(total_vacuum_delay_time); + PGSTAT_ACCUM_DBCOUNT(total_autovacuum_delay_time); PGSTAT_ACCUM_DBCOUNT(sessions); PGSTAT_ACCUM_DBCOUNT(session_time); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 0a47a18e638..ec67130c336 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -216,7 +216,7 @@ pgstat_drop_relation(Relation rel) void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime) + TimestampTz starttime, PgStat_Counter delaytime) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -259,16 +259,39 @@ pgstat_report_vacuum(Oid tableoid, bool shared, tabentry->last_autovacuum_time = ts; tabentry->autovacuum_count++; tabentry->total_autovacuum_time += elapsedtime; + tabentry->total_autovacuum_delay_time += delaytime; } else { tabentry->last_vacuum_time = ts; tabentry->vacuum_count++; tabentry->total_vacuum_time += elapsedtime; + tabentry->total_vacuum_delay_time += delaytime; } pgstat_unlock_entry(entry_ref); + /* + * Accumulate the same times into the database-wide totals. Index + * processing happens inside the table's run, so per-index times reported + * via pgstat_report_index_vacuum_time() are not added here again. The + * database entry is stored in microseconds, as its other time counters. + */ + { + PgStat_StatDBEntry *dbentry = pgstat_prep_database_pending(dboid); + + if (IsAutoVacuumWorkerProcess()) + { + dbentry->total_autovacuum_time += elapsedtime * 1000; + dbentry->total_autovacuum_delay_time += delaytime * 1000; + } + else + { + dbentry->total_vacuum_time += elapsedtime * 1000; + dbentry->total_vacuum_delay_time += delaytime * 1000; + } + } + /* * Flush IO statistics now. pgstat_report_stat() will flush IO stats, * however this will not be called until after an entire autovacuum cycle @@ -278,6 +301,49 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } +/* + * Report the time spent vacuuming an index. + * + * Vacuum may process an index several times: a bulkdelete pass per index + * scan cycle plus a final cleanup pass, possibly spread across parallel + * workers. Each pass adds its elapsed and delay time here, accumulating + * into the index's total_vacuum_time or total_autovacuum_time, mirroring + * the table-level counters. The caller says whether this is autovacuum: + * parallel workers of an autovacuum leader are regular background workers, + * so IsAutoVacuumWorkerProcess() cannot be relied upon here. + */ +void +pgstat_report_index_vacuum_time(Relation rel, PgStat_Counter elapsedtime, + PgStat_Counter delaytime, bool is_autovacuum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Relation *shtabentry; + PgStat_StatTabEntry *tabentry; + Oid dboid = (rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId); + + if (!pgstat_track_counts) + return; + + /* block acquiring lock for the same reason as pgstat_report_autovac() */ + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, + RelationGetRelid(rel), false); + shtabentry = (PgStatShared_Relation *) entry_ref->shared_stats; + tabentry = &shtabentry->stats; + + if (is_autovacuum) + { + tabentry->total_autovacuum_time += elapsedtime; + tabentry->total_autovacuum_delay_time += delaytime; + } + else + { + tabentry->total_vacuum_time += elapsedtime; + tabentry->total_vacuum_delay_time += delaytime; + } + + pgstat_unlock_entry(entry_ref); +} + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 1193694b330..c114f7eb2e7 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -238,7 +238,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAE +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAF typedef struct PgStat_ArchiverStats { @@ -354,6 +354,17 @@ typedef struct PgStat_StatDBEntry PgStat_Counter sessions_fatal; PgStat_Counter sessions_killed; + /* + * Cumulative time vacuums spent processing tables of this database and + * sleeping in cost-based delay points, in microseconds (the delay parts + * advance only when track_cost_delay_timing is enabled). Times of index + * processing are included in the owning table's run. + */ + PgStat_Counter total_vacuum_time; + PgStat_Counter total_autovacuum_time; + PgStat_Counter total_vacuum_delay_time; + PgStat_Counter total_autovacuum_delay_time; + TimestampTz stat_reset_timestamp; } PgStat_StatDBEntry; @@ -432,6 +443,14 @@ typedef struct PgStat_StatTabEntry PgStat_Counter total_autovacuum_time; PgStat_Counter total_analyze_time; PgStat_Counter total_autoanalyze_time; + + /* + * Time slept in cost-based vacuum delay points while vacuuming this + * relation, in milliseconds; nonzero only when track_cost_delay_timing + * is enabled. + */ + PgStat_Counter total_vacuum_delay_time; + PgStat_Counter total_autovacuum_delay_time; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -657,7 +676,11 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime); + TimestampTz starttime, PgStat_Counter delaytime); +extern void pgstat_report_index_vacuum_time(Relation rel, + PgStat_Counter elapsedtime, + PgStat_Counter delaytime, + bool is_autovacuum); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); From ced04c7846e3f7003376090ddc68c34e850754e5 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:21:14 +0300 Subject: [PATCH 24/50] Count vacuums that enter wraparound failsafe mode Accumulate relation and database failsafe counts for completed heap vacuums. Pass false for AO, which has no heap wraparound failsafe mode. Keep this counter distinct from scans made aggressive by the freeze age. Catalog getters, views and tests follow in the native SQL reporting commit. --- src/backend/access/heap/vacuumlazy.c | 3 ++- src/backend/commands/vacuum_ao.c | 3 ++- src/backend/utils/activity/pgstat_database.c | 1 + src/backend/utils/activity/pgstat_relation.c | 9 ++++++++- src/include/pgstat.h | 17 +++++++++++++++-- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 4c3917f2759..22e077d4e26 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -631,7 +631,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->recently_dead_tuples + vacrel->missed_dead_tuples, starttime, - (PgStat_Counter) rint(VacuumDelayTime - startdelaytime)); + (PgStat_Counter) rint(VacuumDelayTime - startdelaytime), + VacuumFailsafeActive); pgstat_progress_end_command(); if (instrument) diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 58ee8162322..3b1f37335b7 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -316,7 +316,8 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt reltuples, deadtuples, vacrelstats->starttime, - 0); /* AO delay time tracking is added in a later commit. */ + 0, /* AO delay time tracking is added in a later commit. */ + false); /* no failsafe mode for AO tables */ SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 0f6b9f7f1c4..2ec9d11f319 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -422,6 +422,7 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(total_autovacuum_time); PGSTAT_ACCUM_DBCOUNT(total_vacuum_delay_time); PGSTAT_ACCUM_DBCOUNT(total_autovacuum_delay_time); + PGSTAT_ACCUM_DBCOUNT(vacuum_failsafe_count); PGSTAT_ACCUM_DBCOUNT(sessions); PGSTAT_ACCUM_DBCOUNT(session_time); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index ec67130c336..3de04c945c8 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -216,7 +216,8 @@ pgstat_drop_relation(Relation rel) void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime, PgStat_Counter delaytime) + TimestampTz starttime, PgStat_Counter delaytime, + bool failsafe) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -269,6 +270,9 @@ pgstat_report_vacuum(Oid tableoid, bool shared, tabentry->total_vacuum_delay_time += delaytime; } + if (failsafe) + tabentry->vacuum_failsafe_count++; + pgstat_unlock_entry(entry_ref); /* @@ -290,6 +294,9 @@ pgstat_report_vacuum(Oid tableoid, bool shared, dbentry->total_vacuum_time += elapsedtime * 1000; dbentry->total_vacuum_delay_time += delaytime * 1000; } + + if (failsafe) + dbentry->vacuum_failsafe_count++; } /* diff --git a/src/include/pgstat.h b/src/include/pgstat.h index c114f7eb2e7..7c2dcc34aa5 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -238,7 +238,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAF +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB0 typedef struct PgStat_ArchiverStats { @@ -365,6 +365,12 @@ typedef struct PgStat_StatDBEntry PgStat_Counter total_vacuum_delay_time; PgStat_Counter total_autovacuum_delay_time; + /* + * Number of vacuums in this database that entered the wraparound + * failsafe mode (see vacuum_failsafe_age). + */ + PgStat_Counter vacuum_failsafe_count; + TimestampTz stat_reset_timestamp; } PgStat_StatDBEntry; @@ -451,6 +457,12 @@ typedef struct PgStat_StatTabEntry */ PgStat_Counter total_vacuum_delay_time; PgStat_Counter total_autovacuum_delay_time; + + /* + * Number of vacuums of this relation that entered the wraparound + * failsafe mode (see vacuum_failsafe_age). + */ + PgStat_Counter vacuum_failsafe_count; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -676,7 +688,8 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, - TimestampTz starttime, PgStat_Counter delaytime); + TimestampTz starttime, PgStat_Counter delaytime, + bool failsafe); extern void pgstat_report_index_vacuum_time(Relation rel, PgStat_Counter elapsedtime, PgStat_Counter delaytime, From 0a1024142e4fac62bf8926db3cb5a6af8a58038f Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 9 Sep 2026 14:49:54 +0300 Subject: [PATCH 25/50] Measure heap and index vacuum work per call Assemble backend-local tuple and page measurements and take per-call snapshots of the existing index results. Access methods accumulate results across passes, so retain each pass's delta without counting earlier work again. Share the measurement helper between serial and parallel vacuum. Keep SP-GiST's newly deleted page count cumulative within one vacuum, without counting pages that were already empty on entry. Handle AMs that reset their newly deleted page count between passes. Adapt to PostgreSQL 16's existing recently_dead_tuples, missed_dead_tuples, pages_scanned, pages_removed and missed_dead_pages counters. Assemble these measurements independently of optional hooks; storage and SQL follow in separate reporting commits. Adapted-from: f2592a3a09f77a64873b0bf3593ae91991db5430 --- src/backend/access/heap/vacuumlazy.c | 39 +++++++++++++++++++++++++++ src/backend/access/spgist/spgvacuum.c | 5 +++- src/backend/commands/vacuum.c | 18 +++++++++++++ src/backend/commands/vacuumparallel.c | 15 +++++++++++ src/include/commands/vacuum.h | 5 ++++ src/include/pgstat.h | 14 ++++++++++ 6 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..476711ed388 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -605,6 +605,19 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false, true); + /* Assemble backend-local measurements for subsequent reporting. */ + { + PgStat_VacuumStats stats = {0}; + + stats.tuples_deleted = vacrel->tuples_deleted; + stats.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + stats.recently_dead_tuples = vacrel->recently_dead_tuples; + stats.missed_dead_tuples = vacrel->missed_dead_tuples; + stats.pages_scanned = vacrel->scanned_pages; + stats.pages_removed = vacrel->removed_pages; + stats.missed_dead_pages = vacrel->missed_dead_pages; + } + /* * Report results to the cumulative stats system, too. * @@ -2734,7 +2747,18 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + double prev_tuples_removed = 0; + BlockNumber prev_pages_newly_deleted = 0; + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + { + prev_tuples_removed = istat->tuples_removed; + prev_pages_newly_deleted = istat->pages_newly_deleted; + } ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2758,6 +2782,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + vacuum_measure_index_stats(indrel, istat, prev_tuples_removed, + prev_pages_newly_deleted, false); /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); @@ -2783,7 +2809,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + double prev_tuples_removed = 0; + BlockNumber prev_pages_newly_deleted = 0; + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + { + prev_tuples_removed = istat->tuples_removed; + prev_pages_newly_deleted = istat->pages_newly_deleted; + } ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2807,6 +2844,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + vacuum_measure_index_stats(indrel, istat, prev_tuples_removed, + prev_pages_newly_deleted, true); /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index 85bc02b92c8..116574c9088 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -624,6 +624,7 @@ spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) Relation index = bds->info->index; Buffer buffer; Page page; + bool was_empty; /* call vacuum_delay_point while not holding any buffer lock */ vacuum_delay_point(); @@ -632,6 +633,7 @@ spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) RBM_NORMAL, bds->info->strategy); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); page = (Page) BufferGetPage(buffer); + was_empty = PageIsNew(page) || PageIsEmpty(page); if (PageIsNew(page)) { @@ -675,6 +677,8 @@ spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) { RecordFreeIndexPage(index, blkno); bds->stats->pages_deleted++; + if (!was_empty) + bds->stats->pages_newly_deleted++; } else { @@ -902,7 +906,6 @@ spgvacuumscan(spgBulkDeleteState *bds) /* Report final stats */ bds->stats->num_pages = num_pages; - bds->stats->pages_newly_deleted = bds->stats->pages_deleted; bds->stats->pages_free = bds->stats->pages_deleted; } diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index f4ba5615d5a..6058bbe859d 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -3914,3 +3914,21 @@ vac_cmp_itemptr(const void *left, const void *right) return 0; } + +/* Measure index work once per call and remaining dead pages at cleanup. */ +void +vacuum_measure_index_stats(Relation indrel, const IndexBulkDeleteResult *istat, + double prev_tuples_removed, + BlockNumber prev_pages_newly_deleted, bool cleanup) +{ + PgStat_VacuumStats stats = {0}; + + if (istat == NULL) + return; + + stats.tuples_deleted = istat->tuples_removed - prev_tuples_removed; + stats.pages_deleted = + istat->pages_newly_deleted >= prev_pages_newly_deleted ? + istat->pages_newly_deleted - prev_pages_newly_deleted : + istat->pages_newly_deleted; +} diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..3c037e4eaee 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -836,6 +836,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + double prev_tuples_removed = 0; + BlockNumber prev_pages_newly_deleted = 0; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -844,6 +846,15 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + /* + * Snapshot the running bulkdelete totals: an index may be processed + * several times per vacuum, and the report below covers this pass only. + */ + if (istat != NULL) + { + prev_tuples_removed = istat->tuples_removed; + prev_pages_newly_deleted = istat->pages_newly_deleted; + } ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -871,6 +882,10 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + vacuum_measure_index_stats(indrel, istat_res, prev_tuples_removed, + prev_pages_newly_deleted, + indstats->status == PARALLEL_INDVAC_STATUS_NEED_CLEANUP); + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..e2f928a94cd 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -491,6 +491,11 @@ extern Size vac_max_items_to_alloc_size(int max_items); /* In postmaster/autovacuum.c */ extern void AutoVacuumUpdateCostLimit(void); extern void VacuumUpdateCosts(void); +extern void vacuum_measure_index_stats(Relation indrel, + const IndexBulkDeleteResult *istat, + double prev_tuples_removed, + BlockNumber prev_pages_newly_deleted, bool cleanup); + /* in commands/vacuumparallel.c */ extern ParallelVacuumState *parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, int nrequested_workers, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..ca6e3651a42 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -91,6 +91,20 @@ typedef enum SessionEndType */ typedef int64 PgStat_Counter; +/* Backend-local work measurements for one VACUUM operation. */ +typedef struct PgStat_VacuumStats +{ + PgStat_Counter tuples_deleted; + PgStat_Counter dead_tuples; + PgStat_Counter pages_deleted; + PgStat_Counter recently_dead_tuples; + PgStat_Counter missed_dead_tuples; + PgStat_Counter pages_scanned; + PgStat_Counter pages_removed; + PgStat_Counter missed_dead_pages; +} PgStat_VacuumStats; + + /* ------------------------------------------------------------ * Structures kept in backend local memory while accumulating counts From 489a7648d3d40d1f841d7e2cd3b9f128a4669aec Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:21:15 +0300 Subject: [PATCH 26/50] Count vacuums interrupted by errors Count ERROR reports in the heap vacuum callback using local counters for database-local and shared relations. Transfer them to pending database statistics at transaction end. Test cancellation of a running vacuum. --- src/backend/access/heap/vacuumlazy.c | 4 ++ src/backend/utils/activity/pgstat_database.c | 55 ++++++++++++++++++++ src/include/pgstat.h | 6 ++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 22e077d4e26..326bd03eba6 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -3453,6 +3453,10 @@ vacuum_error_callback(void *arg) { LVRelState *errinfo = arg; + /* Ignore non-error reports that also invoke this callback. */ + if (errinfo->rel != NULL && geterrlevel() == ERROR) + pgstat_count_vacuum_error(errinfo->rel->rd_rel->relisshared); + switch (errinfo->phase) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 2ec9d11f319..a320a735c22 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -36,6 +36,12 @@ static int pgStatXactCommit = 0; static int pgStatXactRollback = 0; static PgStat_Counter pgLastSessionReportTime = 0; +/* + * Vacuums interrupted by an error, counted from the vacuum error callback + * and folded into the pending database entries at transaction end. + */ +static PgStat_Counter pgStatVacuumErrors = 0; +static PgStat_Counter pgStatSharedVacuumErrors = 0; /* @@ -246,6 +252,20 @@ pgstat_fetch_stat_dbentry(Oid dboid) pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid); } +/* + * Add the vacuum errors counted for a database to its pending entry, and + * reset the counter. + */ +static void +pgstat_fold_vacuum_errors(Oid dboid, PgStat_Counter *count) +{ + if (*count == 0) + return; + + pgstat_prep_database_pending(dboid)->vacuum_interrupt_count += *count; + *count = 0; +} + void AtEOXact_PgStat_Database(bool isCommit, bool parallel) { @@ -261,6 +281,15 @@ AtEOXact_PgStat_Database(bool isCommit, bool parallel) else pgStatXactRollback++; } + + /* + * Fold in the vacuums that pgstat_count_vacuum_error() saw failing. We + * are past LWLockReleaseAll() of the aborting transaction here, so it is + * safe to touch the pending entries, which the next pgstat_report_stat() + * flushes to shared memory as usual. + */ + pgstat_fold_vacuum_errors(MyDatabaseId, &pgStatVacuumErrors); + pgstat_fold_vacuum_errors(InvalidOid, &pgStatSharedVacuumErrors); } /* @@ -327,6 +356,31 @@ pgstat_should_report_connstat(void) return MyBackendType == B_BACKEND; } +/* + * Count a vacuum that was interrupted by an error. + * + * This is called from the vacuum error callback, that is from inside the + * error handler, so it must not do anything that could fail or block: the + * error may well have been raised by the locking code itself, possibly + * while holding an LWLock. Just bump a counter here; the counter reaches + * the pending database entry in AtEOXact_PgStat_Database(), once the + * transaction end has released the locks we might have been holding. + * + * Vacuums of shared relations are counted in the InvalidOid entry, matching + * how their other stats are accounted. + */ +void +pgstat_count_vacuum_error(bool shared) +{ + if (!pgstat_track_counts) + return; + + if (shared) + pgStatSharedVacuumErrors++; + else + pgStatVacuumErrors++; +} + /* * Find or create a local PgStat_StatDBEntry entry for dboid. */ @@ -423,6 +477,7 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(total_vacuum_delay_time); PGSTAT_ACCUM_DBCOUNT(total_autovacuum_delay_time); PGSTAT_ACCUM_DBCOUNT(vacuum_failsafe_count); + PGSTAT_ACCUM_DBCOUNT(vacuum_interrupt_count); PGSTAT_ACCUM_DBCOUNT(sessions); PGSTAT_ACCUM_DBCOUNT(session_time); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7c2dcc34aa5..24ed05c25dd 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -238,7 +238,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB0 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB1 typedef struct PgStat_ArchiverStats { @@ -371,6 +371,9 @@ typedef struct PgStat_StatDBEntry */ PgStat_Counter vacuum_failsafe_count; + /* # of vacuums in this database interrupted by errors */ + PgStat_Counter vacuum_interrupt_count; + TimestampTz stat_reset_timestamp; } PgStat_StatDBEntry; @@ -694,6 +697,7 @@ extern void pgstat_report_index_vacuum_time(Relation rel, PgStat_Counter elapsedtime, PgStat_Counter delaytime, bool is_autovacuum); +extern void pgstat_count_vacuum_error(bool shared); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter, TimestampTz starttime); From bb18bc26b40b15d9aaeb2a256af7b8092472fdc5 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 9 Sep 2026 14:49:54 +0300 Subject: [PATCH 27/50] Count vacuum pages with unremovable dead tuples Introduce dead_pages in LVRelState and PgStat_VacuumStats. Count heap pages that retain recently dead tuples in both pruning and no-prune scans. Keep this measurement distinct from pages with dead tuples missed because of buffer pins. For index cleanup, derive the count from deleted pages that are not yet reusable. Cumulative statistics storage and SQL exposure follow separately. Adapted-from: 37a475369a033094480a7791d44b11983a247a8b --- src/backend/access/heap/vacuumlazy.c | 6 ++++++ src/backend/commands/vacuum.c | 2 ++ src/include/pgstat.h | 1 + 3 files changed, 9 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 476711ed388..51eb83a3eec 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -207,6 +207,7 @@ typedef struct LVRelState BlockNumber removed_pages; /* # pages removed by relation truncation */ BlockNumber frozen_pages; /* # pages with newly frozen tuples */ BlockNumber lpdead_item_pages; /* # pages with LP_DEAD items */ + BlockNumber dead_pages; /* # pages with not-yet-removable tuples */ BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ @@ -611,6 +612,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, stats.tuples_deleted = vacrel->tuples_deleted; stats.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + stats.dead_pages = vacrel->dead_pages; stats.recently_dead_tuples = vacrel->recently_dead_tuples; stats.missed_dead_tuples = vacrel->missed_dead_tuples; stats.pages_scanned = vacrel->scanned_pages; @@ -1969,6 +1971,8 @@ lazy_scan_prune(LVRelState *vacrel, vacrel->lpdead_items += lpdead_items; vacrel->live_tuples += live_tuples; vacrel->recently_dead_tuples += recently_dead_tuples; + if (recently_dead_tuples > 0) + vacrel->dead_pages++; } /* @@ -2203,6 +2207,8 @@ lazy_scan_noprune(LVRelState *vacrel, */ vacrel->live_tuples += live_tuples; vacrel->recently_dead_tuples += recently_dead_tuples; + if (recently_dead_tuples > 0) + vacrel->dead_pages++; vacrel->missed_dead_tuples += missed_dead_tuples; if (missed_dead_tuples > 0) vacrel->missed_dead_pages++; diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 6058bbe859d..5821a16b33a 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -3931,4 +3931,6 @@ vacuum_measure_index_stats(Relation indrel, const IndexBulkDeleteResult *istat, istat->pages_newly_deleted >= prev_pages_newly_deleted ? istat->pages_newly_deleted - prev_pages_newly_deleted : istat->pages_newly_deleted; + if (cleanup && istat->pages_deleted > istat->pages_free) + stats.dead_pages = istat->pages_deleted - istat->pages_free; } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index ca6e3651a42..c6ffce6e962 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -97,6 +97,7 @@ typedef struct PgStat_VacuumStats PgStat_Counter tuples_deleted; PgStat_Counter dead_tuples; PgStat_Counter pages_deleted; + PgStat_Counter dead_pages; PgStat_Counter recently_dead_tuples; PgStat_Counter missed_dead_tuples; PgStat_Counter pages_scanned; From 2d59d699c28b9db5622eb5843cdad256bcecfde3 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 30 Mar 2026 09:07:24 +0300 Subject: [PATCH 28/50] Track table VM stability. Add visible_page_marks_cleared and frozen_page_marks_cleared counters to pg_stat_all_tables tracking the number of times the all-visible and all-frozen bits are cleared in the visibility map. These bits are cleared by backend processes during regular DML operations. Hence, the counters are placed in table statistic entry. A high visible_page_marks_cleared rate relative to DML volume indicates that modifications are scattered across previously-clean pages rather than concentrated on already-dirty ones, causing index-only scans to fall back to heap fetches. A high frozen_page_marks_cleared rate indicates that vacuum's freezing work is being frequently undone by concurrent DML. Authors: Alena Rybakina , Andrei Lepikhov , Andrei Zubkov Reviewed-by: Dilip Kumar , Masahiko Sawada , Ilia Evdokimov , Jian He , Kirill Reshke , Alexander Korotkov , Jim Nasby , Sami Imseih , Karina Litskevich , Andrey Borodin Backported-from: https://www.postgresql.org/message-id/attachment/205562/v45-0009-Track-table-VM-stability.patch Co-authored-by: Andrei Lepikhov Co-authored-by: Andrei Zubkov --- src/backend/access/heap/heapam.c | 16 +++---- src/backend/access/heap/vacuumlazy.c | 4 +- src/backend/access/heap/visibilitymap.c | 46 ++++++++++++++++++++ src/backend/utils/activity/pgstat_relation.c | 2 + src/include/access/visibilitymap.h | 2 + src/include/pgstat.h | 32 +++++++++++++- 6 files changed, 91 insertions(+), 11 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 19c69270371..7fd1244ed66 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1974,7 +1974,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, + visibilitymap_clear_rel(relation, ItemPointerGetBlockNumber(&(heaptup->t_self)), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -2350,7 +2350,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -2935,7 +2935,7 @@ heap_delete(Relation relation, ItemPointer tid, { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -3764,7 +3764,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * worthwhile. */ if (PageIsAllVisible(page) && - visibilitymap_clear(relation, block, vmbuffer, + visibilitymap_clear_rel(relation, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; @@ -3998,14 +3998,14 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf))) { all_visible_cleared_new = true; PageClearAllVisible(BufferGetPage(newbuf)); - visibilitymap_clear(relation, BufferGetBlockNumber(newbuf), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(newbuf), vmbuffer_new, VISIBILITYMAP_VALID_BITS); } @@ -5095,7 +5095,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, /* Clear only the all-frozen bit on visibility map if needed */ if (PageIsAllVisible(page) && - visibilitymap_clear(relation, block, vmbuffer, + visibilitymap_clear_rel(relation, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; @@ -5852,7 +5852,7 @@ heap_lock_updated_tuple_rec(Relation rel, ItemPointer tid, TransactionId xid, &new_xmax, &new_infomask, &new_infomask2); if (PageIsAllVisible(BufferGetPage(buf)) && - visibilitymap_clear(rel, block, vmbuffer, + visibilitymap_clear_rel(rel, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 326bd03eba6..feab6d5175f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -1165,7 +1165,7 @@ lazy_scan_heap(LVRelState *vacrel) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", vacrel->relname, blkno); - visibilitymap_clear(vacrel->rel, blkno, vmbuffer, + visibilitymap_clear_rel(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1190,7 +1190,7 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); - visibilitymap_clear(vacrel->rel, blkno, vmbuffer, + visibilitymap_clear_rel(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 2e18cd88bcf..562c7fdc10f 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -12,6 +12,8 @@ * * INTERFACE ROUTINES * visibilitymap_clear - clear bits for one page in the visibility map + * visibilitymap_clear_rel - clear bits for an open relation, counting the + * clears in its statistics * visibilitymap_pin - pin a map page for setting a bit * visibilitymap_pin_ok - check whether correct map page is already pinned * visibilitymap_set - set a bit in a previously pinned page @@ -91,6 +93,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" @@ -125,6 +128,8 @@ * bit pair */ /* prototypes for internal routines */ +static bool vm_do_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, + uint8 flags, bool count_stats); static Buffer vm_readbuf(Relation rel, BlockNumber blkno, bool extend); static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks); @@ -135,9 +140,38 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks); * You must pass a buffer containing the correct map page to this function. * Call visibilitymap_pin first to pin the right one. This function doesn't do * any I/O. Returns true if any bits have been cleared and false otherwise. + * + * This variant is for callers such as recovery, which use a fake relcache + * entry and cannot count the clears in the relation's statistics; callers + * holding an open relation should use visibilitymap_clear_rel() instead. */ bool visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags) +{ + return vm_do_clear(rel, heapBlk, vmbuf, flags, false); +} + +/* + * visibilitymap_clear_rel - visibilitymap_clear() for an open relation + * + * Same as visibilitymap_clear(), but additionally counts the cleared + * all-visible and all-frozen marks in the relation's statistics, which + * makes the stability of its visibility map observable. + */ +bool +visibilitymap_clear_rel(Relation rel, BlockNumber heapBlk, Buffer vmbuf, + uint8 flags) +{ + return vm_do_clear(rel, heapBlk, vmbuf, flags, true); +} + +/* + * Workhorse of visibilitymap_clear() and visibilitymap_clear_rel(). + * count_stats is false when the caller has no real relcache entry at hand. + */ +static bool +vm_do_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags, + bool count_stats) { BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); @@ -162,6 +196,18 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * Track how often all-visible or all-frozen bits are cleared in the + * visibility map. + */ + if (count_stats) + { + if (map[mapByte] & ((flags & VISIBILITYMAP_ALL_VISIBLE) << mapOffset)) + pgstat_count_visible_page_marks_cleared(rel); + if (map[mapByte] & ((flags & VISIBILITYMAP_ALL_FROZEN) << mapOffset)) + pgstat_count_frozen_page_marks_cleared(rel); + } + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 3de04c945c8..95423272731 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -920,6 +920,8 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->ins_since_vacuum += lstats->counts.tuples_inserted; tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->visible_page_marks_cleared += lstats->counts.visible_page_marks_cleared; + tabentry->frozen_page_marks_cleared += lstats->counts.frozen_page_marks_cleared; /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h index daaa01a2578..6af0fd5304d 100644 --- a/src/include/access/visibilitymap.h +++ b/src/include/access/visibilitymap.h @@ -26,6 +26,8 @@ #define VM_ALL_FROZEN(r, b, v) \ ((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0) +extern bool visibilitymap_clear_rel(Relation rel, BlockNumber heapBlk, + Buffer vmbuf, uint8 flags); extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags); extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 24ed05c25dd..fa86ac4b218 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -180,6 +180,14 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + /* + * Revocations of all-visible and all-frozen marks in the visibility map. + * The bits are cleared by the page modification itself, so these count + * whether or not the transaction commits. + */ + PgStat_Counter visible_page_marks_cleared; + PgStat_Counter frozen_page_marks_cleared; } PgStat_TableCounts; /* ---------- @@ -238,7 +246,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB1 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB2 typedef struct PgStat_ArchiverStats { @@ -438,6 +446,8 @@ typedef struct PgStat_StatTabEntry PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + PgStat_Counter visible_page_marks_cleared; + PgStat_Counter frozen_page_marks_cleared; TimestampTz last_vacuum_time; /* user initiated vacuum */ PgStat_Counter vacuum_count; @@ -749,6 +759,26 @@ extern void pgstat_report_analyze(Relation rel, (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* + * Count revocations of all-visible and all-frozen marks in the visibility + * map. Only tables have a visibility map, so these always work on the + * table's pending counters. + */ +#define pgstat_count_visible_page_marks_cleared(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + { \ + (rel)->pgstat_info->counts.visible_page_marks_cleared++; \ + } \ + } while (0) +#define pgstat_count_frozen_page_marks_cleared(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + { \ + (rel)->pgstat_info->counts.frozen_page_marks_cleared++; \ + } \ + } while (0) + extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); extern void pgstat_count_heap_delete(Relation rel); From 4a259e1bbf8e10cd09a894476b932b1217baa70f Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 9 Sep 2026 14:49:54 +0300 Subject: [PATCH 29/50] Count heap pages where vacuum freezes tuples Include pages_frozen and tuples_frozen in the backend-local VACUUM measurements. PostgreSQL 16 already counts frozen_pages and tuples_frozen and prints them in VACUUM VERBOSE, so reuse those counters without adding a second increment at the freezing site. Cumulative statistics storage and SQL exposure follow separately. Adapted-from: 93ae8117a6d5d45d0e2a20a0677724b38103e4e6 --- src/backend/access/heap/vacuumlazy.c | 2 ++ src/include/pgstat.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 51eb83a3eec..bbf76ba608f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -613,6 +613,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, stats.tuples_deleted = vacrel->tuples_deleted; stats.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; stats.dead_pages = vacrel->dead_pages; + stats.pages_frozen = vacrel->frozen_pages; + stats.tuples_frozen = vacrel->tuples_frozen; stats.recently_dead_tuples = vacrel->recently_dead_tuples; stats.missed_dead_tuples = vacrel->missed_dead_tuples; stats.pages_scanned = vacrel->scanned_pages; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index c6ffce6e962..1fd511fe28f 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -98,6 +98,8 @@ typedef struct PgStat_VacuumStats PgStat_Counter dead_tuples; PgStat_Counter pages_deleted; PgStat_Counter dead_pages; + PgStat_Counter pages_frozen; + PgStat_Counter tuples_frozen; PgStat_Counter recently_dead_tuples; PgStat_Counter missed_dead_tuples; PgStat_Counter pages_scanned; From 1e86b0486df3ed163ab954ada80d7f0c4630e780 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:21:15 +0300 Subject: [PATCH 30/50] Report core vacuum timing for append-optimized tables Connect AO index-pass timing measurements to cumulative statistics and report interrupted vacuums. Add cumulative delay reporting for AO tables. Failsafe and visibility-map counters remain inapplicable to AO. The cluster isolation test is included later with its gp_stat_vacuum dependency. Active-phase elapsed reporting is connected in a later commit. --- src/backend/commands/vacuum_ao.c | 52 +++++++++++++++++----- src/include/access/appendonly_compaction.h | 1 + 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 3b1f37335b7..f644b737009 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -165,8 +165,9 @@ static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *p BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static AOVacuumRelStats *init_vacrelstats(void); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); -static void ao_measure_index_vacuum_time(Relation indrel, TimestampTz starttime, +static void ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, double startdelaytime); +static void ao_vacuum_error_callback(void *arg); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -316,7 +317,8 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt reltuples, deadtuples, vacrelstats->starttime, - 0, /* AO delay time tracking is added in a later commit. */ + (PgStat_Counter) rint(VacuumDelayTime - + vacrelstats->startdelaytime), false); /* no failsafe mode for AO tables */ @@ -427,6 +429,7 @@ init_vacrelstats() old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); vacrelstats->starttime = GetCurrentTimestamp(); + vacrelstats->startdelaytime = VacuumDelayTime; MemoryContextSwitchTo(old_context); return vacrelstats; @@ -441,6 +444,7 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + ErrorContextCallback errcallback; instr_time phase_start; instr_time phase_end; double phase_start_delay; @@ -482,6 +486,12 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy vacrelstats->relid = RelationGetRelid(rel); } + /* Count the vacuums of the relation that an error interrupts */ + errcallback.callback = ao_vacuum_error_callback; + errcallback.arg = rel; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + INSTR_TIME_SET_CURRENT(phase_start); phase_start_delay = VacuumDelayTime; @@ -511,6 +521,25 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } + error_context_stack = errcallback.previous; +} + +/* + * Error context callback of an append-optimized vacuum phase. + * + * Like vacuum_error_callback() of the heap, count a vacuum interrupted by an + * actual ERROR, not by a lower-severity report that merely carries this + * context, in vacuum_interrupt_count of the database. We are inside the + * error handler, so pgstat_count_vacuum_error() only bumps a counter. The + * callback adds no context line: the phases report their own progress. + */ +static void +ao_vacuum_error_callback(void *arg) +{ + Relation rel = (Relation) arg; + + if (geterrlevel() == ERROR) + pgstat_count_vacuum_error(rel->rd_rel->relisshared); } /* @@ -519,16 +548,15 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy * lazy_vacuum_one_index() and lazy_cleanup_one_index(). */ static void -ao_measure_index_vacuum_time(Relation indrel, TimestampTz starttime, +ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, double startdelaytime) { - PgStat_Counter elapsedtime = - TimestampDifferenceMilliseconds(starttime, GetCurrentTimestamp()); - PgStat_Counter delaytime = (PgStat_Counter) rint(VacuumDelayTime - startdelaytime); - - /* Backend-local measurements; cumulative reporting follows separately. */ - (void) elapsedtime; - (void) delaytime; + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(starttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); } /* @@ -644,7 +672,7 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs aoRelation, elevel, bstrategy); - ao_measure_index_vacuum_time(Irel[i], istarttime, startdelaytime); + ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } else @@ -660,7 +688,7 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs elevel, bstrategy, vacrelstats); - ao_measure_index_vacuum_time(Irel[i], istarttime, startdelaytime); + ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } } diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 40f6d937cd5..ec3161854b3 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -34,6 +34,7 @@ typedef struct AOVacuumRelStats int num_index_vacuumed; /* current # of indexes been vacuumed */ /* Legacy timing-report baseline, retained until elapsed-time reporting. */ TimestampTz starttime; + double startdelaytime; /* Active phase durations and delays in milliseconds, excluding phase gaps. */ double vacuum_time; double delay_time; From fce9e8dc6b483a7b333d8b7ba82c6436a7ace7e5 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 9 Sep 2026 14:49:54 +0300 Subject: [PATCH 31/50] Count heap pages marked all-visible by vacuum Add the backend-local pages_all_visible measurement. Count new visibility map marks under the heap page lock, including empty pages and second-pass cleanup. Test the current VM bit, rather than a potentially stale skipping decision, and do not recount existing marks or all-frozen-only upgrades. Cumulative statistics storage and SQL exposure follow separately. Adapted-from: 3aefc13e227940be30f42b781bee1d30c1148868 --- src/backend/access/heap/vacuumlazy.c | 15 +++++++++++++++ src/include/pgstat.h | 1 + 2 files changed, 16 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index bbf76ba608f..0fc1796bb68 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -206,6 +206,7 @@ typedef struct LVRelState BlockNumber scanned_pages; /* # pages examined (not skipped via VM) */ BlockNumber removed_pages; /* # pages removed by relation truncation */ BlockNumber frozen_pages; /* # pages with newly frozen tuples */ + BlockNumber all_visible_pages; /* # pages newly marked all-visible in VM */ BlockNumber lpdead_item_pages; /* # pages with LP_DEAD items */ BlockNumber dead_pages; /* # pages with not-yet-removable tuples */ BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ @@ -443,6 +444,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->scanned_pages = 0; vacrel->removed_pages = 0; vacrel->frozen_pages = 0; + vacrel->all_visible_pages = 0; vacrel->lpdead_item_pages = 0; vacrel->missed_dead_pages = 0; vacrel->nonempty_pages = 0; @@ -614,6 +616,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, stats.dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; stats.dead_pages = vacrel->dead_pages; stats.pages_frozen = vacrel->frozen_pages; + stats.pages_all_visible = vacrel->all_visible_pages; stats.tuples_frozen = vacrel->tuples_frozen; stats.recently_dead_tuples = vacrel->recently_dead_tuples; stats.missed_dead_tuples = vacrel->missed_dead_tuples; @@ -1152,6 +1155,12 @@ lazy_scan_heap(LVRelState *vacrel) */ PageSetAllVisible(page); MarkBufferDirty(buf); + /* + * Count only new VM marks, using the current bit under the heap + * lock rather than the possibly stale lazy_scan_skip() result. + */ + if (!VM_ALL_VISIBLE(vacrel->rel, blkno, &vmbuffer)) + vacrel->all_visible_pages++; visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, vmbuffer, prunestate.visibility_cutoff_xid, flags); @@ -1225,6 +1234,8 @@ lazy_scan_heap(LVRelState *vacrel) * safe for REDO was logged when the page's tuples were frozen. */ Assert(!TransactionIdIsValid(prunestate.visibility_cutoff_xid)); + if (!VM_ALL_VISIBLE(vacrel->rel, blkno, &vmbuffer)) + vacrel->all_visible_pages++; visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, vmbuffer, InvalidTransactionId, VISIBILITYMAP_ALL_VISIBLE | @@ -1533,6 +1544,8 @@ lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, log_newpage_buffer(buf, true); PageSetAllVisible(page); + if (!VM_ALL_VISIBLE(vacrel->rel, blkno, &vmbuffer)) + vacrel->all_visible_pages++; visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, vmbuffer, InvalidTransactionId, VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN); @@ -2637,6 +2650,8 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, } PageSetAllVisible(page); + if (!VM_ALL_VISIBLE(vacrel->rel, blkno, &vmbuffer)) + vacrel->all_visible_pages++; visibilitymap_set(vacrel->rel, blkno, buffer, InvalidXLogRecPtr, vmbuffer, visibility_cutoff_xid, flags); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 1fd511fe28f..eb52e290f16 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -99,6 +99,7 @@ typedef struct PgStat_VacuumStats PgStat_Counter pages_deleted; PgStat_Counter dead_pages; PgStat_Counter pages_frozen; + PgStat_Counter pages_all_visible; PgStat_Counter tuples_frozen; PgStat_Counter recently_dead_tuples; PgStat_Counter missed_dead_tuples; From f7c350d8f337cbf40cbbc0d3eff3c5becc6a3811 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 8 Oct 2026 00:00:00 +0300 Subject: [PATCH 32/50] Document cost-based VACUUM delay timing Describe track_cost_delay_timing alongside the instrumentation it enables. Give the accumulated instrumentation fields their own statistics-file format identifier independent of the pluggable-statistics series. --- doc/src/sgml/config.sgml | 24 ++++++++++++++++++++++++ src/include/pgstat.h | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 3b8b60d9347..1873cf1f972 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8155,6 +8155,30 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + track_cost_delay_timing (boolean) + + track_cost_delay_timing configuration parameter + + + + + Enables timing of cost-based vacuum delay (see + ). This parameter + is off by default, as it will repeatedly query the operating system for + the current time, which may cause significant overhead on some + platforms. You can use the tool to + measure the overhead of timing on your system. Cost-based vacuum delay + timing information is displayed in + pg_stat_progress_vacuum + and + pg_stat_progress_analyze. + Only superusers and users with the appropriate SET + privilege can change this setting. + + + + track_io_timing (boolean) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index fa86ac4b218..a761000e264 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -246,7 +246,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB2 +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB4 typedef struct PgStat_ArchiverStats { From 3bf458e0434dc470d59de7e0f9d26329bc5bdc20 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 9 Sep 2026 14:49:54 +0300 Subject: [PATCH 33/50] Count vacuum runs made aggressive by the freeze age Remember the result of vacuum_get_cutoffs before DISABLE_PAGE_SKIPPING can force aggressive mode. Include this classification in backend-local work measurements, keeping freeze-age vacuums distinct from page-skipping options and wraparound failsafe activation. Cumulative statistics storage and SQL exposure follow separately. Adapted-from: cefc6baf144b7f610d86f6fa9d49ae31ad71f585 --- src/backend/access/heap/vacuumlazy.c | 4 ++++ src/include/pgstat.h | 1 + 2 files changed, 5 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 0fc1796bb68..80a28ac5c19 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -166,6 +166,7 @@ typedef struct LVRelState /* Aggressive VACUUM? (must set relfrozenxid >= FreezeLimit) */ bool aggressive; + bool freeze_age_vacuum; /* aggressive before DISABLE_PAGE_SKIPPING */ /* Use visibility map to skip? (disabled by DISABLE_PAGE_SKIPPING) */ bool skipwithvm; /* Consider index vacuuming bypass optimization? */ @@ -481,6 +482,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * time to time, to increase the number of dead tuples it can prune away.) */ vacrel->aggressive = vacuum_get_cutoffs(rel, params, &vacrel->cutoffs); + /* DISABLE_PAGE_SKIPPING alone must not count as a freeze-age run. */ + vacrel->freeze_age_vacuum = vacrel->aggressive; vacrel->rel_pages = orig_rel_pages = RelationGetNumberOfBlocks(rel); vacrel->vistest = GlobalVisTestFor(rel); /* Initialize state used to track oldest extant XID/MXID */ @@ -617,6 +620,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, stats.dead_pages = vacrel->dead_pages; stats.pages_frozen = vacrel->frozen_pages; stats.pages_all_visible = vacrel->all_visible_pages; + stats.freeze_age_vacuum_count = vacrel->freeze_age_vacuum ? 1 : 0; stats.tuples_frozen = vacrel->tuples_frozen; stats.recently_dead_tuples = vacrel->recently_dead_tuples; stats.missed_dead_tuples = vacrel->missed_dead_tuples; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index eb52e290f16..67c25efee96 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -100,6 +100,7 @@ typedef struct PgStat_VacuumStats PgStat_Counter dead_pages; PgStat_Counter pages_frozen; PgStat_Counter pages_all_visible; + PgStat_Counter freeze_age_vacuum_count; PgStat_Counter tuples_frozen; PgStat_Counter recently_dead_tuples; PgStat_Counter missed_dead_tuples; From 7d15c613adc81277fdcc84d1717154a5b6ad05e6 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:28:59 +0300 Subject: [PATCH 34/50] Measure VACUUM buffer usage in core Snapshot buffer activity and block I/O timing around heap, index and AO vacuum work. Assemble backend-local resource records and accumulate AO resources over active phases. Keep relation-local block counts, and subtract index resources from table-only totals to avoid double counting. Introduce optional instrumentation and its consumer hook. Allocate it only when a consumer is installed. Persistent storage and SQL access through the extension follow separately. Native timing and work counters are introduced in independent branches. --- src/backend/access/heap/vacuumlazy.c | 150 +++++++++++++++++++ src/backend/commands/vacuum_ao.c | 127 +++++++++++++++- src/backend/commands/vacuumparallel.c | 11 ++ src/backend/utils/activity/pgstat_relation.c | 6 + src/include/access/appendonly_compaction.h | 16 ++ src/include/commands/vacuum.h | 15 ++ src/include/pgstat.h | 35 +++++ 7 files changed, 357 insertions(+), 3 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..02bbe3c529d 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -187,6 +187,8 @@ typedef struct LVRelState /* Error reporting state */ char *dbname; char *relnamespace; + Oid reloid; + Oid indoid; char *relname; char *indname; /* Current index name */ BlockNumber blkno; /* used only for heap operations */ @@ -225,6 +227,18 @@ typedef struct LVRelState int64 live_tuples; /* # live tuples remaining */ int64 recently_dead_tuples; /* # dead, but not yet removable */ int64 missed_dead_tuples; /* # removable, but not removed */ + + /* + * Resource usage of the index passes this process ran, subtracted from + * the heap report to avoid double-counting (see + * accumulate_heap_vacuum_statistics). + */ + PgStat_CommonCounts *extVacReportIdx; + + /* + * We need to accumulate index statistics for later subtraction from heap + * stats. + */ } LVRelState; /* @@ -312,6 +326,100 @@ static void update_vacuum_error_info(LVRelState *vacrel, static void restore_vacuum_error_info(LVRelState *vacrel, const LVSavedErrInfo *saved_vacrel); +/* Extended vacuum statistics functions */ + +/* + * extvac_stats_start - Allocate and snapshot extended instrumentation. + * + * Return NULL without allocating anything when there is no consumer. The + * caller frees a non-NULL result after reporting the phase, or its vacuum + * memory context cleans it up on error. + */ +LVExtStatCounters * +extvac_stats_start(Relation rel) +{ + LVExtStatCounters *counters; + + if (set_report_vacuum_hook == NULL) + return NULL; + + counters = palloc0(sizeof(LVExtStatCounters)); + counters->bufusage = pgBufferUsage; + + if (rel->pgstat_info && pgstat_track_counts) + { + counters->blocks_fetched = rel->pgstat_info->counts.blocks_fetched; + counters->blocks_hit = rel->pgstat_info->counts.blocks_hit; + } + return counters; +} + +/* + * extvac_stats_end - Diff resource usage since extvac_stats_start into report. + */ +void +extvac_stats_end(Relation rel, LVExtStatCounters * counters, + PgStat_CommonCounts * report) +{ + BufferUsage bufusage; + + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); + + report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; + report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; + report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; + report->total_blks_written = bufusage.shared_blks_written; + /* PostgreSQL 16 keeps a single timer for shared and local blocks */ + report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); + report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); + + if (rel->pgstat_info && pgstat_track_counts) + { + report->blks_fetched = rel->pgstat_info->counts.blocks_fetched - counters->blocks_fetched; + report->blks_hit = rel->pgstat_info->counts.blocks_hit - counters->blocks_hit; + } +} + +/* + * Build the heap-specific part of the extended vacuum report from the + * counters gathered in vacrel. + */ +static void +accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) +{ + extVacStats->type = PGSTAT_EXTVAC_TABLE; + + /* + * Subtract the resource usage of the index passes this process ran: they + * are reported per index, and the database-wide aggregate would count + * them twice otherwise. Parallel workers report their own usage with + * their index passes, so it never enters the leader's counters. + */ + extVacStats->common.total_blks_read -= vacrel->extVacReportIdx->total_blks_read; + extVacStats->common.total_blks_hit -= vacrel->extVacReportIdx->total_blks_hit; + extVacStats->common.total_blks_dirtied -= vacrel->extVacReportIdx->total_blks_dirtied; + extVacStats->common.total_blks_written -= vacrel->extVacReportIdx->total_blks_written; + extVacStats->common.blk_read_time -= vacrel->extVacReportIdx->blk_read_time; + extVacStats->common.blk_write_time -= vacrel->extVacReportIdx->blk_write_time; +} + +/* + * Accumulate an index pass report into the running total that is later + * subtracted from the heap report. + */ +static void +accumulate_idxs_vacuum_statistics(LVRelState *vacrel, + PgStat_VacuumRelationCounts * extVacIdxStats) +{ + vacrel->extVacReportIdx->total_blks_read += extVacIdxStats->common.total_blks_read; + vacrel->extVacReportIdx->total_blks_hit += extVacIdxStats->common.total_blks_hit; + vacrel->extVacReportIdx->total_blks_dirtied += extVacIdxStats->common.total_blks_dirtied; + vacrel->extVacReportIdx->total_blks_written += extVacIdxStats->common.total_blks_written; + vacrel->extVacReportIdx->blk_read_time += extVacIdxStats->common.blk_read_time; + vacrel->extVacReportIdx->blk_write_time += extVacIdxStats->common.blk_write_time; +} + /* * lazy_vacuum_rel_heap() -- perform VACUUM for one heap relation * @@ -340,6 +448,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, startwritetime = 0; + LVExtStatCounters *extVacCounters; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -359,6 +468,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } + extVacCounters = extvac_stats_start(rel); + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -375,9 +486,12 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * these temp copies. */ vacrel = (LVRelState *) palloc0(sizeof(LVRelState)); + if (extVacCounters != NULL) + vacrel->extVacReportIdx = palloc0(sizeof(PgStat_CommonCounts)); vacrel->dbname = get_database_name(MyDatabaseId); vacrel->relnamespace = get_namespace_name(RelationGetNamespace(rel)); vacrel->relname = pstrdup(RelationGetRelationName(rel)); + vacrel->reloid = RelationGetRelid(rel); vacrel->indname = NULL; vacrel->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrel->verbose = verbose; @@ -615,6 +729,16 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(rel, extVacCounters, &extVacReport->common); + accumulate_heap_vacuum_statistics(vacrel, extVacReport); + + pfree(extVacCounters); + pfree(vacrel->extVacReportIdx); + } pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), @@ -2734,7 +2858,9 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2752,6 +2878,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, InvalidOffsetNumber); @@ -2759,6 +2886,16 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2783,7 +2920,9 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + LVExtStatCounters *extVacCounters; + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2802,12 +2941,23 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, */ Assert(vacrel->indname == NULL); vacrel->indname = pstrdup(RelationGetRelationName(indrel)); + vacrel->indoid = RelationGetRelid(indrel); update_vacuum_error_info(vacrel, &saved_err_info, VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + accumulate_idxs_vacuum_statistics(vacrel, extVacReport); + pfree(extVacCounters); + } + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..f951390cf61 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -163,6 +163,13 @@ static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *p BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static AOVacuumRelStats *init_vacrelstats(void); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); +static void ao_accum_resources(PgStat_CommonCounts *dst, + const PgStat_CommonCounts *src, bool subtract); +static void ao_measure_index_resources(Relation indrel, + LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats); +static void ao_measure_table_resources(Relation rel, + AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -414,6 +421,8 @@ init_vacrelstats() old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); + if (set_report_vacuum_hook != NULL) + vacrelstats->extstats = palloc0(sizeof(AOVacuumExtStats)); MemoryContextSwitchTo(old_context); return vacrelstats; @@ -428,11 +437,26 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + LVExtStatCounters *extcounters; + bool extstats = (set_report_vacuum_hook != NULL); Assert(RelationStorageIsAO(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -450,8 +474,12 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); vacrelstats = init_vacrelstats(); + vacrelstats->relid = RelationGetRelid(rel); } + /* Sample the resource usage of the phase for the extended statistics */ + extcounters = extvac_stats_start(rel); + /* * Do the actual work --- either FULL or "lazy" vacuum */ @@ -460,14 +488,93 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); + else + /* Do nothing here, we will launch the stages later */ + Assert(ao_vacuum_phase == 0); + + if (extstats && ao_vacuum_phase != 0) + { + extvac_stats_end(rel, extcounters, &extcounters->report.common); + ao_accum_resources(&vacrelstats->extstats->phases, + &extcounters->report.common, false); + } + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { + if (extstats) + ao_measure_table_resources(rel, vacrelstats); pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } + + if (extcounters != NULL) + pfree(extcounters); +} + +/* + * Add the resource usage counters of src to dst, or subtract them from it. + */ +static void +ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, + bool subtract) +{ + if (subtract) + { + dst->total_blks_read -= src->total_blks_read; + dst->total_blks_hit -= src->total_blks_hit; + dst->total_blks_dirtied -= src->total_blks_dirtied; + dst->total_blks_written -= src->total_blks_written; + /* Per-relation block counts do not include the indexes. */ + dst->blk_read_time -= src->blk_read_time; + dst->blk_write_time -= src->blk_write_time; + } else - /* Do nothing here, we will launch the stages later */ - Assert(ao_vacuum_phase == 0); + { + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + } +} + +/* + * Report one pass over an index of an append-optimized table to + * set_report_vacuum_hook, as lazy_vacuum_one_index() does for the heap, and + * remember its resource usage, which the table's report must not count + * again. + */ +static void +ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, + AOVacuumRelStats *vacrelstats) +{ + PgStat_VacuumRelationCounts *report = &counters->report; + + extvac_stats_end(indrel, counters, &report->common); + report->type = PGSTAT_EXTVAC_INDEX; + + ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); + pfree(counters); +} + +/* + * Report the vacuum of an append-optimized table, all of its phases, to + * set_report_vacuum_hook. + */ +static void +ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) +{ + PgStat_VacuumRelationCounts report; + + memset(&report, 0, sizeof(report)); + report.type = PGSTAT_EXTVAC_TABLE; + report.common = vacrelstats->extstats->phases; + ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); + } /* @@ -574,22 +681,34 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i]); scan_index(Irel[i], aoRelation, elevel, bstrategy); + if (set_report_vacuum_hook) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats); } } else { for (i = 0; i < nindexes; i++) { + LVExtStatCounters *extcounters; + + extcounters = extvac_stats_start(Irel[i]); vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); + if (set_report_vacuum_hook) + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats); } } } @@ -780,6 +899,8 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstats) { + if ((*vacrelstats)->extstats != NULL) + pfree((*vacrelstats)->extstats); pfree(*vacrelstats); *vacrelstats = NULL; } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..9ec6d8fdd31 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -836,6 +836,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + LVExtStatCounters *extVacCounters; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -844,6 +845,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); + extVacCounters = extvac_stats_start(indrel); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -871,6 +873,15 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + if (extVacCounters != NULL) + { + PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; + + extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + extVacReport->type = PGSTAT_EXTVAC_INDEX; + pfree(extVacCounters); + } + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 45bbf9abcb7..41b528e6ccf 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -273,6 +273,12 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } +/* + * Hook for extensions to receive extended vacuum statistics. + * NULL when no extension has registered. + */ +set_report_vacuum_hook_type set_report_vacuum_hook = NULL; + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..9320a6131b2 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -13,6 +13,7 @@ #ifndef APPENDONLY_COMPACTION_H #define APPENDONLY_COMPACTION_H +#include "pgstat.h" #include "nodes/pg_list.h" #include "access/appendonly_visimap.h" #include "utils/rel.h" @@ -21,6 +22,13 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) +/* Optional resource counters for the extended-statistics hook. */ +typedef struct AOVacuumExtStats +{ + PgStat_CommonCounts phases; + PgStat_CommonCounts indexes; +} AOVacuumExtStats; + /* * Stats for progress reporting. * This is AO/AOCO counterpart of LVRelStats for Heap. It lives throughout @@ -31,6 +39,14 @@ typedef struct AOVacuumRelStats int nbytes_truncated; /* current # of bytes truncated from segment file */ int num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + /* the relation these stats were started for */ + Oid relid; + /* + * Resource usage for set_report_vacuum_hook, accumulated over the phases: + * of the phases as a whole, and of the index passes among them, which are + * reported per index and subtracted from the table's report. + */ + AOVacuumExtStats *extstats; /* allocated only when the hook is installed */ } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..a8c8a455e99 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -21,10 +21,12 @@ #include "catalog/pg_class.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "executor/instrument.h" #include "parser/parse_node.h" #include "storage/buf.h" #include "storage/lock.h" #include "utils/relcache.h" +#include "pgstat.h" #include "utils/snapshot.h" /* @@ -440,6 +442,19 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; +/* Allocated only when a hook consumes the extended statistics. */ +typedef struct LVExtStatCounters +{ + BufferUsage bufusage; + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; + PgStat_VacuumRelationCounts report; +} LVExtStatCounters; + +extern LVExtStatCounters *extvac_stats_start(Relation rel); +extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, + PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..3072794803c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -117,6 +117,36 @@ typedef struct PgStat_FunctionCounts /* * Working state needed to accumulate per-function-call timing statistics. */ +/* + * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. + * Type of entry: table (heap), index, or database aggregate. + */ +typedef enum ExtVacReportType +{ + PGSTAT_EXTVAC_INVALID = 0, + PGSTAT_EXTVAC_TABLE = 1, + PGSTAT_EXTVAC_INDEX = 2, + PGSTAT_EXTVAC_DB = 3, +} ExtVacReportType; + +typedef struct PgStat_CommonCounts +{ + int64 total_blks_read; + int64 total_blks_hit; + int64 total_blks_dirtied; + int64 total_blks_written; + int64 blks_fetched; + int64 blks_hit; + double blk_read_time; + double blk_write_time; +} PgStat_CommonCounts; + +typedef struct PgStat_VacuumRelationCounts +{ + PgStat_CommonCounts common; + ExtVacReportType type; +} PgStat_VacuumRelationCounts; + typedef struct PgStat_FunctionCallUsage { /* Link to function's hashtable entry (must still be there at exit!) */ @@ -652,6 +682,11 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); +/* Hook for extensions to receive extended vacuum statistics */ +typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; + extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); From e6dd6a65b1e38563769250eb7ad8ea5dff83abdc Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Mon, 3 Mar 2025 11:18:05 -0500 Subject: [PATCH 35/50] Add relallfrozen to pg_class Add relallfrozen, an estimate of the number of pages marked all-frozen in the visibility map. pg_class already has relallvisible, an estimate of the number of pages in the relation marked all-visible in the visibility map. This is used primarily for planning. relallfrozen, together with relallvisible, is useful for estimating the outstanding number of all-visible but not all-frozen pages in the relation for the purposes of scheduling manual VACUUMs and tuning vacuum freeze parameters. A future commit will use relallfrozen to trigger more frequent vacuums on insert-focused workloads with significant volume of frozen data. Bump catalog version Author: Melanie Plageman Reviewed-by: Nathan Bossart Reviewed-by: Robert Treat Reviewed-by: Corey Huinker Reviewed-by: Greg Sabino Mullane Discussion: https://postgr.es/m/flat/CAAKRu_aj-P7YyBz_cPNwztz6ohP%2BvWis%3Diz3YcomkB3NpYA--w%40mail.gmail.com (cherry picked from commit 99f8f3fbbc8f743290844e8c676d39dad11c5d5d) Cloudberry adaptation: carry both visibility-map counts through QE-to-QD reports, replicated-table normalization, ANALYZE and CLUSTER. AO and indexes keep relallfrozen zero. Avoid VM I/O on the dispatcher during index catalog updates. Include local and distributed catalog tests. PostgreSQL 18 statistics-import APIs and their tests do not exist on this PostgreSQL 16 branch. Use relallfrozen in the VM isolation test. Hold a repeatable-read snapshot and explicitly populate the statistics cache before concurrent DELETE; flush in the writer to avoid depending on the flush interval. --- doc/src/sgml/catalogs.sgml | 20 +++++++ src/backend/access/heap/vacuumlazy.c | 18 ++++-- src/backend/catalog/heap.c | 2 + src/backend/catalog/index.c | 26 ++++----- src/backend/commands/analyze.c | 84 +++++++++++++--------------- src/backend/commands/cluster.c | 8 ++- src/backend/commands/vacuum.c | 17 +++++- src/backend/commands/vacuum_ao.c | 3 + src/backend/utils/cache/relcache.c | 2 + src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_class.h | 3 + src/include/commands/vacuum.h | 6 +- src/include/storage/bufmgr.h | 3 +- 13 files changed, 125 insertions(+), 69 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 9b498953f45..45102c5d654 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2061,6 +2061,26 @@ SCRAM-SHA-256$<iteration count>:&l + + + relallfrozen int4 + + + Number of pages that are marked all-frozen in the table's visibility + map. This is only an estimate and can be used along with + relallvisible for scheduling vacuums and + tuning vacuum's freezing + behavior. + + It is updated by + VACUUM, + ANALYZE, + and a few DDL commands such as + CREATE INDEX. + + + + reltoastrelid oid diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index feab6d5175f..0cffad86150 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -335,7 +335,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, minmulti_updated; BlockNumber orig_rel_pages, new_rel_pages, - new_rel_allvisible; + new_rel_allvisible, + new_rel_allfrozen; PGRUsage ru0; TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, @@ -593,10 +594,18 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * pg_class.relpages to */ new_rel_pages = vacrel->rel_pages; /* After possible rel truncation */ - visibilitymap_count(rel, &new_rel_allvisible, NULL); + visibilitymap_count(rel, &new_rel_allvisible, &new_rel_allfrozen); if (new_rel_allvisible > new_rel_pages) new_rel_allvisible = new_rel_pages; + /* + * An all-frozen block _must_ be all-visible. As such, clamp the count of + * all-frozen blocks to the count of all-visible blocks. This matches the + * clamping of relallvisible above. + */ + if (new_rel_allfrozen > new_rel_allvisible) + new_rel_allfrozen = new_rel_allvisible; + /* * Now actually update rel's pg_class entry. * @@ -605,7 +614,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * scan every page that isn't skipped using the visibility map. */ vac_update_relstats(rel, new_rel_pages, vacrel->new_live_tuples, - new_rel_allvisible, vacrel->nindexes > 0, + new_rel_allvisible, new_rel_allfrozen, + vacrel->nindexes > 0, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false, true); @@ -3434,7 +3444,7 @@ update_relstats_all_indexes(LVRelState *vacrel) vac_update_relstats(indrel, istat->num_pages, istat->num_index_tuples, - 0, + 0, 0, false, InvalidTransactionId, InvalidMultiXactId, diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 0b62b174afa..52adec9f666 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1264,6 +1264,7 @@ InsertPgClassTuple(Relation pg_class_desc, values[Anum_pg_class_relpages - 1] = Int32GetDatum(rd_rel->relpages); values[Anum_pg_class_reltuples - 1] = Float4GetDatum(rd_rel->reltuples); values[Anum_pg_class_relallvisible - 1] = Int32GetDatum(rd_rel->relallvisible); + values[Anum_pg_class_relallfrozen - 1] = Int32GetDatum(rd_rel->relallfrozen); values[Anum_pg_class_reltoastrelid - 1] = ObjectIdGetDatum(rd_rel->reltoastrelid); values[Anum_pg_class_relhasindex - 1] = BoolGetDatum(rd_rel->relhasindex); values[Anum_pg_class_relisshared - 1] = BoolGetDatum(rd_rel->relisshared); @@ -1337,6 +1338,7 @@ AddNewRelationTuple(Relation pg_class_desc, new_rel_reltup->relpages = 0; new_rel_reltup->reltuples = -1; new_rel_reltup->relallvisible = 0; + new_rel_reltup->relallfrozen = 0; /* Sequences always have a known size */ if (relkind == RELKIND_SEQUENCE) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index fadd32fde9a..7c7df5d4c58 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2945,8 +2945,8 @@ FormIndexDatum(struct IndexInfo *indexInfo, * hasindex: set relhasindex to this value * reltuples: if >= 0, set reltuples to this value; else no change * - * If reltuples >= 0, relpages and relallvisible are also updated (using - * RelationGetNumberOfBlocks() and visibilitymap_count()). + * If reltuples >= 0, relpages, relallvisible, and relallfrozen are also + * updated (using RelationGetNumberOfBlocks() and visibilitymap_count()). * * NOTE: an important side-effect of this operation is that an SI invalidation * message is sent out to all backends --- including me --- causing relcache @@ -2964,6 +2964,7 @@ index_update_stats(Relation rel, bool update_stats; BlockNumber relpages = 0; /* keep compiler quiet */ BlockNumber relallvisible = 0; + BlockNumber relallfrozen = 0; Oid relid = RelationGetRelid(rel); Relation pg_class; ScanKeyData key[1]; @@ -2985,7 +2986,7 @@ index_update_stats(Relation rel, if (reltuples == 0 && rel->rd_rel->reltuples < 0) reltuples = -1; - update_stats = reltuples >= 0; + update_stats = reltuples >= 0 && Gp_role != GP_ROLE_DISPATCH; /* * Finish I/O and visibility map buffer locks before @@ -2998,8 +2999,8 @@ index_update_stats(Relation rel, { relpages = RelationGetNumberOfBlocks(rel); - if (rel->rd_rel->relkind != RELKIND_INDEX) - visibilitymap_count(rel, &relallvisible, NULL); + if (rel->rd_rel->relkind != RELKIND_INDEX && !RelationStorageIsAO(rel)) + visibilitymap_count(rel, &relallvisible, &relallfrozen); } /* @@ -3055,16 +3056,8 @@ index_update_stats(Relation rel, dirty = true; } - if (reltuples >= 0 && Gp_role != GP_ROLE_DISPATCH) + if (update_stats) { - /* - * GPDB: We don't maintain relallvisible for AO/CO tables. - */ - if (rd_rel->relkind != RELKIND_INDEX && !RelationStorageIsAO(rel)) - visibilitymap_count(rel, &relallvisible, NULL); - else /* don't bother for indexes */ - relallvisible = 0; - if (rd_rel->relpages != (int32) relpages) { rd_rel->relpages = (int32) relpages; @@ -3080,6 +3073,11 @@ index_update_stats(Relation rel, rd_rel->relallvisible = (int32) relallvisible; dirty = true; } + if (rd_rel->relallfrozen != (int32) relallfrozen) + { + rd_rel->relallfrozen = (int32) relallfrozen; + dirty = true; + } } /* diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 465ca5b19ed..08110428258 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -1105,12 +1105,12 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ if (!inh) { - BlockNumber relallvisible; + BlockNumber relallvisible = 0; + BlockNumber relallfrozen = 0; - if (RelationStorageIsAO(onerel)) - relallvisible = 0; - else - relallvisible = AcquireNumberOfAllVisibleBlocks(onerel); + if (RELKIND_HAS_STORAGE(onerel->rd_rel->relkind) && + !RelationStorageIsAO(onerel)) + AcquireVisibilityMapCounts(onerel, &relallvisible, &relallfrozen); /* * Update pg_class for table relation. CCI first, in case acquirefunc @@ -1121,6 +1121,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, relpages, totalrows, relallvisible, + relallfrozen, hasindex, InvalidTransactionId, InvalidMultiXactId, @@ -1192,7 +1193,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, vac_update_relstats(Irel[ind], estimatedIndexPages, totalindexrows, - 0, + 0, 0, false, InvalidTransactionId, InvalidMultiXactId, @@ -1209,7 +1210,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ CommandCounterIncrement(); vac_update_relstats(onerel, -1, totalrows, - 0, hasindex, InvalidTransactionId, + 0, 0, hasindex, InvalidTransactionId, InvalidMultiXactId, NULL, NULL, in_outer_xact, @@ -2433,46 +2434,39 @@ AcquireNumberOfBlocks(Relation onerel) } /* - * Collect visibility map of relation in dispatcher. - * - * In GPDB if we're in the dispatcher, we need to collect the number of - * visibility map in pg_class from segments. + * Collect both visibility-map counts locally or from the segment catalogs. + * Replicated relations contribute one copy, just like relpages. */ -BlockNumber -AcquireNumberOfAllVisibleBlocks(Relation onerel) +void +AcquireVisibilityMapCounts(Relation onerel, BlockNumber *all_visible, + BlockNumber *all_frozen) { - int64 totalvms; - - /* collect total vms from segments in master */ - if (Gp_role == GP_ROLE_DISPATCH && - onerel->rd_cdbpolicy && !GpPolicyIsEntry(onerel->rd_cdbpolicy)) - { - /* Query the segments pg_class. */ - char relvm_sql[80]; - - snprintf(relvm_sql, sizeof(relvm_sql), - "select relallvisible from pg_catalog.pg_class where oid = %u", RelationGetRelid(onerel)); - totalvms = get_size_from_segDBs(relvm_sql); - if (GpPolicyIsReplicated(onerel->rd_cdbpolicy)) - { - /* - * If the distribution of the relation is replicated, we will sum up - * vms much twice which we expecting only once. So we need to divide - * up totalvms by numsegments. - */ - totalvms /= onerel->rd_cdbpolicy->numsegments; - } - - return (BlockNumber)totalvms; - } - /* get vms from local in segment */ - else - { - BlockNumber all_visible = 0; - visibilitymap_count(onerel, &all_visible, NULL); - - return all_visible; - } + if (Gp_role == GP_ROLE_DISPATCH && + onerel->rd_cdbpolicy && !GpPolicyIsEntry(onerel->rd_cdbpolicy)) + { + char relvm_sql[100]; + int64 visible; + int64 frozen; + + snprintf(relvm_sql, sizeof(relvm_sql), + "select relallvisible from pg_catalog.pg_class where oid = %u", + RelationGetRelid(onerel)); + visible = get_size_from_segDBs(relvm_sql); + snprintf(relvm_sql, sizeof(relvm_sql), + "select relallfrozen from pg_catalog.pg_class where oid = %u", + RelationGetRelid(onerel)); + frozen = get_size_from_segDBs(relvm_sql); + + if (GpPolicyIsReplicated(onerel->rd_cdbpolicy)) + { + visible /= onerel->rd_cdbpolicy->numsegments; + frozen /= onerel->rd_cdbpolicy->numsegments; + } + *all_visible = (BlockNumber) visible; + *all_frozen = (BlockNumber) frozen; + } + else + visibilitymap_count(onerel, all_visible, all_frozen); } /* diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index a3d2ad87bfb..1c12029334a 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -1511,6 +1511,7 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, int32 swap_pages; float4 swap_tuples; int32 swap_allvisible; + int32 swap_allfrozen; swap_pages = relform1->relpages; relform1->relpages = relform2->relpages; @@ -1523,6 +1524,10 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, swap_allvisible = relform1->relallvisible; relform1->relallvisible = relform2->relallvisible; relform2->relallvisible = swap_allvisible; + + swap_allfrozen = relform1->relallfrozen; + relform1->relallfrozen = relform2->relallfrozen; + relform2->relallfrozen = swap_allfrozen; } /* @@ -1728,7 +1733,8 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, vac_send_relstats_to_qd(rel, relform1->relpages, relform1->reltuples, - relform1->relallvisible); + relform1->relallvisible, + relform1->relallfrozen); relation_close(rel, AccessShareLock); } diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index cae73685cca..286f698b988 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1800,6 +1800,7 @@ void vac_update_relstats(Relation relation, BlockNumber num_pages, double num_tuples, BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages, bool hasindex, TransactionId frozenxid, MultiXactId minmulti, bool *frozenxid_updated, bool *minmulti_updated, @@ -1837,13 +1838,15 @@ vac_update_relstats(Relation relation, num_pages = relation->rd_rel->relpages; num_tuples = relation->rd_rel->reltuples; num_all_visible_pages = relation->rd_rel->relallvisible; + num_all_frozen_pages = relation->rd_rel->relallfrozen; } else if (Gp_role == GP_ROLE_EXECUTE) { vac_send_relstats_to_qd(relation, num_pages, num_tuples, - num_all_visible_pages); + num_all_visible_pages, + num_all_frozen_pages); } } @@ -1923,6 +1926,11 @@ vac_update_relstats(Relation relation, pgcform->relallvisible = (int32) num_all_visible_pages; dirty = true; } + if (pgcform->relallfrozen != (int32) num_all_frozen_pages) + { + pgcform->relallfrozen = (int32) num_all_frozen_pages; + dirty = true; + } elog(DEBUG2, "Vacuum oid=%u pages=%d tuples=%f", relid, pgcform->relpages, pgcform->reltuples); @@ -3616,6 +3624,7 @@ vacuum_combine_stats(VacuumStatsContext *stats_context, CdbPgResults *cdb_pgresu tmp_stats_combo->rel_pages += pgclass_stats_combo->rel_pages; tmp_stats_combo->rel_tuples += pgclass_stats_combo->rel_tuples; tmp_stats_combo->relallvisible += pgclass_stats_combo->relallvisible; + tmp_stats_combo->relallfrozen += pgclass_stats_combo->relallfrozen; /* * Accumulate the number of QEs, assuming sending only once * per QE for each relid in the VACUUM scenario. @@ -3669,6 +3678,7 @@ vac_update_relstats_from_list(VacuumStatsContext *stats_context) stats->rel_pages = stats->rel_pages / rel->rd_cdbpolicy->numsegments; stats->rel_tuples = stats->rel_tuples / rel->rd_cdbpolicy->numsegments; stats->relallvisible = stats->relallvisible / rel->rd_cdbpolicy->numsegments; + stats->relallfrozen = stats->relallfrozen / rel->rd_cdbpolicy->numsegments; } if (RelationIsAppendOptimized(rel)) @@ -3720,6 +3730,7 @@ vac_update_relstats_from_list(VacuumStatsContext *stats_context) vac_update_relstats(rel, stats->rel_pages, stats->rel_tuples, stats->relallvisible, + stats->relallfrozen, rel->rd_rel->relhasindex, InvalidTransactionId, InvalidMultiXactId, @@ -3771,7 +3782,8 @@ void vac_send_relstats_to_qd(Relation relation, BlockNumber num_pages, double num_tuples, - BlockNumber num_all_visible_pages) + BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages) { StringInfoData buf; @@ -3785,6 +3797,7 @@ vac_send_relstats_to_qd(Relation relation, stats.rel_pages = num_pages; stats.rel_tuples = num_tuples; stats.relallvisible = num_all_visible_pages; + stats.relallfrozen = num_all_frozen_pages; pq_sendbyte(&buf, true); /* Mark the result ready when receive this message */ pq_sendint(&buf, PGExtraTypeVacuumStats, sizeof(PGExtraType)); pq_sendint(&buf, sizeof(VPgClassStats), sizeof(int)); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index f644b737009..a469af4aa26 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -299,6 +299,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt total_file_segs, /* AO/AOCO does not currently have an equivalent to Heap's 'all visible pages', use this field to represent AO/AOCO's total segment file count */ + 0, /* relallfrozen: AO has no visibility map */ relhasindex, InvalidTransactionId, InvalidMultiXactId, @@ -758,6 +759,7 @@ vacuum_appendonly_index(Relation indexRelation, vac_update_relstats(indexRelation, stats->num_pages, stats->num_index_tuples, 0, /* relallvisible */ + 0, /* relallfrozen */ false, InvalidTransactionId, InvalidMultiXactId, @@ -925,6 +927,7 @@ scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac vac_update_relstats(indrel, stats->num_pages, stats->num_index_tuples, 0, /* relallvisible, don't bother for indexes */ + 0, /* relallfrozen */ false, InvalidTransactionId, InvalidMultiXactId, diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 812782aabd1..0466a2d561f 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1994,6 +1994,7 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_rel->relpages = 0; relation->rd_rel->reltuples = -1; relation->rd_rel->relallvisible = 0; + relation->rd_rel->relallfrozen = 0; relation->rd_rel->relkind = RELKIND_RELATION; relation->rd_rel->relnatts = (int16) natts; relation->rd_rel->relam = HEAP_TABLE_AM_OID; @@ -4040,6 +4041,7 @@ RelationSetNewRelfilenumber(Relation relation, char persistence) classform->relpages = 0; /* it's empty until further notice */ classform->reltuples = -1; classform->relallvisible = 0; + classform->relallfrozen = 0; } classform->relfrozenxid = freezeXid; classform->relminmxid = minmulti; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c9c2fd83275..77f198458a8 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302608261 +#define CATALOG_VERSION_NO 302610061 #endif diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 0b1964b6288..b4aa1c9db38 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -68,6 +68,9 @@ CATALOG(pg_class,1259,RelationRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83,Relat /* # of all-visible blocks (not always up-to-date) */ int32 relallvisible BKI_DEFAULT(0); + /* # of all-frozen blocks (not always up-to-date) */ + int32 relallfrozen BKI_DEFAULT(0); + /* OID of toast table; 0 if none */ Oid reltoastrelid BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_class); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 54b897a528c..ffbb2a62ecf 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -276,6 +276,7 @@ typedef struct VPgClassStats BlockNumber rel_pages; double rel_tuples; BlockNumber relallvisible; + BlockNumber relallfrozen; } VPgClassStats; typedef struct VPgClassStatsCombo @@ -284,6 +285,7 @@ typedef struct VPgClassStatsCombo BlockNumber rel_pages; double rel_tuples; BlockNumber relallvisible; + BlockNumber relallfrozen; int count; /* expect to equal to the number of dispatched segments */ } VPgClassStatsCombo; @@ -463,11 +465,13 @@ extern double vac_estimate_reltuples(Relation relation, extern void vac_send_relstats_to_qd(Relation relation, BlockNumber num_pages, double num_tuples, - BlockNumber num_all_visible_pages); + BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages); extern void vac_update_relstats(Relation relation, BlockNumber num_pages, double num_tuples, BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages, bool hasindex, TransactionId frozenxid, MultiXactId minmulti, diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index c42970d35b2..3f6f3887d46 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -401,7 +401,8 @@ RelationGuessNumberOfBlocksFromSize(uint64 szbytes) RelationGetNumberOfBlocksInFork(reln, MAIN_FORKNUM) extern BlockNumber AcquireNumberOfBlocks(Relation onerel); -extern BlockNumber AcquireNumberOfAllVisibleBlocks(Relation onerel); +extern void AcquireVisibilityMapCounts(Relation onerel, BlockNumber *all_visible, + BlockNumber *all_frozen); extern bool BufferIsPermanent(Buffer buffer); extern XLogRecPtr BufferGetLSNAtomic(Buffer buffer); From 0ad087adae30d4b4fa94ce122048497d861383e8 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Mon, 21 Sep 2026 01:40:05 +0300 Subject: [PATCH 36/50] Measure append-optimized vacuum work and remaining dead tuples Count completed compactions, moved tuples, scanned pages and the final segment count without an extension hook. Discard state left over from an interrupted operation. Keep per-index work measurements for each pass. Native cumulative storage and SQL reporting follow separately. Adapted-from: 6f38fb242db39b4df922687265f9651bb115b35a --- src/backend/access/aocs/aocs_compaction.c | 6 ++ src/backend/access/appendonly/aomd.c | 3 +- .../access/appendonly/appendonly_compaction.c | 6 ++ src/backend/commands/vacuum_ao.c | 80 ++++++++++++++++--- src/include/access/appendonly_compaction.h | 12 ++- src/include/commands/vacuum.h | 3 +- src/include/pgstat.h | 3 + 7 files changed, 96 insertions(+), 17 deletions(-) diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index 12df08f3d67..ca686617f47 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -350,6 +350,12 @@ AOCSSegmentFileFullCompaction(Relation aorel, MarkAOCSFileSegInfoAwaitingDrop(aorel, compact_segno); + /* Count completed compaction, including a scan with no surviving tuples. */ + vacrelstats->compacted_segments++; + vacrelstats->tuples_moved += tupleCount; + vacrelstats->pages_scanned += + scanDesc->totalBytesRead / BLCKSZ + (scanDesc->totalBytesRead % BLCKSZ != 0); + AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno); diff --git a/src/backend/access/appendonly/aomd.c b/src/backend/access/appendonly/aomd.c index 0793c1a6fa3..1a80f74789f 100644 --- a/src/backend/access/appendonly/aomd.c +++ b/src/backend/access/appendonly/aomd.c @@ -216,7 +216,8 @@ TruncateAOSegmentFile(File fd, Relation rel, int32 segFileNum, int64 offset, AOV /* report heap-equivalent blocks vacuumed */ vacrelstats->nbytes_truncated += filesize_before - offset; pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, - RelationGuessNumberOfBlocksFromSize(vacrelstats->nbytes_truncated)); + vacrelstats->nbytes_truncated / BLCKSZ + + (vacrelstats->nbytes_truncated % BLCKSZ != 0)); } if (XLogIsNeeded() && RelationNeedsWAL(rel)) diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index b8cae6ebf6c..e9fe6785776 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -517,6 +517,12 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, MarkFileSegInfoAwaitingDrop(aorel, compact_segno); + /* Count completed compaction, not merely a candidate considered by VACUUM. */ + vacrelstats->compacted_segments++; + vacrelstats->tuples_moved += movedTupleCount; + vacrelstats->pages_scanned += + fsinfo->eof / BLCKSZ + (fsinfo->eof % BLCKSZ != 0); + AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno); /* Delete all mini pages of the segment files if block directory exists */ diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..0e7d98739da 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -150,13 +150,14 @@ static void vacuum_appendonly_index(Relation indexRelation, Bitmapset *dead_segs, int elevel, BufferAccessStrategy bstrategy, - AOVacuumRelStats *vacrelstats); + AOVacuumRelStats *vacrelstats, + IndexBulkDeleteResult *result); static bool appendonly_tid_reaped(ItemPointer itemptr, void *state); static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, @@ -237,7 +238,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt { BlockNumber relpages; double reltuples; - double deadtuples; + int64 deadtuples; bool relhasindex; /* AO/AOCO total file segment number, use type BlockNumber to * represent same type with num_all_visible_pages in libpq. @@ -302,6 +303,11 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt false, true /* isvacuum */); + vacrelstats->total_file_segs = total_file_segs; + + /* Keep the final dead-tuple sample for native work reporting. */ + vacrelstats->dead_tuples = deadtuples; + /* report results to the stats collector, too */ pgstat_report_vacuum(RelationGetRelid(onerel), onerel->rd_rel->relisshared, @@ -433,6 +439,19 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -450,6 +469,7 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); vacrelstats = init_vacrelstats(); + vacrelstats->relid = RelationGetRelid(rel); } /* @@ -460,14 +480,28 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); - pgstat_progress_end_command(); - cleanup_vacrelstats(&vacrelstats); - } else /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { + PgStat_VacuumStats stats = {0}; + + stats.tuples_deleted = vacrelstats->num_dead_tuples; + stats.dead_tuples = vacrelstats->dead_tuples; + stats.recently_dead_tuples = vacrelstats->dead_tuples; + stats.pages_scanned = vacrelstats->pages_scanned; + stats.total_file_segs = vacrelstats->total_file_segs; + stats.compacted_segments = vacrelstats->compacted_segments; + stats.tuples_moved = vacrelstats->tuples_moved; + stats.pages_removed = vacrelstats->nbytes_truncated / BLCKSZ + + (vacrelstats->nbytes_truncated % BLCKSZ != 0); + + pgstat_progress_end_command(); + cleanup_vacrelstats(&vacrelstats); + } } /* @@ -544,6 +578,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs int i; Relation *Irel; int nindexes; + bool final_cleanup = + (options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE; Assert(RelationStorageIsAO(aoRelation)); @@ -574,29 +610,37 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + IndexBulkDeleteResult result = {0}; + scan_index(Irel[i], aoRelation, elevel, - bstrategy); + bstrategy, + &result); + vacuum_measure_index_stats(Irel[i], &result, 0, 0, final_cleanup); } } else { for (i = 0; i < nindexes; i++) { + IndexBulkDeleteResult result = {0}; + vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, - vacrelstats); + vacrelstats, + &result); + vacuum_measure_index_stats(Irel[i], &result, 0, 0, final_cleanup); } } } vac_close_indexes(nindexes, Irel, NoLock); pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, - ((options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE) ? + final_cleanup ? PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP : PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP); return nindexes; } @@ -613,7 +657,8 @@ vacuum_appendonly_index(Relation indexRelation, Bitmapset *dead_segs, int elevel, BufferAccessStrategy bstrategy, - AOVacuumRelStats *vacrelstats) + AOVacuumRelStats *vacrelstats, + IndexBulkDeleteResult *result) { IndexBulkDeleteResult *stats; IndexVacuumInfo ivinfo = {0}; @@ -651,6 +696,10 @@ vacuum_appendonly_index(Relation indexRelation, if (!stats) return; + /* Hand the counts of the pass to the caller, for the statistics */ + if (result) + *result = *stats; + /* * Now update statistics in pg_class, but only if the index says the count * is accurate. @@ -714,7 +763,7 @@ appendonly_tid_reaped(ItemPointer itemptr, void *state) static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) { FileSegTotals *fstotal; BlockNumber nblocks; @@ -790,7 +839,8 @@ cleanup_vacrelstats(AOVacuumRelStats **vacrelstats) * We use this when we have no deletions to do. */ void -scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac_strategy) +scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac_strategy, + IndexBulkDeleteResult *result) { IndexBulkDeleteResult *stats; IndexVacuumInfo ivinfo = {0}; @@ -818,6 +868,10 @@ scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac if (!stats) return; + /* Hand the counts of the pass to the caller, for the statistics */ + if (result) + *result = *stats; + /* * Now update statistics in pg_class, but only if the index says the count * is accurate. diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..8252fb2e17c 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -28,9 +28,17 @@ */ typedef struct AOVacuumRelStats { - int nbytes_truncated; /* current # of bytes truncated from segment file */ - int num_dead_tuples; /* current # of dead tuples */ + int64 nbytes_truncated; /* current # of bytes truncated from segment file */ + int64 num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + int64 dead_tuples; + int64 pages_scanned; + int64 total_file_segs; + int64 compacted_segments; + int64 tuples_moved; + + /* the relation these stats were started for */ + Oid relid; } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index e2f928a94cd..b49668ba9df 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -522,7 +522,8 @@ extern void analyze_rel(Oid relid, RangeVar *relation, /* in commands/vacuumlazy.c */ extern void lazy_vacuum_rel_heap(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy); -extern void scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy bstrategy); +extern void scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy bstrategy, + IndexBulkDeleteResult *result); /* in commands/vacuum_ao.c */ extern void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 67c25efee96..776ec25ae56 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -107,6 +107,9 @@ typedef struct PgStat_VacuumStats PgStat_Counter pages_scanned; PgStat_Counter pages_removed; PgStat_Counter missed_dead_pages; + PgStat_Counter total_file_segs; + PgStat_Counter compacted_segments; + PgStat_Counter tuples_moved; } PgStat_VacuumStats; From 38c8ffa3065d96f2107e546da07e0de2d62192ae Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 1 Oct 2026 16:28:59 +0300 Subject: [PATCH 37/50] Measure VACUUM WAL usage in core Add WAL record, full-page-image and byte deltas to the backend-local VACUUM resource measurements. Snapshot pgWalUsage around the same heap, index and AO phases as buffer usage. Keep index WAL separate from table resources and accumulate only active AO phases. Reuse PostgreSQL's existing pgWalUsage instrumentation. Delivery, persistence and SQL exposure of these counters are added by the separate extension reporting commit. --- src/backend/access/heap/vacuumlazy.c | 13 +++++++++++++ src/backend/commands/vacuum_ao.c | 6 ++++++ src/include/commands/vacuum.h | 1 + src/include/pgstat.h | 3 +++ 4 files changed, 23 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 02bbe3c529d..5da380c2989 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -344,6 +344,7 @@ extvac_stats_start(Relation rel) return NULL; counters = palloc0(sizeof(LVExtStatCounters)); + counters->walusage = pgWalUsage; counters->bufusage = pgBufferUsage; if (rel->pgstat_info && pgstat_track_counts) @@ -361,8 +362,11 @@ void extvac_stats_end(Relation rel, LVExtStatCounters * counters, PgStat_CommonCounts * report) { + WalUsage walusage; BufferUsage bufusage; + memset(&walusage, 0, sizeof(WalUsage)); + WalUsageAccumDiff(&walusage, &pgWalUsage, &counters->walusage); memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &counters->bufusage); @@ -373,6 +377,9 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, /* PostgreSQL 16 keeps a single timer for shared and local blocks */ report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); + report->wal_records = walusage.wal_records; + report->wal_fpi = walusage.wal_fpi; + report->wal_bytes = walusage.wal_bytes; if (rel->pgstat_info && pgstat_track_counts) { @@ -402,6 +409,9 @@ accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCount extVacStats->common.total_blks_written -= vacrel->extVacReportIdx->total_blks_written; extVacStats->common.blk_read_time -= vacrel->extVacReportIdx->blk_read_time; extVacStats->common.blk_write_time -= vacrel->extVacReportIdx->blk_write_time; + extVacStats->common.wal_records -= vacrel->extVacReportIdx->wal_records; + extVacStats->common.wal_fpi -= vacrel->extVacReportIdx->wal_fpi; + extVacStats->common.wal_bytes -= vacrel->extVacReportIdx->wal_bytes; } /* @@ -418,6 +428,9 @@ accumulate_idxs_vacuum_statistics(LVRelState *vacrel, vacrel->extVacReportIdx->total_blks_written += extVacIdxStats->common.total_blks_written; vacrel->extVacReportIdx->blk_read_time += extVacIdxStats->common.blk_read_time; vacrel->extVacReportIdx->blk_write_time += extVacIdxStats->common.blk_write_time; + vacrel->extVacReportIdx->wal_records += extVacIdxStats->common.wal_records; + vacrel->extVacReportIdx->wal_fpi += extVacIdxStats->common.wal_fpi; + vacrel->extVacReportIdx->wal_bytes += extVacIdxStats->common.wal_bytes; } /* diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index f951390cf61..38a83204700 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -528,6 +528,9 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, /* Per-relation block counts do not include the indexes. */ dst->blk_read_time -= src->blk_read_time; dst->blk_write_time -= src->blk_write_time; + dst->wal_records -= src->wal_records; + dst->wal_fpi -= src->wal_fpi; + dst->wal_bytes -= src->wal_bytes; } else { @@ -539,6 +542,9 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, dst->blks_hit += src->blks_hit; dst->blk_read_time += src->blk_read_time; dst->blk_write_time += src->blk_write_time; + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; } } diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index a8c8a455e99..1dba80b2d84 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -445,6 +445,7 @@ extern PGDLLIMPORT int VacuumCostBalanceLocal; /* Allocated only when a hook consumes the extended statistics. */ typedef struct LVExtStatCounters { + WalUsage walusage; BufferUsage bufusage; PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3072794803c..d0e43e30b8d 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -139,6 +139,9 @@ typedef struct PgStat_CommonCounts int64 blks_hit; double blk_read_time; double blk_write_time; + int64 wal_records; + int64 wal_fpi; + uint64 wal_bytes; } PgStat_CommonCounts; typedef struct PgStat_VacuumRelationCounts From aa98a08da96aa1d30157f20f5070ccf5e62eff3a Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 7 Oct 2026 16:52:12 +0300 Subject: [PATCH 38/50] Report AO active-phase time to cumulative statistics Add an elapsed-time reporting entry point and connect the previously introduced AO active-phase timers. Exclude gaps between phases from cumulative vacuum elapsed and cost-delay time, and remove the obsolete first-phase timing baseline. SQL exposure and tests follow separately. --- src/backend/commands/vacuum_ao.c | 20 +++++++------------- src/backend/utils/activity/pgstat_relation.c | 15 +++++++++++++-- src/include/access/appendonly_compaction.h | 3 --- src/include/pgstat.h | 4 ++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index a469af4aa26..6d654dedd21 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -312,17 +312,6 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt vacrelstats->live_tuples = reltuples; vacrelstats->dead_tuples = deadtuples; - /* report results to the stats collector, too */ - pgstat_report_vacuum(RelationGetRelid(onerel), - onerel->rd_rel->relisshared, - reltuples, - deadtuples, - vacrelstats->starttime, - (PgStat_Counter) rint(VacuumDelayTime - - vacrelstats->startdelaytime), - false); /* no failsafe mode for AO tables */ - - SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); } @@ -429,8 +418,6 @@ init_vacrelstats() old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); - vacrelstats->starttime = GetCurrentTimestamp(); - vacrelstats->startdelaytime = VacuumDelayTime; MemoryContextSwitchTo(old_context); return vacrelstats; @@ -519,6 +506,13 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) { + /* The last phase: report only the phases executed by this worker. */ + pgstat_report_vacuum_elapsed(RelationGetRelid(rel), + rel->rd_rel->relisshared, + vacrelstats->live_tuples, vacrelstats->dead_tuples, + (PgStat_Counter) rint(vacrelstats->vacuum_time), + (PgStat_Counter) rint(vacrelstats->delay_time), + false); /* AO has no failsafe mode. */ pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 95423272731..71212becd50 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -218,20 +218,31 @@ pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, TimestampTz starttime, PgStat_Counter delaytime, bool failsafe) +{ + pgstat_report_vacuum_elapsed(tableoid, shared, livetuples, deadtuples, + TimestampDifferenceMilliseconds(starttime, + GetCurrentTimestamp()), + delaytime, failsafe); +} + +/* Report a completed vacuum whose active phases were timed separately (ms). */ +void +pgstat_report_vacuum_elapsed(Oid tableoid, bool shared, + PgStat_Counter livetuples, PgStat_Counter deadtuples, + PgStat_Counter elapsedtime, PgStat_Counter delaytime, + bool failsafe) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = (shared ? InvalidOid : MyDatabaseId); TimestampTz ts; - PgStat_Counter elapsedtime; if (!pgstat_track_counts) return; /* Store the data in the table's hash table entry. */ ts = GetCurrentTimestamp(); - elapsedtime = TimestampDifferenceMilliseconds(starttime, ts); /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index ec3161854b3..a2039c22145 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -32,9 +32,6 @@ typedef struct AOVacuumRelStats int64 nbytes_truncated; /* current # of bytes truncated from segment file */ int64 num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ - /* Legacy timing-report baseline, retained until elapsed-time reporting. */ - TimestampTz starttime; - double startdelaytime; /* Active phase durations and delays in milliseconds, excluding phase gaps. */ double vacuum_time; double delay_time; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index a761000e264..cc557cf4848 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -699,6 +699,10 @@ extern void pgstat_init_relation(Relation rel); extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); +extern void pgstat_report_vacuum_elapsed(Oid tableoid, bool shared, + PgStat_Counter livetuples, PgStat_Counter deadtuples, + PgStat_Counter elapsedtime, PgStat_Counter delaytime, + bool failsafe); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples, TimestampTz starttime, PgStat_Counter delaytime, From 653679f5f217f28518513a51a866e4fa39219ae8 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 8 Oct 2026 00:00:00 +0300 Subject: [PATCH 39/50] Expose maintenance timing and visibility-map statistics in catalogs Add native SQL getters and local/cluster views for maintenance elapsed and cost-delay time, failsafe runs, interrupted vacuums and cleared VM bits. Expose progress delay and test timing, VM snapshots and relallfrozen for heap and append-optimized relations without a statistics extension. --- doc/src/sgml/monitoring.sgml | 313 ++++++++++++++++++ src/backend/catalog/system_views.sql | 43 ++- src/backend/catalog/system_views_gp.in | 1 + .../catalog/system_views_gp_summary.sql | 53 ++- src/backend/utils/adt/pgstatfuncs.c | 61 ++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 64 ++++ .../expected/vacuum-extending-freeze.out | 197 +++++++++++ src/test/isolation/isolation_schedule | 1 + .../specs/vacuum-extending-freeze.spec | 127 +++++++ .../isolation2/expected/vacuum_ao_stats.out | 140 ++++++++ src/test/isolation2/isolation2_schedule | 1 + src/test/isolation2/sql/vacuum_ao_stats.sql | 84 +++++ src/test/modules/test_misc/meson.build | 4 + .../test_misc/t/016_index_vacuum_time.pl | 80 +++++ .../test_misc/t/017_vacuum_failsafe.pl | 71 ++++ .../test_misc/t/018_vacuum_interrupts.pl | 71 ++++ .../modules/test_misc/t/020_relallfrozen.pl | 67 ++++ src/test/regress/expected/rules.out | 54 ++- src/test/regress/expected/vacuum_stats.out | 74 ++++- src/test/regress/sql/vacuum_stats.sql | 46 ++- 21 files changed, 1536 insertions(+), 18 deletions(-) create mode 100644 src/test/isolation/expected/vacuum-extending-freeze.out create mode 100644 src/test/isolation/specs/vacuum-extending-freeze.spec create mode 100644 src/test/isolation2/expected/vacuum_ao_stats.out create mode 100644 src/test/isolation2/sql/vacuum_ao_stats.sql create mode 100644 src/test/modules/test_misc/t/016_index_vacuum_time.pl create mode 100644 src/test/modules/test_misc/t/017_vacuum_failsafe.pl create mode 100644 src/test/modules/test_misc/t/018_vacuum_interrupts.pl create mode 100644 src/test/modules/test_misc/t/020_relallfrozen.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f9ceead292f..6c36323b843 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -459,6 +459,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_vacuum + One row per database, plus one for shared objects, showing cumulative + vacuum statistics. See . + + pg_stat_database_conflictspg_stat_database_conflicts @@ -4697,6 +4703,137 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + <structname>pg_stat_vacuum</structname> + pg_stat_vacuum + + The pg_stat_vacuum view contains one row for each + database, plus one for shared objects, showing cumulative vacuum statistics + on the current instance. The current database's counters are reset + together with its other statistics by pg_stat_reset(). + + + In Cloudberry, gp_stat_vacuum shows these columns + for every instance, with gp_segment_id identifying + the instance. gp_stat_vacuum_summary sums the times + across instances and takes the maximum failsafe and interrupt counts and + the latest reset time for each database. + + + <structname>pg_stat_vacuum</structname> View + + + + + Column Type + + + Description + + + + + + + datid oid + + + OID of this database, or 0 for objects belonging to a shared + relation + + + + + + datname name + + + Name of this database, or NULL for shared + objects. + + + + + + total_vacuum_time double precision + + + Total time manually invoked vacuums spent processing tables of this + database, in milliseconds (index processing is included in the owning + table's run) + + + + + + total_autovacuum_time double precision + + + Total time the autovacuum daemon spent processing tables of this + database, in milliseconds + + + + + + total_vacuum_delay_time double precision + + + Total time manually invoked vacuums of this database spent sleeping in + cost-based delay points, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time autovacuums of this database spent sleeping in cost-based + delay points, in milliseconds (counted only when + is enabled) + + + + + + vacuum_failsafe_count bigint + + + Number of vacuums in this database that entered the wraparound + failsafe mode (see ) + + + + + + vacuum_interrupt_count bigint + + + Number of times a vacuum in this database was interrupted by an error, + the vacuum of an append-optimized table included. In a Cloudberry + cluster the count is kept by the instance on which the vacuum failed: + a vacuum dispatched to the segments that fails there is counted by + those segments, not by the coordinator. A segment reports the count + once its part of the failed distributed transaction has ended, which + may be as late as the next command of the session. + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset + + + + +
+ + <structname>pg_stat_database_conflicts</structname> @@ -4926,6 +5063,62 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + total_vacuum_time double precision + + + Total time this index has been processed by manually invoked vacuums, + in milliseconds. (This includes the time spent sleeping due to + cost-based delays.) + + + + + + total_autovacuum_time double precision + + + Total time this index has been processed by the autovacuum daemon, + in milliseconds. (This includes the time spent sleeping due to + cost-based delays.) + + + + + + total_vacuum_delay_time double precision + + + Total time manually invoked vacuums spent sleeping in cost-based delay + points while processing this index, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time the autovacuum daemon spent sleeping in cost-based delay + points while processing this index, in milliseconds (counted only when + is enabled) + + + + + + vacuum_failsafe_count bigint + + + Number of vacuums of this table that entered the wraparound failsafe + mode (see ). Always zero for + append-optimized tables, which have no tuple transaction IDs to + freeze. + + + n_tup_ins bigint @@ -5017,6 +5210,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + + + visible_page_marks_cleared bigint + + + Number of times the all-visible mark in the + visibility map was cleared for + pages of this table. The all-visible mark of a heap page is + cleared whenever a backend process modifies a page that was + previously marked all-visible by vacuum activity (whether manual + VACUUM or autovacuum). The page must then be + processed again by vacuum on a subsequent run. A high rate of + change in this counter means that vacuum has to repeatedly + re-process pages of this table. Always zero for append-optimized + tables, which have no visibility map. + + + + + + frozen_page_marks_cleared bigint + + + Number of times the all-frozen mark in the + visibility map was cleared for + pages of this table. The all-frozen mark of a heap page is cleared + whenever a backend process modifies a page that was previously + marked all-frozen by vacuum activity (manual VACUUM + or autovacuum). The page must then be processed again by vacuum on + the next freeze run for this table. Always zero for append-optimized + tables. + + + last_vacuum timestamp with time zone @@ -5094,6 +5323,66 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i daemon + + + + total_vacuum_time double precision + + + Total time this table has been manually vacuumed, in milliseconds + + + + + + total_autovacuum_time double precision + + + Total time this table has been vacuumed by the autovacuum daemon, + in milliseconds + + + + + + total_analyze_time double precision + + + Total time this table has been manually analyzed, in milliseconds + + + + + + total_autoanalyze_time double precision + + + Total time this table has been analyzed by the autovacuum daemon, + in milliseconds + + + + + + total_vacuum_delay_time double precision + + + Total time this table's vacuums spent sleeping in cost-based delay + points, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time autovacuums of this table spent sleeping in cost-based + delay points, in milliseconds (counted only when + is enabled) + + @@ -6496,6 +6785,18 @@ FROM pg_stat_get_backend_idset() AS backendid; acquiring inherited sample rows. + + + + delay_time double precision + + + Total time spent sleeping due to cost-based delay (see + , in milliseconds + (if is enabled, otherwise + zero). + + @@ -7377,6 +7678,18 @@ FROM pg_stat_get_backend_idset() AS backendid; Number of dead tuples collected since the last index vacuum cycle. + + + + delay_time double precision + + + Total time spent sleeping due to cost-based delay (see + ), in milliseconds + (if is enabled, otherwise + zero). + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 24b7fc1d33b..f2b72af85f4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -701,7 +701,16 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_vacuum_count(C.oid) AS vacuum_count, pg_stat_get_autovacuum_count(C.oid) AS autovacuum_count, pg_stat_get_analyze_count(C.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count, + pg_stat_get_total_vacuum_time(C.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(C.oid) AS total_autovacuum_time, + pg_stat_get_total_analyze_time(C.oid) AS total_analyze_time, + pg_stat_get_total_autoanalyze_time(C.oid) AS total_autoanalyze_time, + pg_stat_get_total_vacuum_delay_time(C.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(C.oid) AS total_autovacuum_delay_time, + pg_stat_get_vacuum_failsafe_count(C.oid) AS vacuum_failsafe_count, + pg_stat_get_visible_page_marks_cleared(C.oid) AS visible_page_marks_cleared, + pg_stat_get_frozen_page_marks_cleared(C.oid) AS frozen_page_marks_cleared FROM pg_class C LEFT JOIN pg_index I ON C.oid = I.indrelid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) @@ -821,7 +830,11 @@ CREATE VIEW pg_stat_all_indexes AS pg_stat_get_numscans(I.oid) AS idx_scan, pg_stat_get_lastscan(I.oid) AS last_idx_scan, pg_stat_get_tuples_returned(I.oid) AS idx_tup_read, - pg_stat_get_tuples_fetched(I.oid) AS idx_tup_fetch + pg_stat_get_tuples_fetched(I.oid) AS idx_tup_fetch, + pg_stat_get_total_vacuum_time(I.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(I.oid) AS total_autovacuum_time, + pg_stat_get_total_vacuum_delay_time(I.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(I.oid) AS total_autovacuum_delay_time FROM pg_class C JOIN pg_index X ON C.oid = X.indrelid JOIN pg_class I ON I.oid = X.indexrelid @@ -1205,6 +1218,24 @@ CREATE VIEW pg_stat_database AS UNION ALL SELECT oid, datname FROM pg_database ) D; + +CREATE VIEW pg_stat_vacuum AS + SELECT + D.oid AS datid, + D.datname AS datname, + pg_stat_get_db_total_vacuum_time(D.oid) AS total_vacuum_time, + pg_stat_get_db_total_autovacuum_time(D.oid) AS total_autovacuum_time, + pg_stat_get_db_total_vacuum_delay_time(D.oid) AS total_vacuum_delay_time, + pg_stat_get_db_total_autovacuum_delay_time(D.oid) AS total_autovacuum_delay_time, + pg_stat_get_db_vacuum_failsafe_count(D.oid) AS vacuum_failsafe_count, + pg_stat_get_db_vacuum_interrupt_count(D.oid) AS vacuum_interrupt_count, + pg_stat_get_db_stat_reset_time(D.oid) AS stats_reset + FROM ( + SELECT 0 AS oid, NULL::name AS datname + UNION ALL + SELECT oid, datname FROM pg_database + ) D; + CREATE VIEW pg_resqueue_status AS SELECT q.rsqname, @@ -1564,7 +1595,8 @@ CREATE VIEW pg_stat_progress_analyze AS S.param5 AS ext_stats_computed, S.param6 AS child_tables_total, S.param7 AS child_tables_done, - CAST(S.param8 AS oid) AS current_child_table_relid + CAST(S.param8 AS oid) AS current_child_table_relid, + S.param9 / 1000000::double precision AS delay_time FROM pg_stat_get_progress_info('ANALYZE') AS S LEFT JOIN pg_database D ON S.datid = D.oid; @@ -1585,7 +1617,8 @@ CREATE VIEW pg_stat_progress_vacuum AS END AS phase, S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned, S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count, - S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples + S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples, + S.param8 / 1000000::double precision AS delay_time FROM pg_stat_get_progress_info('VACUUM') AS S LEFT JOIN pg_database D ON S.datid = D.oid; @@ -1933,4 +1966,4 @@ $$ WHERE pg_subscription.subname = sub_info.subname ) from gp_dist_random('gp_id'); -$$ LANGUAGE SQL; \ No newline at end of file +$$ LANGUAGE SQL; diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in index 41e4a2468e5..71fac72cd5f 100644 --- a/src/backend/catalog/system_views_gp.in +++ b/src/backend/catalog/system_views_gp.in @@ -14,6 +14,7 @@ pg_stat_all_tables pg_stat_archiver pg_stat_bgwriter pg_stat_database +pg_stat_vacuum pg_stat_database_conflicts pg_stat_gssapi pg_stat_operations diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql index 62bde4e3d96..8cd6d1638c2 100644 --- a/src/backend/catalog/system_views_gp_summary.sql +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -94,6 +94,23 @@ GROUP BY sdb.datname; +CREATE VIEW gp_stat_vacuum_summary AS +SELECT + sdb.datid, + sdb.datname, + sum(sdb.total_vacuum_time) as total_vacuum_time, + sum(sdb.total_autovacuum_time) as total_autovacuum_time, + sum(sdb.total_vacuum_delay_time) as total_vacuum_delay_time, + sum(sdb.total_autovacuum_delay_time) as total_autovacuum_delay_time, + max(sdb.vacuum_failsafe_count) as vacuum_failsafe_count, + max(sdb.vacuum_interrupt_count) as vacuum_interrupt_count, + max(sdb.stats_reset) as stats_reset +FROM + gp_stat_vacuum sdb +GROUP BY + sdb.datid, + sdb.datname; + -- Gather data from segments on user tables, and use data on coordinator on system tables. CREATE VIEW gp_stat_all_tables_summary AS SELECT @@ -120,7 +137,16 @@ SELECT s.vacuum_count, s.autovacuum_count, s.analyze_count, - s.autoanalyze_count + s.autoanalyze_count, + m.total_vacuum_time, + m.total_autovacuum_time, + m.total_analyze_time, + m.total_autoanalyze_time, + m.total_vacuum_delay_time, + m.total_autovacuum_delay_time, + m.vacuum_failsafe_count, + m.visible_page_marks_cleared, + m.frozen_page_marks_cleared FROM (SELECT allt.relid, @@ -148,7 +174,16 @@ FROM max(vacuum_count) as vacuum_count, max(autovacuum_count) as autovacuum_count, max(analyze_count) as analyze_count, - max(autoanalyze_count) as autoanalyze_count + max(autoanalyze_count) as autoanalyze_count, + case when d.policytype = 'r' then (sum(total_vacuum_time)/d.numsegments) else sum(total_vacuum_time) end total_vacuum_time, + case when d.policytype = 'r' then (sum(total_autovacuum_time)/d.numsegments) else sum(total_autovacuum_time) end total_autovacuum_time, + case when d.policytype = 'r' then (sum(total_analyze_time)/d.numsegments) else sum(total_analyze_time) end total_analyze_time, + case when d.policytype = 'r' then (sum(total_autoanalyze_time)/d.numsegments) else sum(total_autoanalyze_time) end total_autoanalyze_time, + case when d.policytype = 'r' then (sum(total_vacuum_delay_time)/d.numsegments) else sum(total_vacuum_delay_time) end total_vacuum_delay_time, + case when d.policytype = 'r' then (sum(total_autovacuum_delay_time)/d.numsegments) else sum(total_autovacuum_delay_time) end total_autovacuum_delay_time, + max(vacuum_failsafe_count) as vacuum_failsafe_count, + case when d.policytype = 'r' then (sum(visible_page_marks_cleared)/d.numsegments)::bigint else sum(visible_page_marks_cleared) end visible_page_marks_cleared, + case when d.policytype = 'r' then (sum(frozen_page_marks_cleared)/d.numsegments)::bigint else sum(frozen_page_marks_cleared) end frozen_page_marks_cleared FROM gp_dist_random('pg_stat_all_tables') allt inner join pg_class c @@ -229,7 +264,11 @@ SELECT m.idx_scan, m.last_idx_scan, m.idx_tup_read, - m.idx_tup_fetch + m.idx_tup_fetch, + m.total_vacuum_time, + m.total_autovacuum_time, + m.total_vacuum_delay_time, + m.total_autovacuum_delay_time FROM (SELECT alli.relid, @@ -240,7 +279,11 @@ FROM case when d.policytype = 'r' then (sum(alli.idx_scan)/d.numsegments)::bigint else sum(alli.idx_scan) end idx_scan, max(last_idx_scan) as last_idx_scan, case when d.policytype = 'r' then (sum(alli.idx_tup_read)/d.numsegments)::bigint else sum(alli.idx_tup_read) end idx_tup_read, - case when d.policytype = 'r' then (sum(alli.idx_tup_fetch)/d.numsegments)::bigint else sum(alli.idx_tup_fetch) end idx_tup_fetch + case when d.policytype = 'r' then (sum(alli.idx_tup_fetch)/d.numsegments)::bigint else sum(alli.idx_tup_fetch) end idx_tup_fetch, + case when d.policytype = 'r' then (sum(alli.total_vacuum_time)/d.numsegments) else sum(alli.total_vacuum_time) end total_vacuum_time, + case when d.policytype = 'r' then (sum(alli.total_autovacuum_time)/d.numsegments) else sum(alli.total_autovacuum_time) end total_autovacuum_time, + case when d.policytype = 'r' then (sum(alli.total_vacuum_delay_time)/d.numsegments) else sum(alli.total_vacuum_delay_time) end total_vacuum_delay_time, + case when d.policytype = 'r' then (sum(alli.total_autovacuum_delay_time)/d.numsegments) else sum(alli.total_autovacuum_delay_time) end total_autovacuum_delay_time FROM gp_dist_random('pg_stat_all_indexes') alli inner join pg_class c @@ -259,7 +302,7 @@ FROM pg_stat_all_indexes WHERE relid < 16384) m, pg_stat_all_indexes s -WHERE m.relid = s.relid; +WHERE m.relid = s.relid AND m.indexrelid = s.indexrelid; CREATE VIEW gp_stat_sys_indexes_summary AS SELECT * FROM gp_stat_all_indexes_summary diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 61c69b3ca4e..9f494e9fec0 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -120,6 +120,49 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_vacuum_failsafe_count */ +PG_STAT_GET_RELENTRY_INT64(vacuum_failsafe_count) + +/* pg_stat_get_visible_page_marks_cleared */ +PG_STAT_GET_RELENTRY_INT64(visible_page_marks_cleared) + +/* pg_stat_get_frozen_page_marks_cleared */ +PG_STAT_GET_RELENTRY_INT64(frozen_page_marks_cleared) + +#define PG_STAT_GET_RELENTRY_FLOAT8(stat) \ +Datum \ +CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ +{ \ + Oid relid = PG_GETARG_OID(0); \ + double result; \ + PgStat_StatTabEntry *tabentry; \ + \ + if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL) \ + result = 0; \ + else \ + result = (double) (tabentry->stat); \ + \ + PG_RETURN_FLOAT8(result); \ +} + +/* pg_stat_get_total_vacuum_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_vacuum_time) + +/* pg_stat_get_total_autovacuum_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autovacuum_time) + +/* pg_stat_get_total_analyze_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_analyze_time) + +/* pg_stat_get_total_autoanalyze_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autoanalyze_time) + +/* pg_stat_get_total_vacuum_delay_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_vacuum_delay_time) + +/* pg_stat_get_total_autovacuum_delay_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autovacuum_delay_time) + #define PG_STAT_GET_RELENTRY_TIMESTAMPTZ(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -1099,6 +1142,9 @@ PG_STAT_GET_DBENTRY_INT64(conflict_tablespace) /* pg_stat_get_db_deadlocks */ PG_STAT_GET_DBENTRY_INT64(deadlocks) +/* pg_stat_get_db_vacuum_interrupt_count */ +PG_STAT_GET_DBENTRY_INT64(vacuum_interrupt_count) + /* pg_stat_get_db_sessions */ PG_STAT_GET_DBENTRY_INT64(sessions) @@ -1248,6 +1294,21 @@ PG_STAT_GET_DBENTRY_FLOAT8_MS(blk_write_time) /* pg_stat_get_db_idle_in_transaction_time */ PG_STAT_GET_DBENTRY_FLOAT8_MS(idle_in_transaction_time) +/* pg_stat_get_db_total_vacuum_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_vacuum_time) + +/* pg_stat_get_db_total_autovacuum_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_autovacuum_time) + +/* pg_stat_get_db_total_vacuum_delay_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_vacuum_delay_time) + +/* pg_stat_get_db_total_autovacuum_delay_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_autovacuum_delay_time) + +/* pg_stat_get_db_vacuum_failsafe_count */ +PG_STAT_GET_DBENTRY_INT64(vacuum_failsafe_count) + /* pg_stat_get_db_session_time */ PG_STAT_GET_DBENTRY_FLOAT8_MS(session_time) diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 77f198458a8..21dd3822811 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302610061 +#define CATALOG_VERSION_NO 302610070 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 509c2e3be2c..c6a7bca35ce 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5473,6 +5473,22 @@ proname => 'pg_stat_get_autoanalyze_count', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_autoanalyze_count' }, +{ oid => '8406', descr => 'total vacuum time, in milliseconds', + proname => 'pg_stat_get_total_vacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_vacuum_time' }, +{ oid => '8407', descr => 'total autovacuum time, in milliseconds', + proname => 'pg_stat_get_total_autovacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autovacuum_time' }, +{ oid => '8408', descr => 'total analyze time, in milliseconds', + proname => 'pg_stat_get_total_analyze_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_analyze_time' }, +{ oid => '8409', descr => 'total autoanalyze time, in milliseconds', + proname => 'pg_stat_get_total_autoanalyze_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autoanalyze_time' }, { oid => '1936', descr => 'statistics: currently active backend IDs', proname => 'pg_stat_get_backend_idset', prorows => '100', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'int4', @@ -5709,6 +5725,54 @@ proname => 'pg_stat_get_db_blk_write_time', provolatile => 's', proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', prosrc => 'pg_stat_get_db_blk_write_time' }, +{ oid => '4707', + descr => 'statistics: number of times the all-visible marks in the visibility map were cleared for pages of this table', + proname => 'pg_stat_get_visible_page_marks_cleared', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_visible_page_marks_cleared' }, +{ oid => '4708', + descr => 'statistics: number of times the all-frozen marks in the visibility map were cleared for pages of this table', + proname => 'pg_stat_get_frozen_page_marks_cleared', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_frozen_page_marks_cleared' }, +{ oid => '4706', descr => 'statistics: number of vacuums interrupted by errors in database', + proname => 'pg_stat_get_db_vacuum_interrupt_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_vacuum_interrupt_count' }, +{ oid => '4700', descr => 'total vacuum delay time, in milliseconds', + proname => 'pg_stat_get_total_vacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_vacuum_delay_time' }, +{ oid => '4701', descr => 'total autovacuum delay time, in milliseconds', + proname => 'pg_stat_get_total_autovacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autovacuum_delay_time' }, +{ oid => '4702', descr => 'statistics: total vacuum time of tables in database, in milliseconds', + proname => 'pg_stat_get_db_total_vacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_vacuum_time' }, +{ oid => '4703', descr => 'statistics: total autovacuum time of tables in database, in milliseconds', + proname => 'pg_stat_get_db_total_autovacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_autovacuum_time' }, +{ oid => '4704', descr => 'statistics: total vacuum delay time in database, in milliseconds', + proname => 'pg_stat_get_db_total_vacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_vacuum_delay_time' }, +{ oid => '4705', descr => 'statistics: total autovacuum delay time in database, in milliseconds', + proname => 'pg_stat_get_db_total_autovacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_autovacuum_delay_time' }, +{ oid => '8691', + descr => 'statistics: number of vacuums that entered the wraparound failsafe mode', + proname => 'pg_stat_get_vacuum_failsafe_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_vacuum_failsafe_count' }, +{ oid => '8692', + descr => 'statistics: number of vacuums in database that entered the wraparound failsafe mode', + proname => 'pg_stat_get_db_vacuum_failsafe_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_vacuum_failsafe_count' }, { oid => '6185', descr => 'statistics: session time, in milliseconds', proname => 'pg_stat_get_db_session_time', provolatile => 's', proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', diff --git a/src/test/isolation/expected/vacuum-extending-freeze.out b/src/test/isolation/expected/vacuum-extending-freeze.out new file mode 100644 index 00000000000..fbd03d8da92 --- /dev/null +++ b/src/test/isolation/expected/vacuum-extending-freeze.out @@ -0,0 +1,197 @@ +Parsed test spec with 2 sessions + +starting permutation: s2_vacuum_freeze s1_get_set_vm_flags_stats s1_update_table s1_get_cleared_vm_flags_stats s2_vacuum_freeze s1_get_set_vm_flags_stats s2_vacuum_freeze s1_select_from_index s2_delete_from_table s1_get_cleared_vm_flags_stats s2_vacuum_freeze s1_get_set_vm_flags_stats s1_commit s1_get_cleared_vm_flags_stats +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +t |t +(1 row) + +step s1_update_table: + UPDATE vestat SET x = x + 1001 where x >= 2500; + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +t |t +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +t |t +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_select_from_index: + BEGIN ISOLATION LEVEL REPEATABLE READ; + SELECT count(x) FROM vestat WHERE x > 2000; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +count +----- + 3000 +(1 row) + +step s2_delete_from_table: + DELETE FROM vestat WHERE x > 4930; + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +f |f +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +f |f +(1 row) + +step s1_commit: + COMMIT; + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +t |t +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 5418fbf8b39..45cd0847b9f 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -173,3 +173,4 @@ test: vacuum-ao-concurrent-drop test: serializable-parallel-3 test: matview-write-skew +test: vacuum-extending-freeze diff --git a/src/test/isolation/specs/vacuum-extending-freeze.spec b/src/test/isolation/specs/vacuum-extending-freeze.spec new file mode 100644 index 00000000000..21173bfaff7 --- /dev/null +++ b/src/test/isolation/specs/vacuum-extending-freeze.spec @@ -0,0 +1,127 @@ +# Check VM counts after freezing and cumulative counts of cleared VM bits. +# A repeatable-read transaction keeps deleted tuples visible to its snapshot. +# It also caches the table statistics before the other session deletes rows: +# the VM bits clear during DELETE, while the cached counters stay unchanged +# until the reader commits. Force the writer's counters to flush so this +# distinction does not depend on the statistics flush interval. + +setup +{ + CREATE TABLE vestat (x int, y int) + WITH (autovacuum_enabled = off, fillfactor = 70); + + INSERT INTO vestat + SELECT i, i FROM generate_series(1, 5000) AS g(i); + + CREATE INDEX vestat_idx ON vestat (x); + + CREATE TABLE stats_state (frozen_flag_count int, all_visibile_flag_count int, + cleared_frozen_flag_count int, cleared_all_visibile_flag_count int); + INSERT INTO stats_state VALUES (0,0,0,0); + ANALYZE vestat; + + -- Ensure stats are flushed before starting the scenario + SELECT pg_stat_force_next_flush(); +} + +teardown +{ + DROP TABLE IF EXISTS vestat; + DROP TABLE IF EXISTS stats_state; + RESET vacuum_freeze_min_age; + RESET vacuum_freeze_table_age; + +} + +session s1 +setup +{ + SET stats_fetch_consistency = 'cache'; +} + +step s1_get_set_vm_flags_stats +{ + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; +} + +step s1_get_cleared_vm_flags_stats +{ + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; +} + +step s1_select_from_index +{ + BEGIN ISOLATION LEVEL REPEATABLE READ; + SELECT count(x) FROM vestat WHERE x > 2000; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; +} + +step s1_commit +{ + COMMIT; +} + +session s2 +setup +{ + -- Configure aggressive freezing vacuum behavior + SET vacuum_freeze_min_age = 0; + SET vacuum_freeze_table_age = 0; +} +step s2_delete_from_table +{ + DELETE FROM vestat WHERE x > 4930; + SELECT pg_stat_force_next_flush(); +} +step s2_vacuum_freeze +{ + VACUUM FREEZE vestat; +} + +step s1_update_table +{ + UPDATE vestat SET x = x + 1001 where x >= 2500; + SELECT pg_stat_force_next_flush(); +} + +permutation + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s1_update_table + s1_get_cleared_vm_flags_stats + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s2_vacuum_freeze + s1_select_from_index + s2_delete_from_table + s1_get_cleared_vm_flags_stats + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s1_commit + s1_get_cleared_vm_flags_stats diff --git a/src/test/isolation2/expected/vacuum_ao_stats.out b/src/test/isolation2/expected/vacuum_ao_stats.out new file mode 100644 index 00000000000..d70951bc01e --- /dev/null +++ b/src/test/isolation2/expected/vacuum_ao_stats.out @@ -0,0 +1,140 @@ +-- start_matchsubs +-- +-- # the segment that reports the injected error first is not deterministic +-- +-- m/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ +-- s/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error'/ +-- end_matchsubs +-- +-- The vacuum statistics of the core for append-optimized tables: the vacuum +-- and delay times of the tables and their indexes, and vacuum_interrupt_count. +-- +CREATE TABLE vacstat_ao_row (i int, v text) WITH (appendonly=true) DISTRIBUTED BY (i); +CREATE +CREATE TABLE vacstat_ao_col (i int, v text) WITH (appendonly=true, orientation=column) DISTRIBUTED BY (i); +CREATE +CREATE INDEX vacstat_ao_row_idx ON vacstat_ao_row (i); +CREATE +CREATE INDEX vacstat_ao_col_idx ON vacstat_ao_col (i); +CREATE +INSERT INTO vacstat_ao_row SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT 3000 +INSERT INTO vacstat_ao_col SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT 3000 +DELETE FROM vacstat_ao_row WHERE i % 2 = 0; +DELETE 1500 +DELETE FROM vacstat_ao_col WHERE i % 2 = 0; +DELETE 1500 + +-- A cost-limited vacuum sleeps, in the index passes at least, so both the +-- tables and the indexes get vacuum and delay time on every segment. +1: SET track_cost_delay_timing = on; +SET +1: SET vacuum_cost_delay = '1ms'; +SET +1: SET vacuum_cost_limit = 10; +SET +1: VACUUM vacstat_ao_row, vacstat_ao_col; +VACUUM +1: SELECT relname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed FROM gp_dist_random('pg_stat_all_tables') WHERE relname LIKE 'vacstat_ao_%' GROUP BY relname ORDER BY relname; + relname | segments_timed | segments_delayed +----------------+----------------+------------------ + vacstat_ao_col | 3 | 3 + vacstat_ao_row | 3 | 3 +(2 rows) +1: SELECT indexrelname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed FROM gp_dist_random('pg_stat_all_indexes') WHERE indexrelname LIKE 'vacstat_ao_%' GROUP BY indexrelname ORDER BY indexrelname; + indexrelname | segments_timed | segments_delayed +--------------------+----------------+------------------ + vacstat_ao_col_idx | 3 | 3 + vacstat_ao_row_idx | 3 | 3 +(2 rows) +1: RESET vacuum_cost_limit; +RESET +1: RESET vacuum_cost_delay; +RESET +1: RESET track_cost_delay_timing; +RESET + +-- A vacuum interrupted by an error. Start from zero on every instance. +1: SELECT pg_stat_reset() FROM gp_dist_random('gp_id'); + pg_stat_reset +--------------- + + + +(3 rows) +1: SELECT pg_stat_reset(); + pg_stat_reset +--------------- + +(1 row) +INSERT INTO vacstat_ao_row SELECT g, repeat('y', 100) FROM generate_series(1, 100) g; +INSERT 100 +DELETE FROM vacstat_ao_row WHERE i % 2 = 1; +DELETE 1550 + +-- Make the compact phase fail on the segments. The fault is scoped to this +-- session, so that no other backend can consume it. +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'error', '', '', '', 1, -1, 0, dbid, current_setting('gp_session_id')::int) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1: VACUUM vacstat_ao_row; +ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- The phases of an append-optimized vacuum share one AOVacuumRelStats, which +-- is freed at the end of the last phase. A vacuum that fails in between used +-- to leave it behind, and the next vacuum of the relation then reused its +-- start time, so the time of everything that happened in between was added +-- to the relation's cumulative vacuum time. Let time pass. +1: SELECT pg_sleep(5); + pg_sleep +---------- + +(1 row) + +1: VACUUM vacstat_ao_row; +VACUUM + +-- The second vacuum takes milliseconds; without resetting the stats of the +-- failed one it would be credited with the five seconds slept above. +1: SELECT sum(total_vacuum_time) < 3000 AS vacuum_time_is_sane FROM gp_dist_random('pg_stat_all_tables') WHERE relname = 'vacstat_ao_row'; + vacuum_time_is_sane +--------------------- + t +(1 row) + +-- The segments that raised the error counted it; the coordinator, which only +-- dispatched the failing phase, did not. A segment may also be canceled +-- after its phase is already done, so check for "some", not for "all". +-- +-- A segment adds the count to its pending statistics when its transaction +-- ends, and flushes them when it is idle outside a transaction, which it is +-- not while its part of the failed distributed transaction waits for the +-- coordinator; the second vacuum above has ended that. Give the flush the +-- second it may wait for (PGSTAT_MIN_INTERVAL). +1: SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) +1: SELECT gp_segment_id = -1 AS coordinator, sum(vacuum_interrupt_count) > 0 AS interrupted FROM gp_stat_vacuum WHERE datname = current_database() GROUP BY 1 ORDER BY 1; + coordinator | interrupted +-------------+------------- + f | t + t | f +(2 rows) + +1: DROP TABLE vacstat_ao_row, vacstat_ao_col; +DROP +1q: ... diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 82d23731fb6..3e72b37ce0a 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -152,6 +152,7 @@ test: uao/fast_analyze_row test: uao/create_index_allows_readonly_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop +test: vacuum_ao_stats # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/src/test/isolation2/sql/vacuum_ao_stats.sql b/src/test/isolation2/sql/vacuum_ao_stats.sql new file mode 100644 index 00000000000..4fc15732c6c --- /dev/null +++ b/src/test/isolation2/sql/vacuum_ao_stats.sql @@ -0,0 +1,84 @@ +-- start_matchsubs +-- +-- # the segment that reports the injected error first is not deterministic +-- +-- m/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ +-- s/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error'/ +-- end_matchsubs +-- +-- The vacuum statistics of the core for append-optimized tables: the vacuum +-- and delay times of the tables and their indexes, and vacuum_interrupt_count. +-- +CREATE TABLE vacstat_ao_row (i int, v text) WITH (appendonly=true) DISTRIBUTED BY (i); +CREATE TABLE vacstat_ao_col (i int, v text) WITH (appendonly=true, orientation=column) DISTRIBUTED BY (i); +CREATE INDEX vacstat_ao_row_idx ON vacstat_ao_row (i); +CREATE INDEX vacstat_ao_col_idx ON vacstat_ao_col (i); +INSERT INTO vacstat_ao_row SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT INTO vacstat_ao_col SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +DELETE FROM vacstat_ao_row WHERE i % 2 = 0; +DELETE FROM vacstat_ao_col WHERE i % 2 = 0; + +-- A cost-limited vacuum sleeps, in the index passes at least, so both the +-- tables and the indexes get vacuum and delay time on every segment. +1: SET track_cost_delay_timing = on; +1: SET vacuum_cost_delay = '1ms'; +1: SET vacuum_cost_limit = 10; +1: VACUUM vacstat_ao_row, vacstat_ao_col; +1: SELECT relname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, + count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed + FROM gp_dist_random('pg_stat_all_tables') + WHERE relname LIKE 'vacstat_ao_%' GROUP BY relname ORDER BY relname; +1: SELECT indexrelname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, + count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed + FROM gp_dist_random('pg_stat_all_indexes') + WHERE indexrelname LIKE 'vacstat_ao_%' GROUP BY indexrelname ORDER BY indexrelname; +1: RESET vacuum_cost_limit; +1: RESET vacuum_cost_delay; +1: RESET track_cost_delay_timing; + +-- A vacuum interrupted by an error. Start from zero on every instance. +1: SELECT pg_stat_reset() FROM gp_dist_random('gp_id'); +1: SELECT pg_stat_reset(); +INSERT INTO vacstat_ao_row SELECT g, repeat('y', 100) FROM generate_series(1, 100) g; +DELETE FROM vacstat_ao_row WHERE i % 2 = 1; + +-- Make the compact phase fail on the segments. The fault is scoped to this +-- session, so that no other backend can consume it. +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'error', '', '', '', 1, -1, 0, dbid, + current_setting('gp_session_id')::int) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1: VACUUM vacstat_ao_row; +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- The phases of an append-optimized vacuum share one AOVacuumRelStats, which +-- is freed at the end of the last phase. A vacuum that fails in between used +-- to leave it behind, and the next vacuum of the relation then reused its +-- start time, so the time of everything that happened in between was added +-- to the relation's cumulative vacuum time. Let time pass. +1: SELECT pg_sleep(5); + +1: VACUUM vacstat_ao_row; + +-- The second vacuum takes milliseconds; without resetting the stats of the +-- failed one it would be credited with the five seconds slept above. +1: SELECT sum(total_vacuum_time) < 3000 AS vacuum_time_is_sane + FROM gp_dist_random('pg_stat_all_tables') WHERE relname = 'vacstat_ao_row'; + +-- The segments that raised the error counted it; the coordinator, which only +-- dispatched the failing phase, did not. A segment may also be canceled +-- after its phase is already done, so check for "some", not for "all". +-- +-- A segment adds the count to its pending statistics when its transaction +-- ends, and flushes them when it is idle outside a transaction, which it is +-- not while its part of the failed distributed transaction waits for the +-- coordinator; the second vacuum above has ended that. Give the flush the +-- second it may wait for (PGSTAT_MIN_INTERVAL). +1: SELECT pg_sleep(2); +1: SELECT gp_segment_id = -1 AS coordinator, + sum(vacuum_interrupt_count) > 0 AS interrupted + FROM gp_stat_vacuum WHERE datname = current_database() + GROUP BY 1 ORDER BY 1; + +1: DROP TABLE vacstat_ao_row, vacstat_ao_col; +1q: diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..e64c2f843f5 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,10 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/016_index_vacuum_time.pl', + 't/017_vacuum_failsafe.pl', + 't/018_vacuum_interrupts.pl', + 't/020_relallfrozen.pl', ], }, } diff --git a/src/test/modules/test_misc/t/016_index_vacuum_time.pl b/src/test/modules/test_misc/t/016_index_vacuum_time.pl new file mode 100644 index 00000000000..4eb02ab2517 --- /dev/null +++ b/src/test/modules/test_misc/t/016_index_vacuum_time.pl @@ -0,0 +1,80 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test the total_vacuum_time counter of pg_stat_all_indexes: the time vacuum +# spent processing each index, accumulated over bulkdelete and cleanup passes, +# mirroring the table-level counter in pg_stat_all_tables. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +track_cost_delay_timing = on +]); +$node->start; + +# The index has to span enough pages for its vacuum passes to exceed +# vacuum_cost_limit on buffer hits alone (its pages are still dirty from the +# load, so dirtying them costs nothing). Scale the row count with the block +# size, so that builds with larger blocks (32kB in Cloudberry) keep the page +# count of the default 8kB. +my $nrows = 100000 * + ($node->safe_psql('postgres', 'SHOW block_size') / 8192); + +# Enough rows that the bulkdelete pass over the index takes measurable time; +# a small vacuum_cost_delay adds deterministic delay time on fast machines. +$node->safe_psql( + 'postgres', qq[ +CREATE TABLE vactime_t (id int PRIMARY KEY, v text) WITH (autovacuum_enabled = off); +INSERT INTO vactime_t SELECT g, repeat('x', 10) FROM generate_series(1, $nrows) g; +DELETE FROM vactime_t WHERE id % 2 = 0; +]); +$node->safe_psql( + 'postgres', qq[ +SET vacuum_cost_delay = '1ms'; +SET vacuum_cost_limit = 200; +VACUUM vactime_t; +]); + +# The upper bound guards against garbage such as an epoch-based elapsed time +# leaking into the counter. +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_time > 0 AND total_vacuum_time < 600000 + FROM pg_stat_all_indexes WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'total_vacuum_time advanced sanely for the index in pg_stat_all_indexes'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_delay_time > 0 AND total_vacuum_delay_time <= total_vacuum_time + FROM pg_stat_all_indexes WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'total_vacuum_delay_time advanced and not above total_vacuum_time'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_autovacuum_time = 0 FROM pg_stat_all_indexes + WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'manual vacuum did not count into total_autovacuum_time'); + +# The same run must have accumulated into the database-wide totals. +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_time > 0 AND total_vacuum_time < 600000 + AND total_vacuum_delay_time > 0 + AND total_vacuum_delay_time <= total_vacuum_time + FROM pg_stat_vacuum WHERE datname = current_database()]), + 't', + 'database-wide vacuum times advanced sanely in pg_stat_vacuum'); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_misc/t/017_vacuum_failsafe.pl b/src/test/modules/test_misc/t/017_vacuum_failsafe.pl new file mode 100644 index 00000000000..9332c250477 --- /dev/null +++ b/src/test/modules/test_misc/t/017_vacuum_failsafe.pl @@ -0,0 +1,71 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that vacuums entering the wraparound failsafe mode are counted in +# pg_stat_all_tables.vacuum_failsafe_count and aggregated per database in +# pg_stat_vacuum.vacuum_failsafe_count. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +# The failsafe cutoff is clamped to 1.05 * autovacuum_freeze_max_age, so use +# the minimum allowed value to keep the number of XIDs to burn small. +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +autovacuum_freeze_max_age = 100000 +]); +$node->start; + +$node->safe_psql( + 'postgres', qq[ + CREATE TABLE tab_failsafe (i int); + INSERT INTO tab_failsafe SELECT generate_series(1, 100); +]); + +# A vacuum without failsafe pressure must not bump the counter. +$node->safe_psql('postgres', 'VACUUM tab_failsafe;'); +my $count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_all_tables WHERE relname = 'tab_failsafe';] +); +is($count, '0', 'normal vacuum does not count as failsafe'); + +# Age the table past 1.05 * autovacuum_freeze_max_age: burn XIDs with +# aborted subtransactions (each aborted subxact consumes an assigned XID). +$node->safe_psql( + 'postgres', qq[ + CREATE TABLE burn_xids (i int); + DO \$\$ + BEGIN + FOR i IN 1..110000 LOOP + BEGIN + INSERT INTO burn_xids VALUES (1); + RAISE EXCEPTION 'burn'; + EXCEPTION WHEN OTHERS THEN + END; + END LOOP; + END \$\$; +]); + +# vacuum_failsafe_age = 0 makes the (clamped) cutoff kick in immediately. +$node->safe_psql( + 'postgres', qq[ + SET vacuum_failsafe_age = 0; + SET vacuum_multixact_failsafe_age = 0; + VACUUM tab_failsafe; +]); + +$count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_all_tables WHERE relname = 'tab_failsafe';] +); +is($count, '1', 'failsafe vacuum counted in pg_stat_all_tables'); + +my $db_count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_vacuum WHERE datname = 'postgres';] +); +is($db_count, '1', 'failsafe vacuum counted in pg_stat_vacuum'); + +done_testing(); diff --git a/src/test/modules/test_misc/t/018_vacuum_interrupts.pl b/src/test/modules/test_misc/t/018_vacuum_interrupts.pl new file mode 100644 index 00000000000..f1cb11462ac --- /dev/null +++ b/src/test/modules/test_misc/t/018_vacuum_interrupts.pl @@ -0,0 +1,71 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test the vacuum_interrupt_count counter of pg_stat_vacuum. +# +# vacuum_interrupt_count records how many times a vacuum in the database was +# interrupted by an error. We provoke that by starting a vacuum that sleeps +# at its cost-based delay points and canceling it, with pg_cancel_backend(), +# while it is still running. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +]); +$node->start; + +# fillfactor = 10 spreads the rows over many pages so the vacuum hits enough +# cost-delay points to stay running until we cancel it. +$node->safe_psql( + 'postgres', qq[ +CREATE TABLE vacstat_int (id int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vacstat_int SELECT generate_series(1, 1000); +DELETE FROM vacstat_int; +]); + +# Start a vacuum that sleeps at every cost-delay point, in the background. The +# \echo lets query_until() return as soon as the VACUUM has been launched. +my $appname = 'vacuum_interrupt_test'; +my $vac = $node->background_psql('postgres', on_error_stop => 0); +$vac->query_until( + qr/start/, qq[ +SET application_name = '$appname'; +SET vacuum_cost_delay = '100ms'; +SET vacuum_cost_limit = 1; +\\echo start +VACUUM vacstat_int; +]); + +# Wait until the vacuum is actually running, then cancel it. +$node->poll_query_until( + 'postgres', qq[ +SELECT count(*) = 1 FROM pg_stat_activity + WHERE application_name = '$appname' AND query LIKE 'VACUUM%' AND state = 'active']) + or die "timed out waiting for the vacuum to start"; + +my $cancelled = $node->safe_psql( + 'postgres', qq[ +SELECT pg_cancel_backend(pid) FROM pg_stat_activity + WHERE application_name = '$appname' AND query LIKE 'VACUUM%']); +is($cancelled, 't', 'canceled the running vacuum'); + +$vac->quit; +like($vac->{stderr}, qr/canceling statement due to user request/, + 'vacuum canceled by user request'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT vacuum_interrupt_count > 0 FROM pg_stat_vacuum WHERE datname = current_database()]), + 't', + 'vacuum_interrupt_count advanced in pg_stat_vacuum'); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_misc/t/020_relallfrozen.pl b/src/test/modules/test_misc/t/020_relallfrozen.pl new file mode 100644 index 00000000000..5bc2f0b085b --- /dev/null +++ b/src/test/modules/test_misc/t/020_relallfrozen.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Exercise the catalog update paths independently of vacuum reporting. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('relallfrozen'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->start; +$node->safe_psql('postgres', q{ +CREATE EXTENSION pg_visibility; +CREATE TABLE frozen_heap (id int, padding text); +INSERT INTO frozen_heap SELECT i, repeat('x', 100) FROM generate_series(1, 2000) i; +VACUUM (FREEZE) frozen_heap; +}); + +sub check_map +{ + my ($label) = @_; + is($node->safe_psql('postgres', q{ +SELECT relallfrozen = all_frozen AND relallvisible = all_visible + AND relallfrozen <= relallvisible AND relallvisible <= relpages +FROM pg_class, pg_visibility_map_summary('frozen_heap') +WHERE oid = 'frozen_heap'::regclass; +}), 't', $label); +} + +check_map('VACUUM FREEZE records both visibility-map counts'); +is($node->safe_psql('postgres', q{ +SELECT relallfrozen > 0 FROM pg_class WHERE oid = 'frozen_heap'::regclass +}), 't', 'frozen heap has all-frozen pages'); +$node->safe_psql('postgres', q{ +UPDATE frozen_heap SET padding = 'updated' WHERE id = 1; +ANALYZE frozen_heap; +}); +check_map('ANALYZE refreshes both counts after a VM bit is cleared'); +$node->safe_psql('postgres', 'CREATE INDEX frozen_heap_idx ON frozen_heap(id)'); +check_map('CREATE INDEX preserves heap visibility-map counts'); +is($node->safe_psql('postgres', q{ +SELECT relallfrozen = 0 AND relallvisible = 0 +FROM pg_class WHERE oid = 'frozen_heap_idx'::regclass +}), 't', 'index has zero visibility-map counts'); +$node->safe_psql('postgres', 'VACUUM (FULL) frozen_heap'); +check_map('VACUUM FULL swaps visibility-map statistics with the new heap'); +$node->safe_psql('postgres', 'TRUNCATE frozen_heap'); +check_map('TRUNCATE resets both visibility-map counts'); +for my $orientation ('row', 'column') +{ + $node->safe_psql('postgres', qq{ +CREATE TABLE frozen_ao_$orientation (id int) +WITH (appendoptimized=true, orientation=$orientation); +INSERT INTO frozen_ao_$orientation SELECT generate_series(1, 100); +CREATE INDEX frozen_ao_${orientation}_idx ON frozen_ao_$orientation(id); +VACUUM (ANALYZE) frozen_ao_$orientation; +}); + is($node->safe_psql('postgres', qq{ +SELECT bool_and(relallfrozen = 0 AND relallvisible = 0) +FROM pg_class WHERE oid IN ('frozen_ao_$orientation'::regclass, + 'frozen_ao_${orientation}_idx'::regclass) +}), 't', "AO $orientation table and index have no visibility map"); +} +$node->stop; +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 614ffea1784..94c1f53b4c9 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1798,7 +1798,16 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_vacuum_count(c.oid) AS vacuum_count, pg_stat_get_autovacuum_count(c.oid) AS autovacuum_count, pg_stat_get_analyze_count(c.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count, + pg_stat_get_total_vacuum_time(c.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(c.oid) AS total_autovacuum_time, + pg_stat_get_total_analyze_time(c.oid) AS total_analyze_time, + pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time, + pg_stat_get_total_vacuum_delay_time(c.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(c.oid) AS total_autovacuum_delay_time, + pg_stat_get_vacuum_failsafe_count(c.oid) AS vacuum_failsafe_count, + pg_stat_get_visible_page_marks_cleared(c.oid) AS visible_page_marks_cleared, + pg_stat_get_frozen_page_marks_cleared(c.oid) AS frozen_page_marks_cleared FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -1916,7 +1925,8 @@ pg_stat_progress_analyze| SELECT d.gp_segment_id, s.param5 AS ext_stats_computed, s.param6 AS child_tables_total, s.param7 AS child_tables_done, - (s.param8)::oid AS current_child_table_relid + (s.param8)::oid AS current_child_table_relid, + ((s.param9)::double precision / (1000000)::double precision) AS delay_time FROM (pg_stat_get_progress_info('ANALYZE'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_basebackup| SELECT gp_execution_segment() AS gp_segment_id, @@ -2043,7 +2053,8 @@ pg_stat_progress_vacuum| SELECT d.gp_segment_id, s.param4 AS heap_blks_vacuumed, s.param5 AS index_vacuum_count, s.param6 AS max_dead_tuples, - s.param7 AS num_dead_tuples + s.param7 AS num_dead_tuples, + ((s.param8)::double precision / (1000000)::double precision) AS delay_time FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_recovery_prefetch| SELECT stats_reset, @@ -2168,7 +2179,16 @@ pg_stat_sys_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + total_vacuum_time, + total_autovacuum_time, + total_analyze_time, + total_autoanalyze_time, + total_vacuum_delay_time, + total_autovacuum_delay_time, + vacuum_failsafe_count, + visible_page_marks_cleared, + frozen_page_marks_cleared FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2216,9 +2236,33 @@ pg_stat_user_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + total_vacuum_time, + total_autovacuum_time, + total_analyze_time, + total_autoanalyze_time, + total_vacuum_delay_time, + total_autovacuum_delay_time, + vacuum_failsafe_count, + visible_page_marks_cleared, + frozen_page_marks_cleared FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum| SELECT d.oid AS datid, + d.datname, + pg_stat_get_db_total_vacuum_time(d.oid) AS total_vacuum_time, + pg_stat_get_db_total_autovacuum_time(d.oid) AS total_autovacuum_time, + pg_stat_get_db_total_vacuum_delay_time(d.oid) AS total_vacuum_delay_time, + pg_stat_get_db_total_autovacuum_delay_time(d.oid) AS total_autovacuum_delay_time, + pg_stat_get_db_vacuum_failsafe_count(d.oid) AS vacuum_failsafe_count, + pg_stat_get_db_vacuum_interrupt_count(d.oid) AS vacuum_interrupt_count, + pg_stat_get_db_stat_reset_time(d.oid) AS stats_reset + FROM ( SELECT 0 AS oid, + NULL::name AS datname + UNION ALL + SELECT pg_database.oid, + pg_database.datname + FROM pg_database) d; pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index 573f7527fde..8cb79eaae45 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -28,4 +28,76 @@ AND relallvisible = t (1 row) -DROP TABLE vacstat_test +DROP TABLE vacstat_test; +-- relallfrozen follows segment catalogs for dist tables. +CREATE TABLE frozen_stats_dist (a int) DISTRIBUTED BY (a); +INSERT INTO frozen_stats_dist SELECT generate_series(1, 2000); +ANALYZE frozen_stats_dist; +VACUUM (FREEZE) frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +UPDATE frozen_stats_dist SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +VACUUM (FULL) frozen_stats_dist; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + rewrite_clears_map +-------------------- + t +(1 row) + +DROP TABLE frozen_stats_dist; +-- relallfrozen follows segment catalogs for repl tables. +CREATE TABLE frozen_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO frozen_stats_repl SELECT generate_series(1, 2000); +ANALYZE frozen_stats_repl; +VACUUM (FREEZE) frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +UPDATE frozen_stats_repl SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +VACUUM (FULL) frozen_stats_repl; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + rewrite_clears_map +-------------------- + t +(1 row) + +DROP TABLE frozen_stats_repl; diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index 45a9b6466c4..99cf075cc84 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -19,4 +19,48 @@ AND relallvisible = (SELECT SUM(relallvisible) FROM gp_dist_random('pg_class') WHERE oid='vacstat_test'::regclass); -DROP TABLE vacstat_test +DROP TABLE vacstat_test; + +-- relallfrozen follows segment catalogs for dist tables. +CREATE TABLE frozen_stats_dist (a int) DISTRIBUTED BY (a); +INSERT INTO frozen_stats_dist SELECT generate_series(1, 2000); +ANALYZE frozen_stats_dist; +VACUUM (FREEZE) frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +UPDATE frozen_stats_dist SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +VACUUM (FULL) frozen_stats_dist; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +DROP TABLE frozen_stats_dist; + +-- relallfrozen follows segment catalogs for repl tables. +CREATE TABLE frozen_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO frozen_stats_repl SELECT generate_series(1, 2000); +ANALYZE frozen_stats_repl; +VACUUM (FREEZE) frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +UPDATE frozen_stats_repl SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +VACUUM (FULL) frozen_stats_repl; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +DROP TABLE frozen_stats_repl; From 38284e39966325758c27bd29d097e769fb9bbb91 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 7 Oct 2026 17:17:19 +0300 Subject: [PATCH 40/50] Store vacuum work counters in native cumulative statistics Connect the previously introduced heap, index and AO measurements to ordinary relation and database statistics independently of any extension hook. Include them in native snapshots, resets, transactional drops and clean-shutdown persistence. Store dead_tuples and total_file_segs as last-run samples; other work counters accumulate. Exclude index reports and last-run samples from database totals. Advance the statistics-file format for the stored fields. Catalog functions, views and tests follow in a separate SQL reporting commit. --- src/backend/access/heap/vacuumlazy.c | 7 ++- src/backend/commands/vacuum.c | 6 +- src/backend/commands/vacuum_ao.c | 5 +- src/backend/commands/vacuumparallel.c | 2 +- src/backend/utils/activity/pgstat_database.c | 15 +++++ src/backend/utils/activity/pgstat_relation.c | 61 ++++++++++++++++++++ src/include/commands/vacuum.h | 9 ++- src/include/pgstat.h | 13 ++++- 8 files changed, 102 insertions(+), 16 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 80a28ac5c19..d2effdd5fb3 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -611,7 +611,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false, true); - /* Assemble backend-local measurements for subsequent reporting. */ + /* Core work counters are collected even without an extension hook. */ { PgStat_VacuumStats stats = {0}; @@ -627,6 +627,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, stats.pages_scanned = vacrel->scanned_pages; stats.pages_removed = vacrel->removed_pages; stats.missed_dead_pages = vacrel->missed_dead_pages; + pgstat_report_vacuum_stats(rel, &stats); } /* @@ -2809,7 +2810,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); - vacuum_measure_index_stats(indrel, istat, prev_tuples_removed, + vacuum_report_index_stats(indrel, istat, prev_tuples_removed, prev_pages_newly_deleted, false); /* Revert to the previous phase information for error traceback */ @@ -2871,7 +2872,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, InvalidBlockNumber, InvalidOffsetNumber); istat = vac_cleanup_one_index(&ivinfo, istat); - vacuum_measure_index_stats(indrel, istat, prev_tuples_removed, + vacuum_report_index_stats(indrel, istat, prev_tuples_removed, prev_pages_newly_deleted, true); /* Revert to the previous phase information for error traceback */ diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 5821a16b33a..9b6eb745e5c 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -3914,10 +3914,9 @@ vac_cmp_itemptr(const void *left, const void *right) return 0; } - -/* Measure index work once per call and remaining dead pages at cleanup. */ +/* Report index work once per call; report remaining dead pages at cleanup. */ void -vacuum_measure_index_stats(Relation indrel, const IndexBulkDeleteResult *istat, +vacuum_report_index_stats(Relation indrel, const IndexBulkDeleteResult *istat, double prev_tuples_removed, BlockNumber prev_pages_newly_deleted, bool cleanup) { @@ -3933,4 +3932,5 @@ vacuum_measure_index_stats(Relation indrel, const IndexBulkDeleteResult *istat, istat->pages_newly_deleted; if (cleanup && istat->pages_deleted > istat->pages_free) stats.dead_pages = istat->pages_deleted - istat->pages_free; + pgstat_report_vacuum_stats(indrel, &stats); } diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 0e7d98739da..76c3fca4585 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -498,6 +498,7 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy stats.tuples_moved = vacrelstats->tuples_moved; stats.pages_removed = vacrelstats->nbytes_truncated / BLCKSZ + (vacrelstats->nbytes_truncated % BLCKSZ != 0); + pgstat_report_vacuum_stats(rel, &stats); pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); @@ -617,7 +618,7 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs elevel, bstrategy, &result); - vacuum_measure_index_stats(Irel[i], &result, 0, 0, final_cleanup); + vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); } } else @@ -633,7 +634,7 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs bstrategy, vacrelstats, &result); - vacuum_measure_index_stats(Irel[i], &result, 0, 0, final_cleanup); + vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); } } } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 3c037e4eaee..e8d39e9c011 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -882,7 +882,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } - vacuum_measure_index_stats(indrel, istat_res, prev_tuples_removed, + vacuum_report_index_stats(indrel, istat_res, prev_tuples_removed, prev_pages_newly_deleted, indstats->status == PARALLEL_INDVAC_STATUS_NEED_CLEANUP); diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 813aa359fa0..c0a4ce061fc 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -418,6 +418,21 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(blk_read_time); PGSTAT_ACCUM_DBCOUNT(blk_write_time); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.tuples_deleted); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.pages_deleted); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.dead_pages); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.pages_frozen); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.pages_all_visible); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.freeze_age_vacuum_count); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.tuples_frozen); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.recently_dead_tuples); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.missed_dead_tuples); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.pages_scanned); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.pages_removed); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.missed_dead_pages); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.compacted_segments); + PGSTAT_ACCUM_DBCOUNT(vacuum_stats.tuples_moved); + PGSTAT_ACCUM_DBCOUNT(sessions); PGSTAT_ACCUM_DBCOUNT(session_time); PGSTAT_ACCUM_DBCOUNT(active_time); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 45bbf9abcb7..698f57b8394 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -359,6 +359,67 @@ pgstat_report_analyze(Relation rel, pgstat_flush_io(false); } +/* + * Persist the work measured by VACUUM independently of any extension hook. + * Index calls report deltas, while a table reports once on completion. The + * database aggregate describes table work only. Use the regular relation + * kind so snapshots, resets, drop handling and persistence work as usual. + */ +void +pgstat_report_vacuum_stats(Relation rel, const PgStat_VacuumStats *stats) +{ + Oid dboid = rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId; + PgStat_EntryRef *entry_ref; + PgStatShared_Relation *shared; + PgStat_VacuumStats *dst; + + if (!pgstat_track_counts) + return; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, + RelationGetRelid(rel), false); + shared = (PgStatShared_Relation *) entry_ref->shared_stats; + dst = &shared->stats.vacuum_stats; + dst->tuples_deleted += stats->tuples_deleted; + dst->pages_deleted += stats->pages_deleted; + dst->dead_pages += stats->dead_pages; + dst->pages_frozen += stats->pages_frozen; + dst->pages_all_visible += stats->pages_all_visible; + dst->freeze_age_vacuum_count += stats->freeze_age_vacuum_count; + dst->tuples_frozen += stats->tuples_frozen; + dst->recently_dead_tuples += stats->recently_dead_tuples; + dst->missed_dead_tuples += stats->missed_dead_tuples; + dst->pages_scanned += stats->pages_scanned; + dst->pages_removed += stats->pages_removed; + dst->missed_dead_pages += stats->missed_dead_pages; + dst->compacted_segments += stats->compacted_segments; + dst->tuples_moved += stats->tuples_moved; + dst->dead_tuples = stats->dead_tuples; + dst->total_file_segs = stats->total_file_segs; + pgstat_unlock_entry(entry_ref); + + if (rel->rd_rel->relkind != RELKIND_INDEX) + { + PgStat_StatDBEntry *dbentry = pgstat_prep_database_pending(dboid); + + dst = &dbentry->vacuum_stats; + dst->tuples_deleted += stats->tuples_deleted; + dst->pages_deleted += stats->pages_deleted; + dst->dead_pages += stats->dead_pages; + dst->pages_frozen += stats->pages_frozen; + dst->pages_all_visible += stats->pages_all_visible; + dst->freeze_age_vacuum_count += stats->freeze_age_vacuum_count; + dst->tuples_frozen += stats->tuples_frozen; + dst->recently_dead_tuples += stats->recently_dead_tuples; + dst->missed_dead_tuples += stats->missed_dead_tuples; + dst->pages_scanned += stats->pages_scanned; + dst->pages_removed += stats->pages_removed; + dst->missed_dead_pages += stats->missed_dead_pages; + dst->compacted_segments += stats->compacted_segments; + dst->tuples_moved += stats->tuples_moved; + } +} + /* * count a tuple insertion of n tuples */ diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index b49668ba9df..c9c28a8972d 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -475,6 +475,10 @@ extern bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs); extern bool vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs); extern void vac_update_datfrozenxid(void); +extern void vacuum_report_index_stats(Relation indrel, + const IndexBulkDeleteResult *istat, + double prev_tuples_removed, + BlockNumber prev_pages_newly_deleted, bool cleanup); extern void vacuum_delay_point(void); extern bool vacuum_is_relation_owner(Oid relid, Form_pg_class reltuple, bits32 options); @@ -491,11 +495,6 @@ extern Size vac_max_items_to_alloc_size(int max_items); /* In postmaster/autovacuum.c */ extern void AutoVacuumUpdateCostLimit(void); extern void VacuumUpdateCosts(void); -extern void vacuum_measure_index_stats(Relation indrel, - const IndexBulkDeleteResult *istat, - double prev_tuples_removed, - BlockNumber prev_pages_newly_deleted, bool cleanup); - /* in commands/vacuumparallel.c */ extern ParallelVacuumState *parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, int nrequested_workers, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 776ec25ae56..0fe91d31d07 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -91,7 +91,12 @@ typedef enum SessionEndType */ typedef int64 PgStat_Counter; -/* Backend-local work measurements for one VACUUM operation. */ +/* + * Work measured by VACUUM, stored in ordinary cumulative statistics. + * dead_tuples and total_file_segs are samples from the last completed run; + * all other fields accumulate across runs. Database totals omit snapshots + * and index reports, whose work is already part of the table vacuum. + */ typedef struct PgStat_VacuumStats { PgStat_Counter tuples_deleted; @@ -260,7 +265,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB8 typedef struct PgStat_ArchiverStats { @@ -377,6 +382,7 @@ typedef struct PgStat_StatDBEntry PgStat_Counter sessions_killed; TimestampTz stat_reset_timestamp; + PgStat_VacuumStats vacuum_stats; } PgStat_StatDBEntry; typedef struct PgStat_StatFuncEntry @@ -449,6 +455,7 @@ typedef struct PgStat_StatTabEntry PgStat_Counter analyze_count; TimestampTz last_autoanalyze_time; /* autovacuum initiated */ PgStat_Counter autoanalyze_count; + PgStat_VacuumStats vacuum_stats; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -672,6 +679,8 @@ extern void pgstat_init_relation(Relation rel); extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); +extern void pgstat_report_vacuum_stats(Relation rel, + const PgStat_VacuumStats *stats); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); extern void pgstat_report_analyze(Relation rel, From 33647dd98d5ab72dea1617c9bdaf7efe77a01bcc Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 8 Oct 2026 00:00:00 +0300 Subject: [PATCH 41/50] Expose native VACUUM work counters in catalog functions and views Publish cumulative tuple, page and AO compaction counters through native SQL getters and local, per-segment and cluster summary views. Keep last-run snapshots separate from cumulative work and normalize replicated relations. Include heap/index/AO tests for counters, resets, snapshots and persistence. --- doc/src/sgml/monitoring.sgml | 113 ++++++++++++++ src/backend/catalog/system_views.sql | 56 +++++++ src/backend/catalog/system_views_gp.in | 3 + .../catalog/system_views_gp_summary.sql | 113 ++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 54 +++++++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 16 ++ src/test/modules/test_misc/meson.build | 6 + .../test_misc/t/021_heap_page_counters.pl | 92 +++++++++++ .../test_misc/t/022_ao_hidden_tuples.pl | 104 +++++++++++++ .../modules/test_misc/t/023_dead_pages.pl | 145 ++++++++++++++++++ .../modules/test_misc/t/024_ao_dead_pages.pl | 67 ++++++++ .../t/027_freeze_age_vacuum_count.pl | 77 ++++++++++ .../test_misc/t/028_vacuum_work_core.pl | 97 ++++++++++++ src/test/regress/expected/rules.out | 57 +++++++ src/test/regress/expected/vacuum_stats.out | 72 +++++++++ src/test/regress/sql/vacuum_stats.sql | 45 ++++++ 17 files changed, 1118 insertions(+), 1 deletion(-) create mode 100644 src/test/modules/test_misc/t/021_heap_page_counters.pl create mode 100644 src/test/modules/test_misc/t/022_ao_hidden_tuples.pl create mode 100644 src/test/modules/test_misc/t/023_dead_pages.pl create mode 100644 src/test/modules/test_misc/t/024_ao_dead_pages.pl create mode 100644 src/test/modules/test_misc/t/027_freeze_age_vacuum_count.pl create mode 100644 src/test/modules/test_misc/t/028_vacuum_work_core.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f9ceead292f..78f47575cef 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -459,6 +459,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_vacuum, + pg_stat_vacuum_tables, + pg_stat_vacuum_indexes + One row per database, table or index, showing vacuum work counters. + See . + + pg_stat_database_conflictspg_stat_database_conflicts @@ -4697,6 +4705,111 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + VACUUM Work Statistics + pg_stat_vacuum_tables + pg_stat_vacuum_indexes + + pg_stat_vacuum_tables and + pg_stat_vacuum_indexes report work performed by + VACUUM, including autovacuum. Collection uses + the built-in cumulative statistics system and + ; no extension or preload library is + required. Heap VACUUM FULL rewrites the relation and does + not update these work counters; AO VACUUM FULL uses the + AO compaction phases and reports their work. + Each view identifies the relation by relid or + indexrelid, its schema, + relname or indexrelname, + and dbname. Table statistics include materialized + views and TOAST relations. Relations without statistics have zero counters. + + + VACUUM Work Counters + ColumnsMeaning + + tuples_deleted + Cumulative tuples removed from a table, or entries removed from an index. + Index reports add only the work of each bulkdelete or cleanup call. + + dead_tuples + Table dead tuples observed after the latest completed vacuum, including + missed removable tuples for heap and hidden rows for AO. This is a + snapshot; skipped pages may limit the heap observation. + + recently_dead_tuples, + missed_dead_tuples + Cumulative observations of heap tuples not yet removable, and removable + tuples missed because a cleanup lock could not be obtained. For AO, + recently dead tuples accumulate hidden rows left after each vacuum; + missed tuples remain zero. Repeated observations count again. + + dead_pages, + missed_dead_pages + Cumulative observations of heap pages with not-yet-removable tuples, + and pages with missed removable tuples. For indexes, dead pages are + deleted pages not yet reusable, observed once after final cleanup. + AO tables have no heap dead-page counters. + + pages_scanned, + pages_removed, + pages_deleted + Cumulative table pages scanned and removed by vacuum; AO uses + heap-equivalent pages, rounded up per scanned segment or per completed + vacuum's released space. The index-only pages_deleted counter counts + newly deleted pages, excluding already empty or deleted pages. + + tuples_frozen, + pages_frozen, + pages_all_visible + Cumulative heap tuples frozen, pages on which tuples were frozen, and + pages whose all-visible VM bit was newly set. Rescanning unchanged + pages adds no work. These counters are zero for AO. + + freeze_age_vacuum_count + Heap vacuums made aggressive by the XID or MultiXact freeze table age, + including VACUUM FREEZE. Forcing an aggressive scan + with DISABLE_PAGE_SKIPPING alone does not count. + + total_file_segs, + compacted_segments, + tuples_moved + AO segment metadata entries at the last vacuum, cumulative segments + actually compacted, and cumulative live tuples moved. Skipped candidates + add no compaction work. Heap tables report zero for these columns. + + + +
+ + All work columns have type bigint. The relation and database + getters are pg_stat_get_vacuum_stats(oid) and + pg_stat_get_vacuum_database_stats(oid). + pg_stat_vacuum additionally exposes cumulative table + work per database. It excludes index reports to avoid counting table work + twice, and excludes the dead_tuples and + total_file_segs snapshots. Shared relations use + the database entry with OID zero. + + + The gp_stat_vacuum_tables and + gp_stat_vacuum_indexes views expose individual + instances. Their _summary views sum distributed table + work and normalize replicated table work by the segment count. System + relations use coordinator statistics. gp_stat_vacuum_summary + sums database work across instances. + + + Core counters survive clean shutdown and restart, use the ordinary + statistics snapshot rules, and are removed when their relation or database + is dropped. pg_stat_reset_single_table_counters(oid) + resets a relation's counters; pg_stat_reset() resets + the current database statistics. These resets are independent of the + compatibility counters and resource statistics in + ext_vacuum_statistics. + + + <structname>pg_stat_database_conflicts</structname> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 24b7fc1d33b..ca40335c5f8 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1205,6 +1205,62 @@ CREATE VIEW pg_stat_database AS UNION ALL SELECT oid, datname FROM pg_database ) D; +CREATE VIEW pg_stat_vacuum AS + SELECT + D.oid AS datid, + D.datname AS datname, + pg_stat_get_db_stat_reset_time(D.oid) AS stats_reset, + V.tuples_deleted, + V.dead_pages, + V.pages_frozen, + V.pages_all_visible, + V.freeze_age_vacuum_count, + V.tuples_frozen, + V.recently_dead_tuples, + V.missed_dead_tuples, + V.pages_scanned, + V.pages_removed, + V.missed_dead_pages, + V.compacted_segments, + V.tuples_moved + FROM ( + SELECT 0 AS oid, NULL::name AS datname + UNION ALL + SELECT oid, datname FROM pg_database + ) D, LATERAL pg_stat_get_vacuum_database_stats(D.oid) V; + +CREATE VIEW pg_stat_vacuum_tables AS + SELECT C.oid AS relid, N.nspname AS schema, C.relname, + current_database() AS dbname, + V.tuples_deleted, + V.dead_tuples, + V.dead_pages, + V.pages_frozen, + V.pages_all_visible, + V.freeze_age_vacuum_count, + V.tuples_frozen, + V.recently_dead_tuples, + V.missed_dead_tuples, + V.pages_scanned, + V.pages_removed, + V.missed_dead_pages, + V.total_file_segs, + V.compacted_segments, + V.tuples_moved + FROM pg_class C JOIN pg_namespace N ON N.oid = C.relnamespace, + LATERAL pg_stat_get_vacuum_stats(C.oid) V + WHERE C.relkind IN ('r', 't', 'm'); + +CREATE VIEW pg_stat_vacuum_indexes AS + SELECT C.oid AS indexrelid, N.nspname AS schema, C.relname AS indexrelname, + current_database() AS dbname, + V.tuples_deleted, + V.pages_deleted, + V.dead_pages + FROM pg_class C JOIN pg_namespace N ON N.oid = C.relnamespace, + LATERAL pg_stat_get_vacuum_stats(C.oid) V + WHERE C.relkind IN ('i'); + CREATE VIEW pg_resqueue_status AS SELECT q.rsqname, diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in index 41e4a2468e5..ec83417757c 100644 --- a/src/backend/catalog/system_views_gp.in +++ b/src/backend/catalog/system_views_gp.in @@ -10,6 +10,9 @@ pg_replication_slots pg_settings pg_stat_activity pg_stat_all_indexes +pg_stat_vacuum +pg_stat_vacuum_tables +pg_stat_vacuum_indexes pg_stat_all_tables pg_stat_archiver pg_stat_bgwriter diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql index 62bde4e3d96..d67a60bc190 100644 --- a/src/backend/catalog/system_views_gp_summary.sql +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -94,6 +94,30 @@ GROUP BY sdb.datname; +CREATE VIEW gp_stat_vacuum_summary AS +SELECT + sdb.datid, + sdb.datname, + max(sdb.stats_reset) as stats_reset, + sum(sdb.tuples_deleted)::bigint AS tuples_deleted, + sum(sdb.dead_pages)::bigint AS dead_pages, + sum(sdb.pages_frozen)::bigint AS pages_frozen, + sum(sdb.pages_all_visible)::bigint AS pages_all_visible, + sum(sdb.freeze_age_vacuum_count)::bigint AS freeze_age_vacuum_count, + sum(sdb.tuples_frozen)::bigint AS tuples_frozen, + sum(sdb.recently_dead_tuples)::bigint AS recently_dead_tuples, + sum(sdb.missed_dead_tuples)::bigint AS missed_dead_tuples, + sum(sdb.pages_scanned)::bigint AS pages_scanned, + sum(sdb.pages_removed)::bigint AS pages_removed, + sum(sdb.missed_dead_pages)::bigint AS missed_dead_pages, + sum(sdb.compacted_segments)::bigint AS compacted_segments, + sum(sdb.tuples_moved)::bigint AS tuples_moved +FROM + gp_stat_vacuum sdb +GROUP BY + sdb.datid, + sdb.datname; + -- Gather data from segments on user tables, and use data on coordinator on system tables. CREATE VIEW gp_stat_all_tables_summary AS SELECT @@ -520,3 +544,92 @@ SELECT max(stats_reset) as stats_reset FROM gp_stat_io GROUP BY backend_type, object, context; + +CREATE VIEW gp_stat_vacuum_tables_summary AS +SELECT + s.relid, + s.schema, + s.relname, + s.dbname, + (sum(s.tuples_deleted) / s.divisor)::bigint AS tuples_deleted, + (sum(s.pages_scanned) / s.divisor)::bigint AS pages_scanned, + (sum(s.pages_removed) / s.divisor)::bigint AS pages_removed, + (sum(s.tuples_frozen) / s.divisor)::bigint AS tuples_frozen, + (sum(s.recently_dead_tuples) / s.divisor)::bigint AS recently_dead_tuples, + (sum(s.missed_dead_pages) / s.divisor)::bigint AS missed_dead_pages, + (sum(s.missed_dead_tuples) / s.divisor)::bigint AS missed_dead_tuples, + (sum(s.pages_frozen) / s.divisor)::bigint AS pages_frozen, + (sum(s.pages_all_visible) / s.divisor)::bigint AS pages_all_visible, + (sum(s.freeze_age_vacuum_count) / s.divisor)::bigint AS freeze_age_vacuum_count, + (sum(s.total_file_segs) / s.divisor)::bigint AS total_file_segs, + (sum(s.compacted_segments) / s.divisor)::bigint AS compacted_segments, + (sum(s.tuples_moved) / s.divisor)::bigint AS tuples_moved, + (sum(s.dead_tuples) / s.divisor)::bigint AS dead_tuples, + (sum(s.dead_pages) / s.divisor)::bigint AS dead_pages +FROM ( + SELECT v.*, + CASE WHEN d.policytype = 'r' THEN d.numsegments ELSE 1 END AS divisor + FROM gp_dist_random('pg_stat_vacuum_tables') v + LEFT JOIN gp_distribution_policy d ON d.localoid = v.relid + WHERE v.relid >= 16384 +) s +GROUP BY s.relid, s.schema, s.relname, s.dbname, s.divisor +UNION ALL +SELECT + relid, + schema, + relname, + dbname, + tuples_deleted, + pages_scanned, + pages_removed, + tuples_frozen, + recently_dead_tuples, + missed_dead_pages, + missed_dead_tuples, + pages_frozen, + pages_all_visible, + freeze_age_vacuum_count, + total_file_segs, + compacted_segments, + tuples_moved, + dead_tuples, + dead_pages +FROM pg_stat_vacuum_tables +WHERE relid < 16384; + +COMMENT ON VIEW gp_stat_vacuum_tables_summary IS + 'Vacuum work statistics per table, summed over the cluster'; + +CREATE VIEW gp_stat_vacuum_indexes_summary AS +SELECT + s.indexrelid, + s.schema, + s.indexrelname, + s.dbname, + (sum(s.tuples_deleted) / s.divisor)::bigint AS tuples_deleted, + (sum(s.pages_deleted) / s.divisor)::bigint AS pages_deleted, + (sum(s.dead_pages) / s.divisor)::bigint AS dead_pages +FROM ( + SELECT v.*, + CASE WHEN d.policytype = 'r' THEN d.numsegments ELSE 1 END AS divisor + FROM gp_dist_random('pg_stat_vacuum_indexes') v + JOIN pg_index i ON i.indexrelid = v.indexrelid + LEFT JOIN gp_distribution_policy d ON d.localoid = i.indrelid + WHERE v.indexrelid >= 16384 +) s +GROUP BY s.indexrelid, s.schema, s.indexrelname, s.dbname, s.divisor +UNION ALL +SELECT + indexrelid, + schema, + indexrelname, + dbname, + tuples_deleted, + pages_deleted, + dead_pages +FROM pg_stat_vacuum_indexes +WHERE indexrelid < 16384; + +COMMENT ON VIEW gp_stat_vacuum_indexes_summary IS + 'Vacuum work statistics per index, summed over the cluster'; diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 61c69b3ca4e..5fa97736091 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -2211,3 +2211,57 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objoid)); } + +/* Native VACUUM work statistics, independent of ext_vacuum_statistics. */ +static Datum +pgstat_vacuum_stats_tuple(FunctionCallInfo fcinfo, const PgStat_VacuumStats *stats, + bool database) +{ + TupleDesc tupdesc; + Datum values[16]; + bool nulls[16] = {false}; + int i = 0; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + values[i++] = Int64GetDatum(stats ? stats->tuples_deleted : 0); + if (!database) + values[i++] = Int64GetDatum(stats ? stats->dead_tuples : 0); + if (!database) + values[i++] = Int64GetDatum(stats ? stats->pages_deleted : 0); + values[i++] = Int64GetDatum(stats ? stats->dead_pages : 0); + values[i++] = Int64GetDatum(stats ? stats->pages_frozen : 0); + values[i++] = Int64GetDatum(stats ? stats->pages_all_visible : 0); + values[i++] = Int64GetDatum(stats ? stats->freeze_age_vacuum_count : 0); + values[i++] = Int64GetDatum(stats ? stats->tuples_frozen : 0); + values[i++] = Int64GetDatum(stats ? stats->recently_dead_tuples : 0); + values[i++] = Int64GetDatum(stats ? stats->missed_dead_tuples : 0); + values[i++] = Int64GetDatum(stats ? stats->pages_scanned : 0); + values[i++] = Int64GetDatum(stats ? stats->pages_removed : 0); + values[i++] = Int64GetDatum(stats ? stats->missed_dead_pages : 0); + if (!database) + values[i++] = Int64GetDatum(stats ? stats->total_file_segs : 0); + values[i++] = Int64GetDatum(stats ? stats->compacted_segments : 0); + values[i++] = Int64GetDatum(stats ? stats->tuples_moved : 0); + Assert(i == tupdesc->natts); + return HeapTupleGetDatum(heap_form_tuple(BlessTupleDesc(tupdesc), values, nulls)); +} + +Datum +pg_stat_get_vacuum_stats(PG_FUNCTION_ARGS) +{ + PgStat_StatTabEntry *entry = pgstat_fetch_stat_tabentry(PG_GETARG_OID(0)); + + return pgstat_vacuum_stats_tuple(fcinfo, entry ? &entry->vacuum_stats : NULL, + false); +} + +Datum +pg_stat_get_vacuum_database_stats(PG_FUNCTION_ARGS) +{ + PgStat_StatDBEntry *entry = pgstat_fetch_stat_dbentry(PG_GETARG_OID(0)); + + return pgstat_vacuum_stats_tuple(fcinfo, entry ? &entry->vacuum_stats : NULL, + true); +} diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c9c2fd83275..cf6355bcc46 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302608261 +#define CATALOG_VERSION_NO 302610082 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 509c2e3be2c..9c170fc383c 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -13019,3 +13019,19 @@ proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o}', proargnames => '{queueid,queueid,queries_submitted,queries_admitted,queries_rejected,queries_completed,elapsed_wait_secs,max_wait_secs,elapsed_exec_secs,max_exec_secs,total_cost,total_memory_kb,stat_reset_timestamp,have_stats}', prosrc => 'pg_stat_get_resqueue_stats' }, + +{ oid => '9220', descr => 'statistics: vacuum work counters', + proname => 'pg_stat_get_vacuum_stats', provolatile => 's', proparallel => 'r', + prorettype => 'record', proargtypes => 'oid', + proallargtypes => '{oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{relid,tuples_deleted,dead_tuples,pages_deleted,dead_pages,pages_frozen,pages_all_visible,freeze_age_vacuum_count,tuples_frozen,recently_dead_tuples,missed_dead_tuples,pages_scanned,pages_removed,missed_dead_pages,total_file_segs,compacted_segments,tuples_moved}', + prosrc => 'pg_stat_get_vacuum_stats' }, + +{ oid => '9221', descr => 'statistics: vacuum work counters', + proname => 'pg_stat_get_vacuum_database_stats', provolatile => 's', proparallel => 'r', + prorettype => 'record', proargtypes => 'oid', + proallargtypes => '{oid,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{datid,tuples_deleted,dead_pages,pages_frozen,pages_all_visible,freeze_age_vacuum_count,tuples_frozen,recently_dead_tuples,missed_dead_tuples,pages_scanned,pages_removed,missed_dead_pages,compacted_segments,tuples_moved}', + prosrc => 'pg_stat_get_vacuum_database_stats' }, diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..7543a0c5f55 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,12 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/021_heap_page_counters.pl', + 't/022_ao_hidden_tuples.pl', + 't/023_dead_pages.pl', + 't/024_ao_dead_pages.pl', + 't/027_freeze_age_vacuum_count.pl', + 't/028_vacuum_work_core.pl', ], }, } diff --git a/src/test/modules/test_misc/t/021_heap_page_counters.pl b/src/test/modules/test_misc/t/021_heap_page_counters.pl new file mode 100644 index 00000000000..89ebf852963 --- /dev/null +++ b/src/test/modules/test_misc/t/021_heap_page_counters.pl @@ -0,0 +1,92 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Count work on heap pages, rather than the current visibility-map state. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_heap_pages'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +}); +$node->start; +$node->safe_psql('postgres', q{ +CREATE TABLE heap_pages (id int PRIMARY KEY, padding text); +INSERT INTO heap_pages SELECT g, repeat('x', 100) FROM generate_series(1, 10000) g; +}); +my $pages = $node->safe_psql('postgres', q{ +SELECT count(DISTINCT split_part(ctid::text, ',', 1)) FROM heap_pages}); +my $counts_sql = q{ +SELECT pages_frozen, pages_all_visible + FROM pg_stat_vacuum_tables WHERE relname = 'heap_pages'}; + +# No pruning or freezing is needed, but all populated pages become visible. +$node->safe_psql('postgres', 'VACUUM heap_pages'); +is($node->safe_psql('postgres', $counts_sql), "0|$pages", + 'ordinary vacuum marks all populated pages visible without freezing'); + +# Upgrading all-visible pages to all-frozen must not count them visible again. +$node->safe_psql('postgres', 'VACUUM (FREEZE) heap_pages'); +is($node->safe_psql('postgres', $counts_sql), "$pages|$pages", + 'freeze counts each affected page without recounting its all-visible bit'); +is($node->safe_psql('postgres', q{ +SELECT tuples_frozen FROM pg_stat_vacuum_tables + WHERE relname = 'heap_pages'}), '10000', + 'tuple and page freezing counters have distinct units'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) heap_pages'); +is($node->safe_psql('postgres', $counts_sql), "$pages|$pages", + 'rescanning already frozen and visible pages adds no work'); + +# Changing the indexed key requires index cleanup and a second heap pass. +# The old and new tuple can occupy either one or two distinct heap pages. +$node->safe_psql('postgres', q{ +CREATE TABLE changed_pages AS + SELECT split_part(ctid::text, ',', 1) AS blk FROM heap_pages WHERE id = 1; +UPDATE heap_pages SET id = 10001 WHERE id = 1; +INSERT INTO changed_pages + SELECT split_part(ctid::text, ',', 1) FROM heap_pages WHERE id = 10001; +}); +my $changed = $node->safe_psql('postgres', 'SELECT count(DISTINCT blk) FROM changed_pages'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, INDEX_CLEANUP ON) heap_pages'); +my $expected = ($pages + 1) . '|' . ($pages + $changed); +is($node->safe_psql('postgres', $counts_sql), $expected, + 'index cleanup restores changed visibility bits and freezes the new tuple'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) heap_pages'); +is($node->safe_psql('postgres', $counts_sql), $expected, + 'another full scan does not count second-pass visibility work again'); + +# Empty pages can become all-visible without any tuples being frozen. +$node->safe_psql('postgres', q{ +CREATE TABLE empty_pages (id int); +INSERT INTO empty_pages SELECT generate_series(1, 10000); +}); +my $empty_pages = $node->safe_psql('postgres', q{ +SELECT count(DISTINCT split_part(ctid::text, ',', 1)) FROM empty_pages}); +$node->safe_psql('postgres', 'DELETE FROM empty_pages'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, TRUNCATE OFF) empty_pages'); +my $empty_sql = q{ +SELECT pages_frozen, pages_all_visible + FROM pg_stat_vacuum_tables WHERE relname = 'empty_pages'}; +is($node->safe_psql('postgres', $empty_sql), "0|$empty_pages", + 'empty pages become visible without adding frozen pages'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, TRUNCATE OFF, DISABLE_PAGE_SKIPPING) empty_pages'); +is($node->safe_psql('postgres', $empty_sql), "0|$empty_pages", + 'rescanning empty pages does not count existing visibility marks'); + +$node->restart; +is($node->safe_psql('postgres', $counts_sql), $expected, + 'heap page counters survive a clean restart'); + +$node->safe_psql('postgres', q{ +SELECT pg_stat_reset_single_table_counters('heap_pages'::regclass)}); +is($node->safe_psql('postgres', $counts_sql), '0|0', + 'relation reset clears both heap page counters'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) heap_pages'); +is($node->safe_psql('postgres', $counts_sql), '0|0', + 'post-reset vacuum does not report existing frozen or visible pages as work'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_misc/t/022_ao_hidden_tuples.pl b/src/test/modules/test_misc/t/022_ao_hidden_tuples.pl new file mode 100644 index 00000000000..80ee4ba4f51 --- /dev/null +++ b/src/test/modules/test_misc/t/022_ao_hidden_tuples.pl @@ -0,0 +1,104 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AO hidden rows have both a cumulative observation count and a latest sample. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_ao_hidden'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +}); +$node->start; +$node->safe_psql('postgres', q{ +CREATE TABLE hidden_heap (id int); +INSERT INTO hidden_heap VALUES (1); +VACUUM hidden_heap; +}); +is($node->safe_psql('postgres', q{ +SELECT dead_tuples FROM pg_stat_vacuum_tables + WHERE relname = 'hidden_heap'}), '0', 'AO hidden-row snapshot stays zero for heap'); + +for my $orientation ('row', 'column') +{ + my $table = "hidden_$orientation"; + my $count_sql = qq{ +SELECT recently_dead_tuples, dead_tuples FROM pg_stat_vacuum_tables + WHERE relname = '$table'}; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int) WITH (appendonly = true, orientation = $orientation); +INSERT INTO $table SELECT generate_series(1, 1000); +DELETE FROM $table WHERE id <= 100; +SET gp_appendonly_compaction_threshold = 100; +SET vacuum_freeze_table_age = 0; +VACUUM $table; +}); + is($node->safe_psql('postgres', qq{ +SELECT recently_dead_tuples = 100 AND tuples_deleted = 0 AND compacted_segments = 0 + AND dead_pages = 0 AND dead_tuples = 100 AND freeze_age_vacuum_count = 0 + FROM pg_stat_vacuum_tables WHERE relname = '$table'}), 't', + "$orientation AO counts hidden rows left below the compaction threshold"); + + $node->safe_psql('postgres', "SET gp_appendonly_compaction = off; VACUUM $table"); + is($node->safe_psql('postgres', $count_sql), '200|100', + "$orientation AO accumulates remaining rows but replaces the latest snapshot"); + + $node->safe_psql('postgres', qq{ +DELETE FROM $table WHERE id = 101; +SET gp_appendonly_compaction = off; +SET track_counts = off; +VACUUM $table; +}); + is($node->safe_psql('postgres', $count_sql), '200|100', + "$orientation AO disabled collection preserves both counts despite another hidden row"); + + $node->safe_psql('postgres', "SET gp_appendonly_compaction_threshold = 1; VACUUM $table"); + is($node->safe_psql('postgres', qq{ +SELECT recently_dead_tuples = 200 AND dead_tuples = 0 + AND tuples_deleted = 101 AND compacted_segments > 0 + FROM pg_stat_vacuum_tables WHERE relname = '$table'}), 't', + "$orientation AO samples after compaction and clears the latest hidden-row count"); + $node->safe_psql('postgres', "VACUUM $table"); + is($node->safe_psql('postgres', $count_sql), '200|0', + "$orientation AO empty hidden-row sample does not reuse an earlier count"); + + $node->safe_psql('postgres', qq{ +DELETE FROM $table WHERE id = 102; +SET gp_appendonly_compaction_threshold = 100; +VACUUM $table; +}); + is($node->safe_psql('postgres', $count_sql), '201|1', + "$orientation AO adds a fresh remaining-row sample after compaction"); +} + +$node->restart; +for my $orientation ('row', 'column') +{ + my $table = "hidden_$orientation"; + my $count_sql = qq{ +SELECT recently_dead_tuples, dead_tuples FROM pg_stat_vacuum_tables + WHERE relname = '$table'}; + is($node->safe_psql('postgres', $count_sql), '201|1', + "$orientation AO remaining-row total and snapshot survive a clean restart"); + $node->safe_psql('postgres', qq{ +SELECT pg_stat_reset_single_table_counters('$table'::regclass)}); + is($node->safe_psql('postgres', $count_sql), '0|0', + "$orientation AO relation reset clears the remaining-row total and snapshot"); + $node->safe_psql('postgres', "SET gp_appendonly_compaction_threshold = 100; VACUUM $table"); + is($node->safe_psql('postgres', $count_sql), '1|1', + "$orientation AO resumes accumulating remaining rows after reset"); + + # Unlike heap's rewrite, AO VACUUM FULL still runs compaction phases. + $node->safe_psql('postgres', "VACUUM FULL $table"); + is($node->safe_psql('postgres', qq{ +SELECT tuples_deleted = 1 AND compacted_segments > 0 AND dead_tuples = 0 + AND freeze_age_vacuum_count = 0 + FROM pg_stat_vacuum_tables WHERE relname = '$table'}), 't', + "$orientation AO VACUUM FULL reports compaction work without heap freezing"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_misc/t/023_dead_pages.pl b/src/test/modules/test_misc/t/023_dead_pages.pl new file mode 100644 index 00000000000..4931d1a000c --- /dev/null +++ b/src/test/modules/test_misc/t/023_dead_pages.pl @@ -0,0 +1,145 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_dead_pages'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +max_parallel_maintenance_workers = 2 +min_parallel_index_scan_size = 0 +}); +$node->start; +$node->safe_psql('postgres', q{ +CREATE TABLE retained_heap (id int, padding text); +INSERT INTO retained_heap SELECT g, repeat('x', 100) FROM generate_series(1, 2000) g; +}); +my $pages = $node->safe_psql('postgres', q{ +SELECT count(DISTINCT split_part(ctid::text, ',', 1)) FROM retained_heap WHERE id <= 1000}); +my $heap_sql = q{ +SELECT dead_pages FROM pg_stat_vacuum_tables WHERE relname = 'retained_heap'}; +my $snapshot = $node->background_psql('postgres'); +$snapshot->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM retained_heap'); +$node->safe_psql('postgres', 'DELETE FROM retained_heap WHERE id <= 1000; VACUUM retained_heap'); +is($node->safe_psql('postgres', $heap_sql), $pages, + 'heap counts pages containing tuples retained for an older snapshot'); +is($node->safe_psql('postgres', q{ +SELECT recently_dead_tuples = 1000 AND missed_dead_pages = 0 + FROM pg_stat_vacuum_tables WHERE relname = 'retained_heap'}), 't', + 'retained dead pages are independent of cleanup-lock misses'); +$node->safe_psql('postgres', 'VACUUM retained_heap'); +is($node->safe_psql('postgres', $heap_sql), 2 * $pages, + 'another vacuum accumulates the same retained heap pages again'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted, recently_dead_tuples FROM pg_stat_vacuum_tables + WHERE relname = 'retained_heap'}), '0|2000', + 'both vacuums accumulate retained tuples without reporting removals'); +$snapshot->query_safe('COMMIT'); +$snapshot->quit; +$node->safe_psql('postgres', 'VACUUM retained_heap'); +is($node->safe_psql('postgres', $heap_sql), 2 * $pages, + 'removing the retained tuples does not add dead pages'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted, recently_dead_tuples FROM pg_stat_vacuum_tables + WHERE relname = 'retained_heap'}), '1000|2000', + 'releasing the snapshot permits removal without adding retained tuples'); + +# A cursor holds a buffer pin, forcing the scan without a cleanup lock. +for my $recent (0, 1) +{ + my $table = "pinned_$recent"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int); +INSERT INTO $table SELECT generate_series(1, 20); +}); + $node->safe_psql('postgres', "DELETE FROM $table WHERE id = 20") unless $recent; + my $pin = $node->background_psql('postgres'); + $pin->query_safe("BEGIN ISOLATION LEVEL REPEATABLE READ; DECLARE c CURSOR FOR SELECT * FROM $table; FETCH 1 FROM c"); + $node->safe_psql('postgres', "DELETE FROM $table WHERE id = 20") if $recent; + $node->safe_psql('postgres', "VACUUM (TRUNCATE OFF) $table"); + is($node->safe_psql('postgres', qq{ +SELECT dead_pages, missed_dead_pages FROM pg_stat_vacuum_tables + WHERE relname = '$table'}), $recent ? '1|0' : '0|1', + "pinned page distinguishes recently-dead tuples from missed removable tuples ($recent)"); + $pin->query_safe('COMMIT'); + $pin->quit; +} + +# A snapshot taken after DELETE permits removal of heap tuples, but prevents +# recycling index pages deleted by the following VACUUM. Two indexes exercise +# serial reporting with PARALLEL 0 and Cloudberry's PARALLEL 2 fallback. +my $nrows = 20000 * ($node->safe_psql('postgres', 'SHOW block_size') / 8192); +my %index_totals; +for my $parallel (0, 2) +{ + my $table = "index_pages_$parallel"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int); +CREATE INDEX ${table}_a ON $table (id); +CREATE INDEX ${table}_b ON $table ((id + 1)); +INSERT INTO $table SELECT generate_series(1, $nrows); +DELETE FROM $table WHERE id <= $nrows / 2; +}); + my $hold = $node->background_psql('postgres'); + $hold->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT txid_current_snapshot()'); + my ($stdout, $stderr); + my $result = $node->psql('postgres', "VACUUM (VERBOSE, INDEX_CLEANUP ON, PARALLEL $parallel) $table", + stdout => \$stdout, stderr => \$stderr); + is($result, 0, "vacuum with PARALLEL $parallel succeeds"); + like($stderr, qr/disabling parallel option of vacuum.*cannot vacuum tables in parallel/, + 'Cloudberry reports the serial fallback for PARALLEL') if $parallel; + for my $suffix ('a', 'b') + { + my $index = "${table}_$suffix"; + my $deleted = $node->safe_psql('postgres', qq{ +SELECT pages_deleted FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}); + cmp_ok($deleted, '>', 0, "$index has newly deleted pages"); + is($node->safe_psql('postgres', qq{ +SELECT dead_pages FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}), + $deleted, "$index counts its non-reusable pages once after cleanup"); + $index_totals{$index} = 2 * $deleted; + } + $node->safe_psql('postgres', "VACUUM (INDEX_CLEANUP ON, PARALLEL $parallel) $table"); + for my $suffix ('a', 'b') + { + my $index = "${table}_$suffix"; + is($node->safe_psql('postgres', qq{ +SELECT dead_pages FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}), + $index_totals{$index}, "$index accumulates pages still waiting for reuse"); + } + $hold->query_safe('COMMIT'); + $hold->quit; + $node->safe_psql('postgres', 'SELECT txid_current()'); + $node->safe_psql('postgres', "VACUUM (INDEX_CLEANUP ON, PARALLEL $parallel) $table"); + for my $suffix ('a', 'b') + { + my $index = "${table}_$suffix"; + is($node->safe_psql('postgres', qq{ +SELECT dead_pages FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}), + $index_totals{$index}, "$index does not count pages that became reusable"); + } +} + +$node->restart; +is($node->safe_psql('postgres', $heap_sql), 2 * $pages, + 'heap dead-page counter survives a clean restart'); +for my $index (sort keys %index_totals) +{ + is($node->safe_psql('postgres', qq{ +SELECT dead_pages FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}), + $index_totals{$index}, "$index dead-page counter survives a clean restart"); +} +$node->safe_psql('postgres', 'SELECT pg_stat_reset()'); +is($node->safe_psql('postgres', qq{ +SELECT dead_pages FROM pg_stat_vacuum_tables WHERE relname = 'retained_heap'}), + '0', 'reset clears heap dead pages'); +is($node->safe_psql('postgres', q{ +SELECT bool_and(dead_pages = 0) FROM pg_stat_vacuum_indexes + WHERE indexrelname LIKE 'index_pages_%'}), 't', 'reset clears index dead pages'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_misc/t/024_ao_dead_pages.pl b/src/test/modules/test_misc/t/024_ao_dead_pages.pl new file mode 100644 index 00000000000..e5388558ba8 --- /dev/null +++ b/src/test/modules/test_misc/t/024_ao_dead_pages.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# AO pre-cleanup and post-cleanup can both scan an index. Only the final +# state contributes dead_pages; the AO table itself has no heap dead pages. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_ao_dead_pages'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +gp_appendonly_compaction_threshold = 1 +}); +$node->start; +my $nrows = 20000 * ($node->safe_psql('postgres', 'SHOW block_size') / 8192); +for my $orientation ('row', 'column') +{ + my $table = "ao_pages_$orientation"; + my $index = "${table}_idx"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int) WITH (appendonly = true, orientation = $orientation); +CREATE INDEX $index ON $table (id); +INSERT INTO $table SELECT generate_series(1, $nrows); +DELETE FROM $table WHERE id <= $nrows / 2; +}); + # This snapshot defers recycling the segment compacted by the first run. + my $old = $node->background_psql('postgres'); + $old->query_safe("BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM $table"); + $node->safe_psql('postgres', "VACUUM $table"); + is($node->safe_psql('postgres', qq{ +SELECT dead_pages = 0 AND tuples_deleted = $nrows / 2 + FROM pg_stat_vacuum_tables WHERE relname = '$table'}), 't', + "$orientation AO compaction leaves the heap-only dead-pages counter zero"); + $old->query_safe('COMMIT'); + $old->quit; + + # This later snapshot permits segment recycling, but holds back reuse + # of index pages newly deleted during the second run's pre-cleanup. + my $hold = $node->background_psql('postgres'); + $hold->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT txid_current_snapshot()'); + $node->safe_psql('postgres', "VACUUM $table"); + my $deleted = $node->safe_psql('postgres', qq{ +SELECT pages_deleted FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}); + cmp_ok($deleted, '>', 0, "$orientation AO recycling deletes index pages"); + my $count_sql = qq{ +SELECT dead_pages FROM pg_stat_vacuum_indexes WHERE indexrelname = '$index'}; + is($node->safe_psql('postgres', $count_sql), $deleted, + "$orientation AO counts non-reusable index pages once across both cleanup phases"); + $node->safe_psql('postgres', "VACUUM $table"); + is($node->safe_psql('postgres', $count_sql), 2 * $deleted, + "$orientation AO accumulates index pages still waiting for reuse"); + $hold->query_safe('COMMIT'); + $hold->quit; + $node->safe_psql('postgres', 'SELECT txid_current()'); + $node->safe_psql('postgres', "VACUUM $table"); + # Cloudberry's BTPageIsRecyclable deliberately permits recycling only + # for heap relations. AO index tombstones remain non-reusable even after + # the snapshot ends, and each subsequent cleanup observes them once. + is($node->safe_psql('postgres', $count_sql), 3 * $deleted, + "$orientation AO retains non-reusable index pages after the snapshot ends"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_misc/t/027_freeze_age_vacuum_count.pl b/src/test/modules/test_misc/t/027_freeze_age_vacuum_count.pl new file mode 100644 index 00000000000..9d215c3989b --- /dev/null +++ b/src/test/modules/test_misc/t/027_freeze_age_vacuum_count.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_freeze_age'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +}); +$node->start; +$node->safe_psql('postgres', q{ +CREATE TABLE freeze_age_heap (id int); +INSERT INTO freeze_age_heap SELECT generate_series(1, 100); +VACUUM freeze_age_heap; +}); +my $count_sql = q{ +SELECT freeze_age_vacuum_count FROM pg_stat_vacuum_tables + WHERE relname = 'freeze_age_heap'}; +is($node->safe_psql('postgres', $count_sql), '0', + 'ordinary vacuum of a young table does not count as freeze-age driven'); + +$node->safe_psql('postgres', 'VACUUM (DISABLE_PAGE_SKIPPING) freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '0', + 'DISABLE_PAGE_SKIPPING alone does not count despite an aggressive scan'); + +$node->safe_psql('postgres', q{ +SET vacuum_freeze_table_age = 0; +VACUUM freeze_age_heap; +}); +is($node->safe_psql('postgres', $count_sql), '1', + 'XID freeze age counts a manual vacuum'); + +# safe_psql opens a new connection, restoring the default XID freeze age. +$node->safe_psql('postgres', q{ +SET vacuum_multixact_freeze_table_age = 0; +VACUUM freeze_age_heap; +}); +is($node->safe_psql('postgres', $count_sql), '2', + 'MultiXact freeze age independently counts a vacuum'); + +$node->safe_psql('postgres', 'VACUUM (FREEZE) freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '3', + 'VACUUM FREEZE counts once even when both age conditions hold'); +$node->safe_psql('postgres', 'VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '4', + 'explicit page scanning does not double count a freeze-age vacuum'); +$node->safe_psql('postgres', 'VACUUM freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '4', + 'a later ordinary vacuum does not reuse the preceding classification'); +$node->safe_psql('postgres', 'VACUUM (FULL, FREEZE) freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '4', + 'VACUUM FULL does not enter the lazy-vacuum counter'); + +$node->safe_psql('postgres', q{ +SET track_counts = off; +VACUUM (FREEZE) freeze_age_heap; +}); +is($node->safe_psql('postgres', $count_sql), '4', + 'disabled collection does not count a freeze-age vacuum'); +$node->restart; +is($node->safe_psql('postgres', $count_sql), '4', + 'freeze-age vacuum count survives a clean restart'); + +$node->safe_psql('postgres', q{ +SELECT pg_stat_reset_single_table_counters('freeze_age_heap'::regclass)}); +is($node->safe_psql('postgres', $count_sql), '0', + 'relation reset clears the freeze-age vacuum count'); +$node->safe_psql('postgres', 'VACUUM (FREEZE) freeze_age_heap'); +is($node->safe_psql('postgres', $count_sql), '1', + 'a freeze-age vacuum after reset starts a new total'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_misc/t/028_vacuum_work_core.pl b/src/test/modules/test_misc/t/028_vacuum_work_core.pl new file mode 100644 index 00000000000..6e776e3657b --- /dev/null +++ b/src/test/modules/test_misc/t/028_vacuum_work_core.pl @@ -0,0 +1,97 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Ordinary statistics storage must collect VACUUM work without an extension. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacuum_work_core'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->start; +is($node->safe_psql('postgres', 'SHOW shared_preload_libraries'), '', + 'no statistics extension is preloaded'); +$node->safe_psql('postgres', q{ +CREATE TABLE core_work (id int PRIMARY KEY); +INSERT INTO core_work SELECT generate_series(1, 2000); +DELETE FROM core_work WHERE id <= 1000; +SELECT pg_stat_force_next_flush(); +}); +my $db_sql = q{SELECT tuples_deleted FROM pg_stat_vacuum WHERE datname = current_database()}; +my $before = $node->safe_psql('postgres', $db_sql); +$node->safe_psql('postgres', q{ +VACUUM (INDEX_CLEANUP ON) core_work; +SELECT pg_stat_force_next_flush(); +}); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted FROM pg_stat_vacuum_tables WHERE relname = 'core_work' +}), '1000', 'core reports heap tuples removed'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted FROM pg_stat_vacuum_indexes WHERE indexrelname = 'core_work_pkey' +}), '1000', 'core separately reports index entries removed'); +$node->poll_query_until('postgres', "SELECT tuples_deleted = " . ($before + 1000) . " FROM pg_stat_vacuum WHERE datname = current_database()") + or die 'database vacuum work did not flush'; +is($node->safe_psql('postgres', $db_sql), $before + 1000, + 'database counts table work once, excluding index entries'); +my $saved = $node->safe_psql('postgres', q{ +SELECT pg_stat_get_vacuum_stats('core_work'::regclass)::text +}); +$node->restart; +is($node->safe_psql('postgres', q{ +SELECT pg_stat_get_vacuum_stats('core_work'::regclass)::text +}), $saved, 'ordinary statistics persistence preserves vacuum work'); +$node->safe_psql('postgres', q{ +SELECT pg_stat_reset_single_table_counters('core_work'::regclass); +}); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted = 0 AND pages_scanned = 0 AND pages_all_visible = 0 +FROM pg_stat_vacuum_tables WHERE relname = 'core_work' +}), 't', 'ordinary relation reset clears core vacuum counters'); +is($node->safe_psql('postgres', $db_sql), $before + 1000, + 'relation reset leaves database totals intact'); +$node->safe_psql('postgres', 'SELECT pg_stat_reset()'); +is($node->safe_psql('postgres', $db_sql), '0', + 'ordinary database reset clears database vacuum work'); + +# Dropping a relation discards its native work counters transactionally. +my $relid = $node->safe_psql('postgres', "SELECT 'core_work'::regclass::oid"); +$node->safe_psql('postgres', 'VACUUM (FREEZE) core_work'); +$node->safe_psql('postgres', 'BEGIN; DROP TABLE core_work; ROLLBACK;'); +is($node->safe_psql('postgres', q{ +SELECT freeze_age_vacuum_count FROM pg_stat_vacuum_tables WHERE relname = 'core_work'}), + '1', 'rolled-back DROP preserves core counters'); +$node->safe_psql('postgres', 'DROP TABLE core_work'); +is($node->safe_psql('postgres', "SELECT tuples_deleted = 0 AND freeze_age_vacuum_count = 0 FROM pg_stat_get_vacuum_stats($relid)"), + 't', 'committed DROP removes core vacuum statistics'); + +# Simulate the zero page that can remain after a crash during index extension. +# It is reusable before vacuum starts, so discovering it is not deletion work. +$node->safe_psql('postgres', q{ +CREATE TABLE spg_pages (p point); +CREATE INDEX spg_pages_idx ON spg_pages USING spgist (p); +INSERT INTO spg_pages SELECT point(g, g) FROM generate_series(1, 10) g; +}); +my $index_path = $node->safe_psql('postgres', q{ +SELECT pg_relation_filepath('spg_pages_idx')}); +my $block_size = $node->safe_psql('postgres', 'SHOW block_size'); +$node->stop; +append_to_file($node->data_dir . '/' . $index_path, "\0" x $block_size); +$node->start; +$node->safe_psql('postgres', 'VACUUM (INDEX_CLEANUP ON) spg_pages'); +my $spg_sql = q{ +SELECT pages_deleted FROM pg_stat_vacuum_indexes + WHERE indexrelname = 'spg_pages_idx'}; +my $deleted = $node->safe_psql('postgres', $spg_sql); +is($deleted, '0', 'SP-GiST does not count a pre-existing zero page as newly deleted'); +$node->safe_psql('postgres', 'VACUUM (INDEX_CLEANUP ON, DISABLE_PAGE_SKIPPING) spg_pages'); +is($node->safe_psql('postgres', $spg_sql), $deleted, + 'SP-GiST rescan does not count previously emptied pages again'); +is($node->safe_psql('postgres', q{ +SELECT dead_pages FROM pg_stat_vacuum_indexes + WHERE indexrelname = 'spg_pages_idx'}), '0', + 'reusable SP-GiST pages are not counted as dead pages'); + +$node->stop; +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 614ffea1784..cdcb636858a 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2219,6 +2219,63 @@ pg_stat_user_tables| SELECT relid, autoanalyze_count FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum| SELECT d.oid AS datid, + d.datname, + pg_stat_get_db_stat_reset_time(d.oid) AS stats_reset, + v.tuples_deleted, + v.dead_pages, + v.pages_frozen, + v.pages_all_visible, + v.freeze_age_vacuum_count, + v.tuples_frozen, + v.recently_dead_tuples, + v.missed_dead_tuples, + v.pages_scanned, + v.pages_removed, + v.missed_dead_pages, + v.compacted_segments, + v.tuples_moved + FROM ( SELECT 0 AS oid, + NULL::name AS datname + UNION ALL + SELECT pg_database.oid, + pg_database.datname + FROM pg_database) d, + LATERAL pg_stat_get_vacuum_database_stats(d.oid) v(tuples_deleted, dead_pages, pages_frozen, pages_all_visible, freeze_age_vacuum_count, tuples_frozen, recently_dead_tuples, missed_dead_tuples, pages_scanned, pages_removed, missed_dead_pages, compacted_segments, tuples_moved); +pg_stat_vacuum_indexes| SELECT c.oid AS indexrelid, + n.nspname AS schema, + c.relname AS indexrelname, + current_database() AS dbname, + v.tuples_deleted, + v.pages_deleted, + v.dead_pages + FROM (pg_class c + JOIN pg_namespace n ON ((n.oid = c.relnamespace))), + LATERAL pg_stat_get_vacuum_stats(c.oid) v(tuples_deleted, dead_tuples, pages_deleted, dead_pages, pages_frozen, pages_all_visible, freeze_age_vacuum_count, tuples_frozen, recently_dead_tuples, missed_dead_tuples, pages_scanned, pages_removed, missed_dead_pages, total_file_segs, compacted_segments, tuples_moved) + WHERE (c.relkind = 'i'::"char"); +pg_stat_vacuum_tables| SELECT c.oid AS relid, + n.nspname AS schema, + c.relname, + current_database() AS dbname, + v.tuples_deleted, + v.dead_tuples, + v.dead_pages, + v.pages_frozen, + v.pages_all_visible, + v.freeze_age_vacuum_count, + v.tuples_frozen, + v.recently_dead_tuples, + v.missed_dead_tuples, + v.pages_scanned, + v.pages_removed, + v.missed_dead_pages, + v.total_file_segs, + v.compacted_segments, + v.tuples_moved + FROM (pg_class c + JOIN pg_namespace n ON ((n.oid = c.relnamespace))), + LATERAL pg_stat_get_vacuum_stats(c.oid) v(tuples_deleted, dead_tuples, pages_deleted, dead_pages, pages_frozen, pages_all_visible, freeze_age_vacuum_count, tuples_frozen, recently_dead_tuples, missed_dead_tuples, pages_scanned, pages_removed, missed_dead_pages, total_file_segs, compacted_segments, tuples_moved) + WHERE (c.relkind = ANY (ARRAY['r'::"char", 't'::"char", 'm'::"char"])); pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index 573f7527fde..fe323292d8b 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -29,3 +29,75 @@ AND relallvisible = (1 row) DROP TABLE vacstat_test + +-- Native work counters and cluster summaries need no statistics extension. +CREATE TABLE work_stats_dist (a int PRIMARY KEY) DISTRIBUTED BY (a); +CREATE TABLE work_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO work_stats_dist SELECT generate_series(1, 2000); +ANALYZE work_stats_dist; +INSERT INTO work_stats_repl SELECT generate_series(1, 2000); +ANALYZE work_stats_repl; +VACUUM (FREEZE) work_stats_dist, work_stats_repl; +SELECT relname, freeze_age_vacuum_count = + CASE WHEN relname = 'work_stats_repl' THEN 1 + ELSE (SELECT numsegments FROM gp_distribution_policy + WHERE localoid = 'work_stats_dist'::regclass) END AS freeze_count_matches, + pages_frozen > 0 AND pages_all_visible > 0 AS page_work_recorded + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_dist', 'work_stats_repl') ORDER BY relname; + relname | freeze_count_matches | page_work_recorded +-----------------+----------------------+-------------------- + work_stats_dist | t | t + work_stats_repl | t | t +(2 rows) + +SELECT count(*) = 2 * (SELECT numsegments FROM gp_distribution_policy + WHERE localoid = 'work_stats_dist'::regclass) + AND bool_and(freeze_age_vacuum_count = 1) AS per_segment_counts + FROM gp_stat_vacuum_tables + WHERE relname IN ('work_stats_dist', 'work_stats_repl') AND gp_segment_id >= 0; + per_segment_counts +-------------------- + t +(1 row) + +DELETE FROM work_stats_dist WHERE a <= 100; +DELETE FROM work_stats_repl WHERE a <= 100; +VACUUM work_stats_dist, work_stats_repl; +SELECT relname, tuples_deleted = 100 AS deleted_matches + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_dist', 'work_stats_repl') ORDER BY relname; + relname | deleted_matches +-----------------+----------------- + work_stats_dist | t + work_stats_repl | t +(2 rows) + +SELECT tuples_deleted = 100 AS index_work_matches + FROM gp_stat_vacuum_indexes_summary WHERE indexrelname = 'work_stats_dist_pkey'; + index_work_matches +-------------------- + t +(1 row) + +DROP TABLE work_stats_dist, work_stats_repl; +CREATE TABLE work_stats_ao (a int) WITH (appendonly = true) DISTRIBUTED BY (a); +CREATE TABLE work_stats_aoco (a int) WITH (appendonly = true, orientation = column) DISTRIBUTED BY (a); +INSERT INTO work_stats_ao SELECT generate_series(1, 2000); +ANALYZE work_stats_ao; +INSERT INTO work_stats_aoco SELECT generate_series(1, 2000); +ANALYZE work_stats_aoco; +DELETE FROM work_stats_ao WHERE a <= 1000; +DELETE FROM work_stats_aoco WHERE a <= 1000; +VACUUM work_stats_ao, work_stats_aoco; +SELECT relname, tuples_deleted = 1000 AND tuples_moved = 1000 + AND compacted_segments > 0 AND pages_scanned > 0 AS compaction_matches + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_ao', 'work_stats_aoco') ORDER BY relname; + relname | compaction_matches +-----------------+-------------------- + work_stats_ao | t + work_stats_aoco | t +(2 rows) + +DROP TABLE work_stats_ao, work_stats_aoco; diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index 45a9b6466c4..e4f0298096d 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -20,3 +20,48 @@ AND relallvisible = WHERE oid='vacstat_test'::regclass); DROP TABLE vacstat_test + +-- Native work counters and cluster summaries need no statistics extension. +CREATE TABLE work_stats_dist (a int PRIMARY KEY) DISTRIBUTED BY (a); +CREATE TABLE work_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO work_stats_dist SELECT generate_series(1, 2000); +ANALYZE work_stats_dist; +INSERT INTO work_stats_repl SELECT generate_series(1, 2000); +ANALYZE work_stats_repl; +VACUUM (FREEZE) work_stats_dist, work_stats_repl; +SELECT relname, freeze_age_vacuum_count = + CASE WHEN relname = 'work_stats_repl' THEN 1 + ELSE (SELECT numsegments FROM gp_distribution_policy + WHERE localoid = 'work_stats_dist'::regclass) END AS freeze_count_matches, + pages_frozen > 0 AND pages_all_visible > 0 AS page_work_recorded + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_dist', 'work_stats_repl') ORDER BY relname; +SELECT count(*) = 2 * (SELECT numsegments FROM gp_distribution_policy + WHERE localoid = 'work_stats_dist'::regclass) + AND bool_and(freeze_age_vacuum_count = 1) AS per_segment_counts + FROM gp_stat_vacuum_tables + WHERE relname IN ('work_stats_dist', 'work_stats_repl') AND gp_segment_id >= 0; +DELETE FROM work_stats_dist WHERE a <= 100; +DELETE FROM work_stats_repl WHERE a <= 100; +VACUUM work_stats_dist, work_stats_repl; +SELECT relname, tuples_deleted = 100 AS deleted_matches + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_dist', 'work_stats_repl') ORDER BY relname; +SELECT tuples_deleted = 100 AS index_work_matches + FROM gp_stat_vacuum_indexes_summary WHERE indexrelname = 'work_stats_dist_pkey'; +DROP TABLE work_stats_dist, work_stats_repl; + +CREATE TABLE work_stats_ao (a int) WITH (appendonly = true) DISTRIBUTED BY (a); +CREATE TABLE work_stats_aoco (a int) WITH (appendonly = true, orientation = column) DISTRIBUTED BY (a); +INSERT INTO work_stats_ao SELECT generate_series(1, 2000); +ANALYZE work_stats_ao; +INSERT INTO work_stats_aoco SELECT generate_series(1, 2000); +ANALYZE work_stats_aoco; +DELETE FROM work_stats_ao WHERE a <= 1000; +DELETE FROM work_stats_aoco WHERE a <= 1000; +VACUUM work_stats_ao, work_stats_aoco; +SELECT relname, tuples_deleted = 1000 AND tuples_moved = 1000 + AND compacted_segments > 0 AND pages_scanned > 0 AS compaction_matches + FROM gp_stat_vacuum_tables_summary + WHERE relname IN ('work_stats_ao', 'work_stats_aoco') ORDER BY relname; +DROP TABLE work_stats_ao, work_stats_aoco; From 57948368b54b05bc8da07265424f991c021bf6b5 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Thu, 8 Oct 2026 00:00:00 +0300 Subject: [PATCH 42/50] Report local VACUUM resource statistics through ext_vacuum_statistics Connect heap and index reporting hooks to pluggable per-relation and per-database storage. Expose buffer and WAL usage with basic vacuum work in local SQL views. Include collection control, reset permissions, transactional DROP/OID-reuse cleanup, persistence and their tests. --- contrib/Makefile | 1 + contrib/ext_vacuum_statistics/.gitignore | 5 + contrib/ext_vacuum_statistics/Makefile | 22 + contrib/ext_vacuum_statistics/README.md | 108 ++++ .../ext_vacuum_statistics--1.0.sql | 206 +++++++ .../ext_vacuum_statistics.conf | 17 + .../ext_vacuum_statistics.control | 5 + contrib/ext_vacuum_statistics/meson.build | 36 ++ .../t/052_vacuum_extending_basic_test.pl | 368 ++++++++++++ .../t/054_vacuum_extending_gucs_test.pl | 99 ++++ .../t/056_vacuum_stats_gc.pl | 228 ++++++++ .../t/060_vacuum_stats_reset.pl | 114 ++++ .../ext_vacuum_statistics/vacuum_statistics.c | 538 ++++++++++++++++++ contrib/meson.build | 1 + doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/extvacuumstatistics.sgml | 193 +++++++ doc/src/sgml/filelist.sgml | 1 + src/backend/access/heap/vacuumlazy.c | 60 +- src/backend/commands/vacuum_ao.c | 35 +- src/backend/commands/vacuumparallel.c | 8 + src/backend/utils/activity/pgstat_relation.c | 24 + src/include/pgstat.h | 29 + 22 files changed, 2085 insertions(+), 14 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/.gitignore create mode 100644 contrib/ext_vacuum_statistics/Makefile create mode 100644 contrib/ext_vacuum_statistics/README.md create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf create mode 100644 contrib/ext_vacuum_statistics/ext_vacuum_statistics.control create mode 100644 contrib/ext_vacuum_statistics/meson.build create mode 100644 contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl create mode 100644 contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl create mode 100644 contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl create mode 100644 contrib/ext_vacuum_statistics/vacuum_statistics.c create mode 100644 doc/src/sgml/extvacuumstatistics.sgml diff --git a/contrib/Makefile b/contrib/Makefile index d41d20614f2..45d08c2d429 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -60,6 +60,7 @@ SUBDIRS = \ tsm_system_rows \ tsm_system_time \ unaccent \ + ext_vacuum_statistics \ vacuumlo # Cloudberry-specific additions (to ease merge pain). diff --git a/contrib/ext_vacuum_statistics/.gitignore b/contrib/ext_vacuum_statistics/.gitignore new file mode 100644 index 00000000000..4d88e8bdc41 --- /dev/null +++ b/contrib/ext_vacuum_statistics/.gitignore @@ -0,0 +1,5 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ +/output_iso/ diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile new file mode 100644 index 00000000000..1e53a7dbda4 --- /dev/null +++ b/contrib/ext_vacuum_statistics/Makefile @@ -0,0 +1,22 @@ +# contrib/ext_vacuum_statistics/Makefile + +EXTENSION = ext_vacuum_statistics +MODULE_big = ext_vacuum_statistics +OBJS = vacuum_statistics.o +DATA = ext_vacuum_statistics--1.0.sql +PGFILEDESC = "ext_vacuum_statistics - convenience views for extended vacuum statistics" + +TAP_TESTS = 1 + +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/ext_vacuum_statistics +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md new file mode 100644 index 00000000000..b3bbb4cb131 --- /dev/null +++ b/contrib/ext_vacuum_statistics/README.md @@ -0,0 +1,108 @@ +# ext_vacuum_statistics + +Extended vacuum statistics extension for PostgreSQL. It collects and exposes detailed per-table, per-index, and per-database vacuum statistics (buffer I/O, WAL, general, timing) via convenient views in the `ext_vacuum_statistics` schema. + +## Installation + +``` +./configure tmp_install="$(pwd)/my/inst" +make clean && make && make install +cd contrib/ext_vacuum_statistics +make && make install +``` + +It is essential that the extension is listed in `shared_preload_libraries` because it registers a vacuum hook at server startup. + +In your `postgresql.conf`: + +``` +shared_preload_libraries = 'ext_vacuum_statistics' +``` + +Restart PostgreSQL. + +In your database: + +```sql +CREATE EXTENSION ext_vacuum_statistics; +``` + +## Usage + +Query vacuum statistics via the provided views: + +```sql +-- Per-table heap vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + +-- Per-index vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +-- Per-database aggregate vacuum statistics +SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; +``` + +Example output: + +``` + relname | total_blks_read | total_blks_hit | wal_records | tuples_deleted | pages_removed +-----------+-----------------+----------------+-------------+----------------+--------------- + mytable | 120 | 340 | 15 | 500 | 10 +``` + +Reset statistics when needed: + +```sql +SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +``` + +## Configuration (GUCs) + +| GUC | Default | Description | +|-----|---------|-------------| +| `vacuum_statistics.enabled` | on | Enable extended vacuum statistics collection | + +## Memory usage + +Each tracked object (table or index) uses a fixed-size shared memory entry; the exact size depends on the platform. + +Example: a database with 1000 tables and 2000 indexes, all tracked, uses about **700 KB** on Ubuntu (3001 entries × 232 bytes). Per-database entries add one entry per tracked database. + +The entry of a table or an index is dropped when the relation is dropped (at +commit, so a rolled back `DROP` keeps it), and a new relation that gets the OID +of an old one starts from zero. The module does that with an +`object_access_hook`. + +## Recipes + +**Disable statistics collection temporarily:** + +```sql +SET vacuum_statistics.enabled = off; +``` + +## Views + +| View | Description | +|------|-------------| +| `ext_vacuum_statistics.pg_stats_vacuum_tables` | Per-table heap vacuum stats (pages scanned, tuples deleted and resource usage) | +| `ext_vacuum_statistics.pg_stats_vacuum_indexes` | Per-index vacuum stats | +| `ext_vacuum_statistics.pg_stats_vacuum_database` | Per-database aggregate vacuum stats | + +## Limitations + +- Must be loaded via `shared_preload_libraries`; it cannot be loaded on demand. +- Starting a server without the module, even once, makes it treat the whole + statistics file as corrupted and reset all cumulative statistics, the + built-in ones included. Use `vacuum_statistics.enabled = off` rather than + removing the module. + +## Native work counters + +Heap, index and AO work is also collected by the built-in statistics system +without this module. Use `pg_stat_vacuum_tables`, `pg_stat_vacuum_indexes` +and `pg_stat_vacuum` (and their `gp_stat_*` counterparts) for native counters. +They obey `track_counts` and ordinary `pg_stat_reset*` functions. The extension +retains its existing counters for compatibility and adds resource measurements +such as buffers and WAL. Its counters obey `vacuum_statistics.enabled` and +its own reset functions; resetting one collection does not reset the other. diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql new file mode 100644 index 00000000000..f43067992ba --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -0,0 +1,206 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +/*------------------------------------------------------------------------- + * + * ext_vacuum_statistics--1.0.sql + * Extended vacuum statistics via hook and custom storage + * + * This extension collects extended vacuum statistics via set_report_vacuum_hook + * and stores them in shared memory. + * + *------------------------------------------------------------------------- + */ + +\echo Use "CREATE EXTENSION ext_vacuum_statistics" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS ext_vacuum_statistics; + +COMMENT ON SCHEMA ext_vacuum_statistics IS + 'Extended vacuum statistics (heap, index, database)'; + +-- Reset functions +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_entry( + dboid oid, + relid oid +) +RETURNS void +AS 'MODULE_PATHNAME', 'extvac_reset_entry' +LANGUAGE C STRICT VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) +RETURNS void +AS 'MODULE_PATHNAME', 'extvac_reset_db_entry' +LANGUAGE C STRICT VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() +RETURNS void +AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' +LANGUAGE C STRICT VOLATILE PARALLEL UNSAFE; + +-- Reset privileges can be delegated explicitly, as for pg_stat_reset(). +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(oid) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() FROM PUBLIC; + +-- Internal C function to fetch table vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_scanned bigint, + OUT pages_removed bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_tables' +LANGUAGE C STRICT STABLE; + +-- Internal C function to fetch index vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( + IN dboid oid, + IN reloid oid, + OUT relid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision, + OUT rel_blks_read bigint, + OUT rel_blks_hit bigint, + OUT tuples_deleted bigint, + OUT pages_deleted bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' +LANGUAGE C STRICT STABLE; + +-- Internal C function to fetch database vacuum stats +CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_database( + IN dboid oid, + OUT dbid oid, + OUT total_blks_read bigint, + OUT total_blks_hit bigint, + OUT total_blks_dirtied bigint, + OUT total_blks_written bigint, + OUT wal_records bigint, + OUT wal_fpi bigint, + OUT wal_bytes numeric, + OUT blk_read_time double precision, + OUT blk_write_time double precision +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_database' +LANGUAGE C STRICT STABLE; + +-- View: vacuum statistics per table (heap) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_tables AS +SELECT + rel.oid AS relid, + ns.nspname AS schema, + rel.relname AS relname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_scanned, + stats.pages_removed +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind IN ('r', 'm', 't') + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_tables IS + 'Extended vacuum statistics per table (heap)'; + +-- View: vacuum statistics per index +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes AS +SELECT + rel.oid AS indexrelid, + ns.nspname AS schema, + rel.relname AS indexrelname, + db.datname AS dbname, + stats.total_blks_read, + stats.total_blks_hit, + stats.total_blks_dirtied, + stats.total_blks_written, + stats.wal_records, + stats.wal_fpi, + stats.wal_bytes, + stats.blk_read_time, + stats.blk_write_time, + stats.rel_blks_read, + stats.rel_blks_hit, + stats.tuples_deleted, + stats.pages_deleted +FROM pg_database db, + pg_class rel, + pg_namespace ns, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats +WHERE db.datname = current_database() + AND rel.relkind = 'i' + AND rel.relnamespace = ns.oid + AND rel.oid = stats.relid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS + 'Extended vacuum statistics per index'; + +-- View: vacuum statistics per database (aggregate) +CREATE VIEW ext_vacuum_statistics.pg_stats_vacuum_database AS +SELECT + db.oid AS dboid, + db.datname AS dbname, + stats.total_blks_read AS db_blks_read, + stats.total_blks_hit AS db_blks_hit, + stats.total_blks_dirtied AS db_blks_dirtied, + stats.total_blks_written AS db_blks_written, + stats.wal_records AS db_wal_records, + stats.wal_fpi AS db_wal_fpi, + stats.wal_bytes AS db_wal_bytes, + stats.blk_read_time AS db_blk_read_time, + stats.blk_write_time AS db_blk_write_time +FROM pg_database db +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) stats ON db.oid = stats.dbid; + +COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS + 'Extended vacuum statistics per database (aggregate)'; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf new file mode 100644 index 00000000000..c18eb26c54b --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config for ext_vacuum_statistics regression tests +shared_preload_libraries = 'ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control new file mode 100644 index 00000000000..518350a64b7 --- /dev/null +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics.control @@ -0,0 +1,5 @@ +# ext_vacuum_statistics extension +comment = 'Extended vacuum statistics via hook (requires shared_preload_libraries)' +default_version = '1.0' +relocatable = true +module_pathname = '$libdir/ext_vacuum_statistics' diff --git a/contrib/ext_vacuum_statistics/meson.build b/contrib/ext_vacuum_statistics/meson.build new file mode 100644 index 00000000000..555d2131233 --- /dev/null +++ b/contrib/ext_vacuum_statistics/meson.build @@ -0,0 +1,36 @@ +# Copyright (c) 2022-2026, PostgreSQL Global Development Group +# +# ext_vacuum_statistics - extended vacuum statistics via hook +# Requires shared_preload_libraries = 'ext_vacuum_statistics' + +ext_vacuum_statistics_sources = files( + 'vacuum_statistics.c', +) + +ext_vacuum_statistics = shared_module('ext_vacuum_statistics', + ext_vacuum_statistics_sources, + kwargs: contrib_mod_args + { + 'dependencies': contrib_mod_args['dependencies'], + }, +) +contrib_targets += ext_vacuum_statistics + +install_data( + 'ext_vacuum_statistics.control', + 'ext_vacuum_statistics--1.0.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'ext_vacuum_statistics', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/052_vacuum_extending_basic_test.pl', + 't/054_vacuum_extending_gucs_test.pl', + 't/056_vacuum_stats_gc.pl', + 't/060_vacuum_stats_reset.pl', + ], + }, +} diff --git a/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl new file mode 100644 index 00000000000..e97d2eb9a29 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/052_vacuum_extending_basic_test.pl @@ -0,0 +1,368 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# Test cumulative vacuum stats system using TAP +# +# This test validates the accuracy and behavior of cumulative vacuum statistics +# across heap tables, indexes, and databases using: +# +# • ext_vacuum_statistics.pg_stats_vacuum_tables +# • ext_vacuum_statistics.pg_stats_vacuum_indexes +# • ext_vacuum_statistics.pg_stats_vacuum_database +# +# A polling helper function repeatedly checks the stats views until expected +# deltas appear or a configurable timeout expires. This guarantees that +# stats-collector propagation delays do not lead to flaky test behavior. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +#------------------------------------------------------------------------------ +# Test harness setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('stat_vacuum'); +$node->init; + +# Configure the server: preload extension and logging level +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +my $stderr; +my $base_stats; +my $wals; +my $ibase_stats; +my $iwals; + +$node->start( + '>' => \$base_stats, + '2>' => \$stderr +); + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_database_regression; + CREATE EXTENSION ext_vacuum_statistics; +}); +# Main test database name and number of rows to insert +my $dbname = 'statistic_vacuum_database_regression'; +# The scenarios below expect the table and its index to span several pages +# of the default 8kB size; scale the row count with the block size, so that +# builds with larger blocks (32kB in Cloudberry) keep the same page counts. +my $size_tab = 1000 * + ($node->safe_psql('postgres', 'SHOW block_size') / 8192); + +# Enable required session settings and force the stats collector to flush next +$node->safe_psql($dbname, q{ + SET track_functions = 'all'; + SELECT pg_stat_force_next_flush(); +}); + +#------------------------------------------------------------------------------ +# Create test table and populate it +#------------------------------------------------------------------------------ + +$node->safe_psql( + $dbname, + "CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE vestat (x int PRIMARY KEY) + WITH (autovacuum_enabled = off, fillfactor = 10); + INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + ANALYZE vestat;" +); + +#------------------------------------------------------------------------------ +# Timing parameters for polling loops +#------------------------------------------------------------------------------ + +my $timeout = 30; # overall wait timeout in seconds +my $interval = 0.015; # poll interval in seconds (15 ms) +my $start_time = time(); +my $updated = 0; + +#------------------------------------------------------------------------------ +# wait_for_vacuum_stats +# +# Polls ext_vacuum_statistics.pg_stats_vacuum_tables and ext_vacuum_statistics.pg_stats_vacuum_indexes until both the +# table-level and index-level counters exceed the provided baselines, or until +# the configured timeout elapses. +# +# Expected named args (baseline values): +# tab_tuples_deleted +# tab_wal_records +# idx_tuples_deleted +# idx_wal_records +# +# Returns: 1 if the condition is met before timeout, 0 otherwise. +#------------------------------------------------------------------------------ + +sub wait_for_vacuum_stats { + my (%args) = @_; + my $tab_tuples_deleted = ($args{tab_tuples_deleted} or 0); + my $tab_wal_records = ($args{tab_wal_records} or 0); + my $idx_tuples_deleted = ($args{idx_tuples_deleted} or 0); + my $idx_wal_records = ($args{idx_wal_records} or 0); + + my $start = time(); + while ((time() - $start) < $timeout) { + + my $result_query = $node->safe_psql( + $dbname, + "VACUUM vestat; + SELECT + (SELECT (tuples_deleted > $tab_tuples_deleted AND wal_records > $tab_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat') + AND + (SELECT (tuples_deleted > $idx_tuples_deleted AND wal_records > $idx_wal_records) + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey');" + ); + + return 1 if ($result_query eq 't'); + + sleep($interval); + } + + return 0; +} + +#------------------------------------------------------------------------------ +# Variables to hold vacuum-stat snapshots for later comparisons +#------------------------------------------------------------------------------ + +my $tuples_deleted = 0; +my $pages_scanned = 0; +my $pages_removed = 0; +my $wal_records = 0; +my $wal_bytes = 0; +my $wal_fpi = 0; + +my $index_tuples_deleted = 0; +my $index_pages_deleted = 0; +my $index_wal_records = 0; +my $index_wal_bytes = 0; +my $index_wal_fpi = 0; + +my $tuples_deleted_prev = 0; +my $pages_scanned_prev = 0; +my $pages_removed_prev = 0; +my $wal_records_prev = 0; +my $wal_bytes_prev = 0; +my $wal_fpi_prev = 0; + +my $index_tuples_deleted_prev = 0; +my $index_pages_deleted_prev = 0; +my $index_wal_records_prev = 0; +my $index_wal_bytes_prev = 0; +my $index_wal_fpi_prev = 0; + +#------------------------------------------------------------------------------ +# fetch_vacuum_stats +# +# Reads current values of relevant vacuum counters for the test table and its +# primary index, storing them in package variables for subsequent comparisons. +#------------------------------------------------------------------------------ + +sub fetch_vacuum_stats { + # fetch actual base vacuum statistics + my $base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_scanned, pages_removed, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'vestat';" + ); + + $base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($tuples_deleted, $pages_scanned, $pages_removed, $wal_records, $wal_bytes, $wal_fpi) + = split /\s+/, $base_statistics; + + # --- index stats --- + my $index_base_statistics = $node->safe_psql( + $dbname, + "SELECT tuples_deleted, pages_deleted, wal_records, wal_bytes, wal_fpi + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'vestat_pkey';" + ); + + $index_base_statistics =~ s/\s*\|\s*/ /g; # transform " | " into space + ($index_tuples_deleted, $index_pages_deleted, $index_wal_records, $index_wal_bytes, $index_wal_fpi) + = split /\s+/, $index_base_statistics; +} + +#------------------------------------------------------------------------------ +# save_vacuum_stats +# +# Save current values (previously fetched by fetch_vacuum_stats) so that we +# later fetch new values and compare them. +#------------------------------------------------------------------------------ +sub save_vacuum_stats { + $tuples_deleted_prev = $tuples_deleted; + $pages_scanned_prev = $pages_scanned; + $pages_removed_prev = $pages_removed; + $wal_records_prev = $wal_records; + $wal_bytes_prev = $wal_bytes; + $wal_fpi_prev = $wal_fpi; + + $index_tuples_deleted_prev = $index_tuples_deleted; + $index_pages_deleted_prev = $index_pages_deleted; + $index_wal_records_prev = $index_wal_records; + $index_wal_bytes_prev = $index_wal_bytes; + $index_wal_fpi_prev = $index_wal_fpi; +} + +#------------------------------------------------------------------------------ +# print_vacuum_stats_on_error +# +# Print values in case of an error +#------------------------------------------------------------------------------ +sub print_vacuum_stats_on_error { + diag( + "Statistics in the failed test\n" . + "Table statistics:\n" . + " Before test:\n" . + " tuples_deleted = $tuples_deleted_prev\n" . + " pages_scanned = $pages_scanned_prev\n" . + " pages_removed = $pages_removed_prev\n" . + " wal_records = $wal_records_prev\n" . + " wal_bytes = $wal_bytes_prev\n" . + " wal_fpi = $wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $tuples_deleted\n" . + " pages_scanned = $pages_scanned\n" . + " pages_removed = $pages_removed\n" . + " wal_records = $wal_records\n" . + " wal_bytes = $wal_bytes\n" . + " wal_fpi = $wal_fpi\n" . + "Index statistics:\n" . + " Before test:\n" . + " tuples_deleted = $index_tuples_deleted_prev\n" . + " pages_deleted = $index_pages_deleted_prev\n" . + " wal_records = $index_wal_records_prev\n" . + " wal_bytes = $index_wal_bytes_prev\n" . + " wal_fpi = $index_wal_fpi_prev\n" . + " After test:\n" . + " tuples_deleted = $index_tuples_deleted\n" . + " pages_deleted = $index_pages_deleted\n" . + " wal_records = $index_wal_records\n" . + " wal_bytes = $index_wal_bytes\n" . + " wal_fpi = $index_wal_fpi\n" + ); +}; + + + +#------------------------------------------------------------------------------ +# Test 1: Delete half the rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 1: Delete half the rows, run VACUUM' => sub +{ + +$node->safe_psql($dbname, "DELETE FROM vestat WHERE x % 2 = 0;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +# Poll the stats view until expected deltas appear or timeout +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => 0, + tab_wal_records => 0, + idx_tuples_deleted => 0, + idx_wal_records => 0, +); +ok($updated, 'vacuum stats updated after vacuuming half-deleted table (tuples_deleted and wal_fpi advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming half-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > $wal_fpi_prev, 'table wal_fpi has increased'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 2: Delete all rows, run VACUUM, and wait for stats to advance +#------------------------------------------------------------------------------ +subtest 'Test 2: Delete all rows, run VACUUM' => sub +{ +save_vacuum_stats(); + +$node->safe_psql($dbname, "DELETE FROM vestat;"); +$node->safe_psql($dbname, "VACUUM vestat;"); + +$updated = wait_for_vacuum_stats( + tab_tuples_deleted => $tuples_deleted_prev, + tab_wal_records => $wal_records_prev, + idx_tuples_deleted => $index_tuples_deleted_prev, + idx_wal_records => $index_wal_records_prev, +); + +ok($updated, 'vacuum stats updated after vacuuming all-deleted table (tuples_deleted and wal_records advanced)') + or diag "Timeout waiting for ext_vacuum_statistics update after $timeout seconds after vacuuming all-deleted table"; + +fetch_vacuum_stats(); + +ok($tuples_deleted > $tuples_deleted_prev, 'table tuples_deleted has increased'); +ok($pages_scanned > $pages_scanned_prev, 'table pages_scanned has increased'); +ok($pages_removed > $pages_removed_prev, 'table pages_removed has increased'); +ok($wal_records > $wal_records_prev, 'table wal_records has increased'); +ok($wal_bytes > $wal_bytes_prev, 'table wal_bytes has increased'); +ok($wal_fpi > 0, 'table wal_fpi has increased'); + +ok($index_pages_deleted > $index_pages_deleted_prev, 'index pages_deleted has increased'); +ok($index_tuples_deleted > $index_tuples_deleted_prev, 'index tuples_deleted has increased'); +ok($index_wal_records > $index_wal_records_prev, 'index wal_records has increased'); +ok($index_wal_bytes > $index_wal_bytes_prev, 'index wal_bytes has increased'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +#------------------------------------------------------------------------------ +# Test 3: Test VACUUM FULL — it should not report to the stats collector +#------------------------------------------------------------------------------ +subtest 'Test 3: Test VACUUM FULL — it should not report to the stats collector' => sub +{ +save_vacuum_stats(); + +$node->safe_psql( + $dbname, + "INSERT INTO vestat SELECT x FROM generate_series(1, $size_tab) AS g(x); + CHECKPOINT; + DELETE FROM vestat; + VACUUM FULL vestat;" +); + +fetch_vacuum_stats(); + +ok($tuples_deleted == $tuples_deleted_prev, 'table tuples_deleted stay the same'); +ok($pages_scanned == $pages_scanned_prev, 'table pages_scanned stay the same'); +ok($pages_removed == $pages_removed_prev, 'table pages_removed stay the same'); +ok($wal_records == $wal_records_prev, 'table wal_records stay the same'); +ok($wal_bytes == $wal_bytes_prev, 'table wal_bytes stay the same'); +ok($wal_fpi == $wal_fpi_prev, 'table wal_fpi stay the same'); + +ok($index_pages_deleted == $index_pages_deleted_prev, 'index pages_deleted stay the same'); +ok($index_tuples_deleted == $index_tuples_deleted_prev, 'index tuples_deleted stay the same'); +ok($index_wal_records == $index_wal_records_prev, 'index wal_records stay the same'); +ok($index_wal_bytes == $index_wal_bytes_prev, 'index wal_bytes stay the same'); +ok($index_wal_fpi == $index_wal_fpi_prev, 'index wal_fpi stay the same'); + +} or print_vacuum_stats_on_error(); + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl new file mode 100644 index 00000000000..238d3e5634b --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl @@ -0,0 +1,99 @@ +# Copyright (c) 2025 PostgreSQL Global Development Group +# +# Test GUC parameters for ext_vacuum_statistics extension: +# vacuum_statistics.enabled +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; + +use Test::More; + +#------------------------------------------------------------------------------ +# Test cluster setup +#------------------------------------------------------------------------------ + +my $node = PostgreSQL::Test::Cluster->new('ext_stat_vacuum_gucs'); +$node->init; + +$node->append_conf('postgresql.conf', q{ + shared_preload_libraries = 'ext_vacuum_statistics' + log_min_messages = notice +}); + +$node->start; + +#------------------------------------------------------------------------------ +# Database creation and initialization +#------------------------------------------------------------------------------ + +$node->safe_psql('postgres', q{ + CREATE DATABASE statistic_vacuum_gucs; +}); + +my $dbname = 'statistic_vacuum_gucs'; + +$node->safe_psql($dbname, q{ + CREATE EXTENSION ext_vacuum_statistics; + CREATE TABLE guc_test (x int PRIMARY KEY) + WITH (autovacuum_enabled = off); + INSERT INTO guc_test SELECT x FROM generate_series(1, 100) AS g(x); + ANALYZE guc_test; +}); + +#------------------------------------------------------------------------------ +# Reset stats and run vacuum (all in one session so GUCs persist) +#------------------------------------------------------------------------------ + +sub reset_and_vacuum { + my ($db, $table, $opts) = @_; + $table ||= 'guc_test'; + my $gucs = $opts && $opts->{gucs} ? $opts->{gucs} : []; + my $modify = $opts && $opts->{modify}; + my $extra = $opts && $opts->{extra_vacuum} ? $opts->{extra_vacuum} : []; + $extra = [$extra] unless ref $extra eq 'ARRAY'; + my $sql = join("\n", (map { "SET $_;" } @$gucs), + "SELECT ext_vacuum_statistics.vacuum_statistics_reset();", + $modify ? ( + "TRUNCATE $table;", + "INSERT INTO $table SELECT x FROM generate_series(1, 100) AS g(x);", + "DELETE FROM $table;", + ) : (), + "VACUUM $table;", + (map { "VACUUM $_;" } @$extra), + # Make pending stats visible to subsequent sessions without sleeping. + "SELECT pg_stat_force_next_flush();"); + $node->safe_psql($db, $sql); +} + +#------------------------------------------------------------------------------ +# Test 1: vacuum_statistics.enabled +#------------------------------------------------------------------------------ +subtest 'vacuum_statistics.enabled' => sub { + reset_and_vacuum($dbname); + + # Default: enabled - should have stats + my $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + ok($count > 0, 'stats collected when enabled'); + + # Reset preserves existing rows. With collection disabled, their counters + # must stay zero even after another vacuum. + reset_and_vacuum($dbname, 'guc_test', { gucs => ['vacuum_statistics.enabled = off'] }); + + $count = $node->safe_psql($dbname, + "SELECT COUNT(*) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test'"); + is($count, 1, 'reset preserves the row when collection is disabled'); + + my $sums = $node->safe_psql($dbname, q{ + SELECT COALESCE(SUM(tuples_deleted), 0) + + COALESCE(SUM(pages_scanned), 0) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'guc_test' + }); + is($sums, '0', 'no counters accumulated when disabled'); +}; + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl b/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl new file mode 100644 index 00000000000..30f8317a197 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/056_vacuum_stats_gc.pl @@ -0,0 +1,228 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that vacuum statistics entries do not outlive the relations they +# describe: DROP TABLE, a rolled back DROP, a restart, DROP DATABASE, and a +# new relation that gets the OID of an old one. +# +# Entries are looked up with pg_stats_get_vacuum_tables() and +# pg_stats_get_vacuum_indexes(), which find an entry by OID even when the +# relation is no longer in pg_class. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_gc'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'ext_vacuum_statistics' +autovacuum = off +}); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION ext_vacuum_statistics'); + +# SQL that counts the statistics entries of the given tables and indexes. +sub entries_sql +{ + my ($dboid, $tables, $indexes) = @_; + my $t = join(',', @$tables); + my $i = join(',', @$indexes); + + return qq{ +SELECT (SELECT count(*) + FROM unnest('{$t}'::oid[]) AS r, + ext_vacuum_statistics.pg_stats_get_vacuum_tables($dboid, r)) + + (SELECT count(*) + FROM unnest('{$i}'::oid[]) AS r, + ext_vacuum_statistics.pg_stats_get_vacuum_indexes($dboid, r))}; +} + +sub count_entries +{ + return $node->safe_psql('postgres', entries_sql(@_)); +} + +# Wait until the entries are gone. +sub wait_for_no_entries +{ + my ($what, @args) = @_; + + $node->poll_query_until('postgres', + 'SELECT (' . entries_sql(@args) . ') = 0') + or die "timed out waiting for $what"; + return; +} + +# OIDs of the relations of the given kind whose names match a pattern. +sub relids +{ + my ($dbname, $relkind, $pattern) = @_; + + return split /\n/, + $node->safe_psql($dbname, + "SELECT oid FROM pg_class WHERE relkind = '$relkind' AND relname LIKE '$pattern' ORDER BY oid" + ); +} + +my $dboid = $node->safe_psql('postgres', + "SELECT oid FROM pg_database WHERE datname = 'postgres'"); + +# Vacuum a handful of tables, each with an index. +my @tables = map { "gc_tab$_" } (1 .. 20); + +$node->safe_psql('postgres', + join('', map { qq[ +CREATE TABLE $_ (id int); +CREATE INDEX ${_}_idx ON $_ (id); +INSERT INTO $_ SELECT generate_series(1, 100); +DELETE FROM $_;] } @tables)); +$node->safe_psql('postgres', 'VACUUM ' . join(', ', @tables)); + +my @tabids = relids('postgres', 'r', 'gc_tab%'); +my @idxids = relids('postgres', 'i', 'gc_tab%'); +is(scalar(@tabids) + scalar(@idxids), 40, 'found the tables and indexes'); +is(count_entries($dboid, \@tabids, \@idxids), + 40, 'vacuum created an entry for each table and index'); + +# A rolled back DROP must keep the statistics. +$node->safe_psql('postgres', 'BEGIN; DROP TABLE gc_tab1; ROLLBACK;'); +is(count_entries($dboid, \@tabids, \@idxids), + 40, 'rolled back DROP TABLE kept the statistics'); + +# Dropping the tables drops their entries and those of their indexes. +$node->safe_psql('postgres', 'DROP TABLE ' . join(', ', @tables)); +wait_for_no_entries('DROP TABLE to drop the statistics', + $dboid, \@tabids, \@idxids); +is(count_entries($dboid, \@tabids, \@idxids), + 0, 'DROP TABLE dropped the statistics entries'); + +# Nothing may come back from the statistics file either. +$node->restart; +is(count_entries($dboid, \@tabids, \@idxids), + 0, 'no dropped entries came back after a restart'); + +# The same for a whole database. +$node->safe_psql('postgres', 'CREATE DATABASE vacstat_gc_db'); +$node->safe_psql( + 'vacstat_gc_db', q{ +CREATE TABLE gc_dbtab (id int); +CREATE INDEX gc_dbtab_idx ON gc_dbtab (id); +INSERT INTO gc_dbtab SELECT generate_series(1, 100); +DELETE FROM gc_dbtab; +VACUUM gc_dbtab; +}); +my $otherdb = $node->safe_psql('postgres', + "SELECT oid FROM pg_database WHERE datname = 'vacstat_gc_db'"); +my @dbtabids = relids('vacstat_gc_db', 'r', 'gc_dbtab'); +my @dbidxids = relids('vacstat_gc_db', 'i', 'gc_dbtab_idx'); +is(count_entries($otherdb, \@dbtabids, \@dbidxids), + 2, 'vacuum in another database created statistics entries'); + +$node->safe_psql('postgres', 'DROP DATABASE vacstat_gc_db'); +wait_for_no_entries('DROP DATABASE to drop the statistics', + $otherdb, \@dbtabids, \@dbidxids); +is(count_entries($otherdb, \@dbtabids, \@dbidxids), + 0, 'DROP DATABASE dropped the statistics entries'); + +# A relation created on the OID of an earlier one must not inherit its +# statistics. DROP never leaves an entry behind, and OIDs are only handed +# out again after a wraparound, so fake the situation: take a vacuumed table +# out of the catalogs behind the back of DROP, which leaves its entry +# orphaned, then create a new table with the very same OIDs in binary upgrade +# mode. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE gc_reuse (id int); +INSERT INTO gc_reuse SELECT generate_series(1, 100); +DELETE FROM gc_reuse; +VACUUM gc_reuse; +}); +my ($relid, $reltype, $typarray) = split /\|/, + $node->safe_psql( + 'postgres', q{ +SELECT c.oid, c.reltype, t.typarray + FROM pg_class c JOIN pg_type t ON t.oid = c.reltype + WHERE c.relname = 'gc_reuse'}); +is( $node->safe_psql( + 'postgres', + "SELECT tuples_deleted FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relid = $relid" + ), + '100', + 'the table whose OID gets recycled has statistics'); + +$node->safe_psql( + 'postgres', qq{ +SET allow_system_table_mods = on; +DELETE FROM pg_depend + WHERE (classid = 'pg_class'::regclass AND objid = $relid) + OR (refclassid = 'pg_class'::regclass AND refobjid = $relid) + OR (classid = 'pg_type'::regclass AND objid IN ($reltype, $typarray)) + OR (refclassid = 'pg_type'::regclass AND refobjid IN ($reltype, $typarray)); +DELETE FROM pg_attribute WHERE attrelid = $relid; +DELETE FROM pg_type WHERE oid IN ($reltype, $typarray); +DELETE FROM pg_class WHERE oid = $relid; +}); + +# Binary upgrade mode is a postmaster switch, so start the node by hand. +# Cloudberry: a standalone node also needs the options that +# PostgreSQL::Test::Cluster::start() passes to it. +$node->stop; +command_ok( + [ + 'pg_ctl', + '--pgdata' => $node->data_dir, + '--log' => $node->logfile, + '--options' => "-b -c gp_role=utility --gp_dbid=$node->{_dbid} " + . '--gp_contentid=0 -c maintenance_mode=on', + '--wait', 'start' + ], + 'started in binary upgrade mode'); + +# The relfilenumber only has to be unused; the old file is still on disk. +# Cloudberry: the OIDs are preassigned for a given namespace and name. +my ($ret, $stdout, $stderr) = $node->psql( + 'postgres', qq{ +SELECT binary_upgrade_set_next_heap_pg_class_oid('$relid'::oid, + 'public'::regnamespace, 'gc_reuse_new'); +SELECT binary_upgrade_set_next_heap_relfilenode('3000000000'::oid); +SELECT binary_upgrade_set_next_pg_type_oid('$reltype'::oid, + 'public'::regnamespace, 'gc_reuse_new'); +SELECT binary_upgrade_set_next_array_pg_type_oid('$typarray'::oid, + 'public'::regnamespace, '_gc_reuse_new'); +CREATE TABLE gc_reuse_new (id int); +}); +is($ret, 0, 'created a table on the recycled OID'); +like( + $stderr, + qr/resetting existing statistics for kind ext_vacuum_statistics_relation/, + 'creating the table reset the orphaned statistics entry'); + +command_ok( + [ + 'pg_ctl', + '--pgdata' => $node->data_dir, + '--mode' => 'fast', + '--wait', 'stop' + ], + 'stopped binary upgrade mode'); +$node->start; + +is( $node->safe_psql( + 'postgres', "SELECT oid FROM pg_class WHERE relname = 'gc_reuse_new'"), + $relid, + 'the new table got the recycled OID'); +is( $node->safe_psql( + 'postgres', q{ +SELECT coalesce((SELECT tuples_deleted + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'gc_reuse_new'), 0)}), + '0', + 'the new table did not inherit the statistics of the old one'); + +$node->stop; + +done_testing(); diff --git a/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl new file mode 100644 index 00000000000..ea28cb5d66d --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl @@ -0,0 +1,114 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Reset privileges, scope, and preservation of the entry's type, including +# after a restart and through the built-in pg_stat_reset() path. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_reset'); +$node->init; +$node->append_conf('postgresql.conf', q{ +shared_preload_libraries = 'ext_vacuum_statistics' +autovacuum = off +}); +$node->start; +$node->safe_psql('postgres', 'CREATE DATABASE other_reset'); +for my $db ('postgres', 'other_reset') +{ + $node->safe_psql($db, q{ +CREATE EXTENSION ext_vacuum_statistics; +CREATE TABLE reset_t (id int PRIMARY KEY) WITH (autovacuum_enabled = off); +INSERT INTO reset_t SELECT generate_series(1, 1000); +DELETE FROM reset_t WHERE id < 500; +}); + $node->safe_psql($db, 'VACUUM reset_t'); +} +my $dboid = $node->safe_psql('postgres', + q[SELECT oid FROM pg_database WHERE datname = current_database()]); +my $relid = $node->safe_psql('postgres', q[SELECT 'reset_t'::regclass::oid]); +my $idxid = $node->safe_psql('postgres', q[SELECT 'reset_t_pkey'::regclass::oid]); +my $table = q[SELECT pages_scanned FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'reset_t']; +my $index = q[SELECT tuples_deleted FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'reset_t_pkey']; +my $database = q[SELECT db_blks_read + db_blks_hit FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dbname = current_database()]; +my $table_before = $node->safe_psql('postgres', $table); +my $index_before = $node->safe_psql('postgres', $index); +my $db_before = $node->safe_psql('postgres', $database); +cmp_ok($table_before, '>', 0, 'table statistics populated'); +cmp_ok($index_before, '>', 0, 'index statistics populated'); +cmp_ok($db_before, '>', 0, 'database statistics populated'); + +$node->safe_psql('postgres', q{ +CREATE ROLE reset_reader; +GRANT USAGE ON SCHEMA ext_vacuum_statistics TO reset_reader; +}); +for my $call ('extvac_reset_entry(0, 0)', 'extvac_reset_db_entry(0)', + 'vacuum_statistics_reset()') +{ + my ($stdout, $stderr); + my $ret = $node->psql('postgres', + "SET ROLE reset_reader; SELECT ext_vacuum_statistics.$call", + stdout => \$stdout, stderr => \$stderr); + isnt($ret, 0, "$call denied to a schema reader"); + like($stderr, qr/permission denied for function/, 'function ACL rejects reset'); +} +is($node->safe_psql('postgres', $table), $table_before, + 'denied resets leave statistics unchanged'); + +$node->safe_psql('postgres', qq{ +GRANT EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) TO reset_reader; +SET ROLE reset_reader; +SELECT ext_vacuum_statistics.extvac_reset_entry($dboid, $relid); +}); +is($node->safe_psql('postgres', $table), '0', + 'delegated relation reset preserves a visible zero row'); +is($node->safe_psql('postgres', $index), $index_before, + 'table reset leaves index counters unchanged'); +is($node->safe_psql('postgres', $database), $db_before, + 'table reset leaves database aggregate unchanged'); +$node->safe_psql('postgres', + "SELECT ext_vacuum_statistics.extvac_reset_entry($dboid, $idxid)"); +is($node->safe_psql('postgres', $index), '0', + 'index reset preserves a visible zero row'); + +# SQL return types do not claim to count reset entries. Missing entries are +# harmless no-ops, consistent with built-in reset functions. +is($node->safe_psql('postgres', q{ +SELECT bool_and(p.prorettype = 'void'::regtype AND p.proparallel = 'u' AND p.provolatile = 'v') +FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname = 'ext_vacuum_statistics' AND p.proname LIKE '%reset%' +}), 't', 'all resets return void and are volatile and parallel unsafe'); +$node->safe_psql('postgres', q{ +SELECT ext_vacuum_statistics.extvac_reset_entry(4294967294, 4294967294); +SELECT ext_vacuum_statistics.extvac_reset_db_entry(4294967294); +}); + +$node->safe_psql('postgres', 'VACUUM reset_t'); +$node->restart; +$node->safe_psql('postgres', + "SELECT ext_vacuum_statistics.extvac_reset_db_entry($dboid)"); +is($node->safe_psql('postgres', $table), '0', 'database reset retains table type after restart'); +is($node->safe_psql('postgres', $index), '0', 'database reset retains index type after restart'); +is($node->safe_psql('postgres', $database), '0', 'database reset retains aggregate type after restart'); +cmp_ok($node->safe_psql('other_reset', $table), '>', 0, + 'database reset leaves another database untouched'); + +$node->safe_psql('postgres', 'VACUUM reset_t'); +$node->safe_psql('postgres', 'SELECT pg_stat_reset()'); +is($node->safe_psql('postgres', $table), '0', 'core reset retains table row'); +is($node->safe_psql('postgres', $index), '0', 'core reset retains index row'); +is($node->safe_psql('postgres', $database), '0', 'core reset retains database row'); +$node->safe_psql('postgres', 'VACUUM reset_t'); +cmp_ok($node->safe_psql('postgres', $table), '>', 0, + 'vacuum resumes accumulation after reset'); +$node->safe_psql('postgres', 'SELECT ext_vacuum_statistics.vacuum_statistics_reset()'); +for my $db ('postgres', 'other_reset') +{ + is($node->safe_psql($db, $table), '0', "global reset clears table in $db"); + is($node->safe_psql($db, $index), '0', "global reset clears index in $db"); + is($node->safe_psql($db, $database), '0', "global reset clears aggregate in $db"); +} +$node->stop; +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c new file mode 100644 index 00000000000..17ed45743b6 --- /dev/null +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -0,0 +1,538 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * ext_vacuum_statistics - Extended vacuum statistics for PostgreSQL + * + * This module collects detailed vacuum statistics (I/O, WAL, timing, etc.) + * at relation and database level by hooking into the vacuum reporting path. + * Statistics are stored via pgstat custom statistics. Management of statistics + * storage and output functions are implemented in this module. + */ +#include "postgres.h" + +#include "catalog/objectaccess.h" +#include "catalog/pg_class.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "utils/builtins.h" +#include "utils/fmgrprotos.h" +#include "utils/guc.h" +#include "utils/pgstat_internal.h" +#include "utils/pgstat_kind.h" +#include "utils/tuplestore.h" +#include "utils/timestamp.h" + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* Two kinds: relations (tables/indexes) and database aggregates */ +#define PGSTAT_KIND_EXTVAC_RELATION 24 +#define PGSTAT_KIND_EXTVAC_DB 25 + +#define SJ_NODENAME "vacuum_statistics" + +/* GUCs */ +static bool evs_enabled = true; + +/* Hooks */ +static set_report_vacuum_hook_type prev_report_vacuum_hook = NULL; +static object_access_hook_type prev_object_access_hook = NULL; + +/* Forward declarations */ +static void pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params); +static void extvac_object_access(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); + +/* Shared memory entry for vacuum stats; one per relation or database. */ +typedef struct PgStatShared_ExtVacEntry +{ + PgStatShared_Common header; + PgStat_VacuumRelationCounts stats; +} PgStatShared_ExtVacEntry; + +/* A reset clears counters, not the table/index/database discriminator. */ +static void +extvac_reset_data(PgStatShared_Common *header) +{ + PgStatShared_ExtVacEntry *entry = (PgStatShared_ExtVacEntry *) header; + ExtVacReportType type = entry->stats.type; + + memset(&entry->stats, 0, sizeof(entry->stats)); + entry->stats.type = type; +} + +/* PgStat kind for per-relation vacuum statistics (tables/indexes) */ +static const PgStat_KindInfo extvac_relation_kind_info = { + .name = "ext_vacuum_statistics_relation", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, + .reset_data_cb = extvac_reset_data, +}; + +/* PgStat kind for per-database aggregated vacuum statistics */ +static const PgStat_KindInfo extvac_db_kind_info = { + .name = "ext_vacuum_statistics_db", + .fixed_amount = false, + .accessed_across_databases = true, + .write_to_file = true, + .shared_size = sizeof(PgStatShared_ExtVacEntry), + .shared_data_off = offsetof(PgStatShared_ExtVacEntry, stats), + .shared_data_len = sizeof(PgStat_VacuumRelationCounts), + .pending_size = 0, + .flush_pending_cb = NULL, + .reset_data_cb = extvac_reset_data, +}; + +static inline void +pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * src) +{ + dst->total_blks_read += src->total_blks_read; + dst->total_blks_hit += src->total_blks_hit; + dst->total_blks_dirtied += src->total_blks_dirtied; + dst->total_blks_written += src->total_blks_written; + dst->blks_fetched += src->blks_fetched; + dst->blks_hit += src->blks_hit; + dst->blk_read_time += src->blk_read_time; + dst->blk_write_time += src->blk_write_time; + dst->wal_records += src->wal_records; + dst->wal_fpi += src->wal_fpi; + dst->wal_bytes += src->wal_bytes; + dst->tuples_deleted += src->tuples_deleted; +} + +static inline void +pgstat_accumulate_extvac_stats(PgStat_VacuumRelationCounts * dst, + const PgStat_VacuumRelationCounts * src) +{ + if (dst->type == PGSTAT_EXTVAC_INVALID) + dst->type = src->type; + + Assert(src->type != PGSTAT_EXTVAC_INVALID && src->type != PGSTAT_EXTVAC_DB); + Assert(src->type == dst->type); + + pgstat_accumulate_common(&dst->common, &src->common); + + if (dst->type == PGSTAT_EXTVAC_TABLE) + { + dst->table.pages_scanned += src->table.pages_scanned; + dst->table.pages_removed += src->table.pages_removed; + } + else if (dst->type == PGSTAT_EXTVAC_INDEX) + { + dst->pages_deleted += src->pages_deleted; + } +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics module could be loaded only on startup."), + errdetail("Add 'ext_vacuum_statistics' into the shared_preload_libraries list."))); + + DefineCustomBoolVariable("vacuum_statistics.enabled", + "Enable extended vacuum statistics collection.", + NULL, &evs_enabled, true, + PGC_SUSET, GUC_GPDB_NEED_SYNC, + NULL, NULL, NULL); + + MarkGUCPrefixReserved(SJ_NODENAME); + + pgstat_register_kind(PGSTAT_KIND_EXTVAC_RELATION, &extvac_relation_kind_info); + pgstat_register_kind(PGSTAT_KIND_EXTVAC_DB, &extvac_db_kind_info); + + prev_report_vacuum_hook = set_report_vacuum_hook; + set_report_vacuum_hook = pgstat_report_vacuum_extstats; + + prev_object_access_hook = object_access_hook; + object_access_hook = extvac_object_access; +} + +/* + * Object access hook: drop the statistics of a dropped relation, and reset + * old statistics when a new relation is created. + * + * Otherwise the statistics of a dropped relation stay in memory and in the + * statistics file, and a new relation with the same OID gets them. Both + * actions are transactional, as for the built-in relation statistics. + * + * Statistics are keyed by the relation OID, and shared catalogs are never + * created or dropped after initdb, so we do not need to read pg_class. + * + * Dropped databases need no handling: dropping a database drops all its + * statistics entries, ours included. + */ +static void +extvac_object_access(ObjectAccessType access, Oid classId, Oid objectId, + int subId, void *arg) +{ + if (prev_object_access_hook) + prev_object_access_hook(access, classId, objectId, subId, arg); + + /* only whole relations are of interest, not their columns */ + if (classId != RelationRelationId || subId != 0) + return; + + if (access == OAT_DROP) + pgstat_drop_transactional(PGSTAT_KIND_EXTVAC_RELATION, MyDatabaseId, + objectId); + else if (access == OAT_POST_CREATE) + { + PgStat_EntryRef *entry_ref; + + /* An OID can now identify a different kind of relation. */ + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_EXTVAC_RELATION, + MyDatabaseId, objectId, false, NULL); + if (entry_ref && !entry_ref->shared_entry->dropped) + { + PgStatShared_ExtVacEntry *entry; + + (void) pgstat_lock_entry(entry_ref, false); + entry = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + entry->stats.type = PGSTAT_EXTVAC_INVALID; + pgstat_unlock_entry(entry_ref); + } + + /* + * Discard whatever an earlier owner of this OID left behind, and + * arrange for the entry to be dropped again should the creating + * transaction roll back. This mirrors what pgstat_create_relation() + * does for the built-in relation statistics. + */ + pgstat_create_transactional(PGSTAT_KIND_EXTVAC_RELATION, MyDatabaseId, + objectId); + } +} + +/* Accumulate common counts for database-level stats. */ +static inline void +pgstat_accumulate_common_for_db(PgStat_CommonCounts * dst, + const PgStat_CommonCounts * src) +{ + pgstat_accumulate_common(dst, src); +} + +/* + * Store incoming vacuum stats into pgstat custom statistics. + * store_relation: create/update per-relation entry + * store_db: accumulate into database-level entry (dboid, objid=0). + * Uses pgstat_get_entry_ref_locked and pgstat_accumulate_* for atomic updates. + */ +static void +extvac_store(Oid dboid, Oid relid, PgStat_VacuumRelationCounts * params, + bool store_relation, bool store_db) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_ExtVacEntry *shared; + + if (!evs_enabled) + return; + + if (store_relation) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = params->type; + } + pgstat_accumulate_extvac_stats(&shared->stats, params); + pgstat_unlock_entry(entry_ref); + } + } + + if (store_db) + { + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, false); + if (entry_ref) + { + shared = (PgStatShared_ExtVacEntry *) entry_ref->shared_stats; + if (shared->stats.type == PGSTAT_EXTVAC_INVALID) + { + memset(&shared->stats, 0, sizeof(shared->stats)); + shared->stats.type = PGSTAT_EXTVAC_DB; + } + pgstat_accumulate_common_for_db(&shared->stats.common, ¶ms->common); + pgstat_unlock_entry(entry_ref); + } + } +} + +/* + * Vacuum report hook: called when vacuum finishes. Stores stats per-relation + * and per-database, then chains to previous hook. + */ +static void +pgstat_report_vacuum_extstats(Oid tableoid, bool shared, + PgStat_VacuumRelationCounts * params) +{ + Oid dboid = shared ? InvalidOid : MyDatabaseId; + + if (evs_enabled) + extvac_store(dboid, tableoid, params, true, true); + if (prev_report_vacuum_hook) + prev_report_vacuum_hook(tableoid, shared, params); +} + +/* Reset statistics for a single relation entry. */ +static void +extvac_reset_by_relid(Oid dboid, Oid relid) +{ + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_RELATION, dboid, relid, + GetCurrentTimestamp()); +} + +/* Callback for pgstat_reset_matching_entries: match relation entries for given db */ +static bool +match_extvac_relations_for_db(PgStatShared_HashEntry *entry, Datum match_data) +{ + return entry->key.kind == PGSTAT_KIND_EXTVAC_RELATION && + entry->key.dboid == DatumGetObjectId(match_data); +} + +/* + * Reset statistics for a database (aggregate entry) and all its relations. + */ +static void +extvac_database_reset(Oid dboid) +{ + TimestampTz ts = GetCurrentTimestamp(); + + pgstat_reset_matching_entries(match_extvac_relations_for_db, + ObjectIdGetDatum(dboid), ts); + pgstat_reset_entry(PGSTAT_KIND_EXTVAC_DB, dboid, InvalidOid, ts); +} + +/* Reset all vacuum statistics (both relation and database entries). */ +static void +extvac_stat_reset(void) +{ + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_RELATION); + pgstat_reset_of_kind(PGSTAT_KIND_EXTVAC_DB); +} + +PG_FUNCTION_INFO_V1(vacuum_statistics_reset); +PG_FUNCTION_INFO_V1(extvac_reset_entry); +PG_FUNCTION_INFO_V1(extvac_reset_db_entry); + +Datum +vacuum_statistics_reset(PG_FUNCTION_ARGS) +{ + extvac_stat_reset(); + PG_RETURN_VOID(); +} + +Datum +extvac_reset_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + Oid relid = PG_GETARG_OID(1); + + extvac_reset_by_relid(dboid, relid); + PG_RETURN_VOID(); +} + +Datum +extvac_reset_db_entry(PG_FUNCTION_ARGS) +{ + Oid dboid = PG_GETARG_OID(0); + + extvac_database_reset(dboid); + PG_RETURN_VOID(); +} + +/* + * Output vacuum statistics (tables, indexes, or per-database aggregates). + */ +#define EXTVAC_COMMON_STAT_COLS 9 + +static void +tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, + Datum *values, bool *nulls, int *i) +{ + char buf[256]; + const int base PG_USED_FOR_ASSERTS_ONLY = *i; + + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_read); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_hit); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_dirtied); + values[(*i)++] = Int64GetDatum(vacuum_ext->total_blks_written); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_records); + values[(*i)++] = Int64GetDatum(vacuum_ext->wal_fpi); + snprintf(buf, sizeof buf, UINT64_FORMAT, vacuum_ext->wal_bytes); + values[(*i)++] = DirectFunctionCall3(numeric_in, + CStringGetDatum(buf), + ObjectIdGetDatum(0), + Int32GetDatum(-1)); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_read_time); + values[(*i)++] = Float8GetDatum(vacuum_ext->blk_write_time); + Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); +} + +#define EXTVAC_HEAP_STAT_COLS 15 +#define EXTVAC_IDX_STAT_COLS 14 +#define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) + +static void +tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, + TupleDesc tupdesc, PgStat_VacuumRelationCounts * vacuum_ext) +{ + Datum values[EXTVAC_MAX_STAT_COLS]; + bool nulls[EXTVAC_MAX_STAT_COLS]; + int i = 0; + + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(relid); + + tuplestore_put_common(&vacuum_ext->common, values, nulls, &i); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_fetched - vacuum_ext->common.blks_hit); + values[i++] = Int64GetDatum(vacuum_ext->common.blks_hit); + + if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_scanned); + values[i++] = Int64GetDatum(vacuum_ext->table.pages_removed); + } + else if (vacuum_ext->type == PGSTAT_EXTVAC_INDEX) + { + values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); + values[i++] = Int64GetDatum(vacuum_ext->pages_deleted); + } + + Assert(i == ((vacuum_ext->type == PGSTAT_EXTVAC_TABLE) ? EXTVAC_HEAP_STAT_COLS : EXTVAC_IDX_STAT_COLS)); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); +} + +static Datum +pg_stats_vacuum(FunctionCallInfo fcinfo, int type) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + Tuplestorestate *tupstore; + TupleDesc tupdesc; + Oid dbid = PG_GETARG_OID(0); + + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ext_vacuum_statistics: materialize mode required"))); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "ext_vacuum_statistics: return type must be a row type"); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + if (type == PGSTAT_EXTVAC_INDEX || type == PGSTAT_EXTVAC_TABLE) + { + Oid relid = PG_GETARG_OID(1); + PgStat_VacuumRelationCounts *stats; + + if (!OidIsValid(relid)) + return (Datum) 0; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, dbid, relid); + + if (!stats) + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_RELATION, InvalidOid, + relid); + + if (stats && stats->type == type) + tuplestore_put_for_relation(relid, tupstore, tupdesc, stats); + } + else if (type == PGSTAT_EXTVAC_DB) + { + if (OidIsValid(dbid)) + { +#define EXTVAC_DB_STAT_COLS 10 + Datum values[EXTVAC_DB_STAT_COLS]; + bool nulls[EXTVAC_DB_STAT_COLS]; + int i = 0; + PgStat_VacuumRelationCounts *stats; + + stats = (PgStat_VacuumRelationCounts *) + pgstat_fetch_entry(PGSTAT_KIND_EXTVAC_DB, dbid, + InvalidOid); + if (stats && stats->type == PGSTAT_EXTVAC_DB) + { + memset(nulls, 0, sizeof(nulls)); + values[i++] = ObjectIdGetDatum(dbid); + tuplestore_put_common(&stats->common, values, nulls, &i); + Assert(i == EXTVAC_DB_STAT_COLS); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + } + /* invalid dbid: return empty set */ + } + else + elog(PANIC, "ext_vacuum_statistics: invalid type %d", type); + + return (Datum) 0; +} + +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_tables); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_indexes); +PG_FUNCTION_INFO_V1(pg_stats_get_vacuum_database); + +Datum +pg_stats_get_vacuum_tables(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_TABLE); +} + +Datum +pg_stats_get_vacuum_indexes(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_INDEX); +} + +Datum +pg_stats_get_vacuum_database(PG_FUNCTION_ARGS) +{ + return pg_stats_vacuum(fcinfo, PGSTAT_EXTVAC_DB); +} diff --git a/contrib/meson.build b/contrib/meson.build index bd4a57c43c0..67b38432c55 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -22,6 +22,7 @@ subdir('cube') subdir('dblink') subdir('dict_int') subdir('dict_xsyn') +subdir('ext_vacuum_statistics') subdir('earthdistance') subdir('file_fdw') subdir('fuzzystrmatch') diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 922421a8971..2471595334b 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -142,6 +142,7 @@ CREATE EXTENSION extension_name; &dict-int; &dict-xsyn; &earthdistance; + &extvacuumstatistics; &file-fdw; &fuzzystrmatch; &hstore; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml new file mode 100644 index 00000000000..6cdc0678f27 --- /dev/null +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -0,0 +1,193 @@ + + + + ext_vacuum_statistics — extended vacuum statistics + + + ext_vacuum_statistics + + + + The ext_vacuum_statistics module provides + extended per-table, per-index, and per-database vacuum statistics + (buffer I/O, WAL, general, timing) via views in the + ext_vacuum_statistics schema. + + + + The module must be loaded by adding ext_vacuum_statistics to + in + postgresql.conf, because it registers a vacuum hook at + server startup. This means that a server restart is needed to add or remove + the module. After installation, run + CREATE EXTENSION ext_vacuum_statistics in each database + where you want to use it. + + + + Note that starting the server without the module in + , even once, discards + all cumulative statistics, not only those of this + module. The statistics file then holds entries of a statistics kind that + is not registered, so the server reports the file as corrupted and resets + every statistics kind, built-in ones such as those shown in + pg_stat_user_tables included. To stop collecting + vacuum statistics without that side effect, keep the module loaded and + set vacuum_statistics.enabled to off + instead. + + + + When active, the module provides views + ext_vacuum_statistics.pg_stats_vacuum_tables, + ext_vacuum_statistics.pg_stats_vacuum_indexes, and + ext_vacuum_statistics.pg_stats_vacuum_database, + plus functions to reset statistics and manage tracking. + + + + Each tracked object (one table or one index) uses a fixed-size shared + memory entry; the exact size depends on the platform. + + + + The statistics of a table or an index are dropped together with it, when + the transaction that drops it commits, and a new table or index starts + from empty statistics even if it gets the OID of an old one; both work + the way they do for the built-in relation statistics. The module does + that with an object_access_hook, so the statistics file + does not accumulate the entries of relations long gone. The statistics + of a dropped database are dropped along with the rest of its cumulative + statistics. + + + + Local Counters + + The table and index views identify entries by relid + or indexrelid, together with the schema, relation + name and database name. They include materialized views and TOAST relations. + Counters accumulate over completed vacuum work on the current instance. + Heap VACUUM FULL does not update these counters. + + + + total_blks_read, total_blks_hit, + total_blks_dirtied, total_blks_written + bigint buffer-manager activity during vacuum. + Table-only totals exclude index work, which is reported separately. + + + rel_blks_read, rel_blks_hit + bigint reads and buffer hits attributed to the + relation itself, excluding accesses to other relations. + + + wal_records, wal_fpi, + wal_bytes + WAL records and full-page images (bigint) and + WAL bytes (numeric) generated during vacuum. + + + blk_read_time, blk_write_time + double precision block I/O time in milliseconds, + available when is enabled. + + + tuples_deleted, pages_scanned, + pages_removed, pages_deleted + bigint tuples removed from a table or index, + table pages scanned and removed, and newly deleted index pages. + Repeated index passes count only their additional deletion work. + + + + The database view identifies entries by dboid + and dbname. Its resource totals sum table-only + and index reports. Columns use the db_ prefix: + db_blks_read, db_blks_hit, + db_blks_dirtied, db_blks_written, + db_wal_records, db_wal_fpi, + db_wal_bytes, db_blk_read_time and + db_blk_write_time. + + + The views read the corresponding + ext_vacuum_statistics.pg_stats_get_vacuum_tables(dboid, reloid), + ext_vacuum_statistics.pg_stats_get_vacuum_indexes(dboid, reloid) + and ext_vacuum_statistics.pg_stats_get_vacuum_database(dboid) + functions. Missing statistics produce no rows. Collection and resets are + independent of the native VACUUM work statistics in the core. + + + + + Functions + + + Reset functions require superuser privileges by default. Access can be + delegated with GRANT EXECUTE. Existing entries remain + visible with zero counters after a reset; their table, index, or database + type is preserved. Missing entries are not created by a reset. + pg_stat_reset() also resets the module's counters for + the current database. + + + + + + ext_vacuum_statistics.vacuum_statistics_reset() + void + + + + Resets all vacuum statistics of the module, those of relations and of + databases. + + + + + + + ext_vacuum_statistics.extvac_reset_entry(dboid oid, relid oid) + void + + + + Resets the vacuum statistics of one table or index. Use + 0 as dboid for a shared + catalog. The database aggregate and other relations are unchanged. + + + + + + + ext_vacuum_statistics.extvac_reset_db_entry(dboid oid) + void + + + + Resets the vacuum statistics of a database: its aggregate and the + statistics of all its tables and indexes. + + + + + + + + Configuration Parameters + + + + vacuum_statistics.enabled (boolean) + + + Enables extended vacuum statistics collection. Default: on. + + + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 4873c85b5f8..e8731e9f29a 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -129,6 +129,7 @@ + diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 3e1297e6844..ffd3d27e413 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -399,6 +399,17 @@ static void accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) { extVacStats->type = PGSTAT_EXTVAC_TABLE; + extVacStats->table.pages_scanned = vacrel->scanned_pages; + extVacStats->table.pages_removed = vacrel->removed_pages; + extVacStats->table.pages_frozen = vacrel->frozen_pages; + extVacStats->table.pages_all_visible = vacrel->all_visible_pages; + extVacStats->common.tuples_deleted = vacrel->tuples_deleted; + extVacStats->table.tuples_frozen = vacrel->tuples_frozen; + extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; + extVacStats->table.missed_dead_tuples = vacrel->missed_dead_tuples; + extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; + extVacStats->dead_pages = vacrel->dead_pages; + extVacStats->table.freeze_age_vacuum_count = vacrel->freeze_age_vacuum ? 1 : 0; /* * Subtract the resource usage of the index passes this process ran: they @@ -794,17 +805,28 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, extvac_stats_end(rel, extVacCounters, &extVacReport->common); accumulate_heap_vacuum_statistics(vacrel, extVacReport); + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + VacuumFailsafeActive, + extVacReport); pfree(extVacCounters); pfree(vacrel->extVacReportIdx); } - pgstat_report_vacuum(RelationGetRelid(rel), - rel->rd_rel->relisshared, - Max(vacrel->new_live_tuples, 0), - vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples, - starttime, - (PgStat_Counter) rint(VacuumDelayTime - startdelaytime), - VacuumFailsafeActive); + else + pgstat_report_vacuum_ext(rel, + Max(vacrel->new_live_tuples, 0), + vacrel->recently_dead_tuples + + vacrel->missed_dead_tuples, + starttime, + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + VacuumFailsafeActive, + NULL); pgstat_progress_end_command(); if (instrument) @@ -2993,6 +3015,16 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, extvac_stats_end(indrel, extVacCounters, &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; + if (istat != NULL) + { + extVacReport->common.tuples_deleted = + istat->tuples_removed - prev_tuples_removed; + extVacReport->pages_deleted = + (istat->pages_newly_deleted >= prev_pages_newly_deleted) ? + istat->pages_newly_deleted - prev_pages_newly_deleted : + istat->pages_newly_deleted; + } + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); } @@ -3078,6 +3110,18 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, extvac_stats_end(indrel, extVacCounters, &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; + if (istat != NULL) + { + extVacReport->common.tuples_deleted = + istat->tuples_removed - prev_tuples_removed; + extVacReport->pages_deleted = + (istat->pages_newly_deleted >= prev_pages_newly_deleted) ? + istat->pages_newly_deleted - prev_pages_newly_deleted : + istat->pages_newly_deleted; + if (istat->pages_deleted > istat->pages_free) + extVacReport->dead_pages = istat->pages_deleted - istat->pages_free; + } + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); } diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 2bf5c71a71f..8c26c8d38dc 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -173,7 +173,8 @@ static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, bool subtract); static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats); + IndexBulkDeleteResult *result, + AOVacuumRelStats *vacrelstats, bool final_cleanup); static void ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats); @@ -597,6 +598,7 @@ ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, /* * Add the resource usage counters of src to dst, or subtract them from it. + * The tuple counter is left alone. */ static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, @@ -639,12 +641,18 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, */ static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats) + IndexBulkDeleteResult *result, + AOVacuumRelStats *vacrelstats, bool final_cleanup) { PgStat_VacuumRelationCounts *report = &counters->report; extvac_stats_end(indrel, counters, &report->common); report->type = PGSTAT_EXTVAC_INDEX; + report->common.tuples_deleted = (int64) result->tuples_removed; + report->pages_deleted = result->pages_newly_deleted; + if (final_cleanup && result->pages_deleted > result->pages_free) + report->dead_pages = result->pages_deleted - result->pages_free; + ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); pfree(counters); @@ -653,6 +661,12 @@ ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, /* * Report the vacuum of an append-optimized table, all of its phases, to * set_report_vacuum_hook. + * + * The counters that mean something for append-optimized storage are set: + * the tuples discarded, moved or left hidden, the segments actually + * compacted, the segment count at completion, and the space scanned or + * released in heap-equivalent pages. Per-tuple freezing, heap pruning and + * heap visibility-map counters stay zero. */ static void ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) @@ -663,6 +677,15 @@ ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) report.type = PGSTAT_EXTVAC_TABLE; report.common = vacrelstats->extstats->phases; ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); + report.common.tuples_deleted = vacrelstats->num_dead_tuples; + report.table.recently_dead_tuples = vacrelstats->dead_tuples; + report.table.pages_scanned = vacrelstats->pages_scanned; + report.table.total_file_segs = vacrelstats->total_file_segs; + report.table.compacted_segments = vacrelstats->compacted_segments; + report.table.tuples_moved = vacrelstats->tuples_moved; + report.table.pages_removed = + vacrelstats->nbytes_truncated / BLCKSZ + + (vacrelstats->nbytes_truncated % BLCKSZ != 0); } @@ -786,8 +809,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); if (set_report_vacuum_hook) - ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + ao_measure_index_resources(Irel[i], extcounters, &result, + vacrelstats, final_cleanup); } } else @@ -810,8 +833,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); if (set_report_vacuum_hook) - ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + ao_measure_index_resources(Irel[i], extcounters, &result, + vacrelstats, final_cleanup); } } } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 9866a34e74c..6375243a158 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -903,6 +903,14 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, extvac_stats_end(indrel, extVacCounters, &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; + if (istat_res != NULL) + { + extVacReport->common.tuples_deleted = + istat_res->tuples_removed - prev_tuples_removed; + extVacReport->pages_deleted = + istat_res->pages_newly_deleted - prev_pages_newly_deleted; + } + pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); pfree(extVacCounters); } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index e163b71e6f7..8d6b8ea9057 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -429,6 +429,30 @@ pgstat_report_vacuum_stats(Relation rel, const PgStat_VacuumStats *stats) */ set_report_vacuum_hook_type set_report_vacuum_hook = NULL; +/* + * Report extended vacuum statistics to extensions via set_report_vacuum_hook. + * When livetuples/deadtuples/starttime are provided (heap case), also calls + * pgstat_report_vacuum. For indexes, pass -1, -1, 0, 0 to skip pgstat_report_vacuum. + */ +void +pgstat_report_vacuum_ext(Relation rel, PgStat_Counter livetuples, + PgStat_Counter deadtuples, TimestampTz starttime, + PgStat_Counter delaytime, bool failsafe, + PgStat_VacuumRelationCounts * extstats) +{ + /* Index reports pass starttime = 0: no per-vacuum pgstat side-effects */ + if (starttime != 0) + pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, + livetuples, deadtuples, starttime, delaytime, + failsafe); + + if (extstats != NULL && set_report_vacuum_hook) + (*set_report_vacuum_hook) (RelationGetRelid(rel), + rel->rd_rel->relisshared, + extstats); +} + + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 52281ca88e9..012e141617c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -142,12 +142,34 @@ typedef struct PgStat_CommonCounts int64 wal_records; int64 wal_fpi; uint64 wal_bytes; + int64 tuples_deleted; } PgStat_CommonCounts; typedef struct PgStat_VacuumRelationCounts { PgStat_CommonCounts common; ExtVacReportType type; + int64 dead_pages; /* remaining dead pages observed at each vacuum */ + union + { + struct + { + int64 tuples_frozen; + int64 recently_dead_tuples; + int64 missed_dead_tuples; + int64 pages_scanned; + int64 pages_removed; + int64 missed_dead_pages; + int64 pages_frozen; + int64 pages_all_visible; + int64 freeze_age_vacuum_count; + /* AO-only: last segment count and cumulative compaction work. */ + int64 total_file_segs; + int64 compacted_segments; + int64 tuples_moved; + } table; + int64 pages_deleted; /* index-only counter */ + }; } PgStat_VacuumRelationCounts; typedef struct PgStat_FunctionCallUsage @@ -750,6 +772,13 @@ extern void pgstat_report_index_vacuum_time(Relation rel, bool is_autovacuum); extern void pgstat_count_vacuum_error(bool shared); +extern void pgstat_report_vacuum_ext(Relation rel, + PgStat_Counter livetuples, + PgStat_Counter deadtuples, + TimestampTz starttime, + PgStat_Counter delaytime, + bool failsafe, + PgStat_VacuumRelationCounts * extstats); /* Hook for extensions to receive extended vacuum statistics */ typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, From 36fb0ad57a5a552cbe1bf10df6f7d1b75d4ccedc Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 17:38:36 +0300 Subject: [PATCH 43/50] Keep segment OID-reuse statistics warnings in the server log Distributed CREATE can encounter the same existing statistics entry on every QE. Use LOG_SERVER_ONLY on executors while retaining WARNING on the coordinator and utility connections. The counter reset is unchanged. Validated OID reuse on a coordinator and three primary segments: one client warning, executor diagnostics only in server logs, and counters reset on every instance. --- src/backend/utils/activity/pgstat_xact.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/activity/pgstat_xact.c b/src/backend/utils/activity/pgstat_xact.c index 110ddca3c91..8ec1b1ae05f 100644 --- a/src/backend/utils/activity/pgstat_xact.c +++ b/src/backend/utils/activity/pgstat_xact.c @@ -14,6 +14,7 @@ #include "access/transam.h" #include "access/xact.h" +#include "cdb/cdbvars.h" #include "pgstat.h" #include "utils/memutils.h" #include "utils/pgstat_internal.h" @@ -369,7 +370,13 @@ pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid) { if (pgstat_get_entry_ref(kind, dboid, objoid, false, NULL)) { - ereport(WARNING, + /* + * Distributed CREATE can reuse the same OID on every segment. + * Keep segment diagnostics in the server log instead of forwarding + * a warning from each QE to the client. Direct utility connections + * and the coordinator retain the warning. + */ + ereport(Gp_role == GP_ROLE_EXECUTE ? LOG_SERVER_ONLY : WARNING, errmsg("resetting existing statistics for kind %s, db=%u, oid=%u", (pgstat_get_kind_info(kind))->name, dboid, objoid)); From b540641e8f83f8adac9a20bac6d3a379283fdfc7 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 17:38:47 +0300 Subject: [PATCH 44/50] Clarify local vacuum statistics reset scope and memory estimates Document that the original reset functions affect only the connected node, including when called on QD, and expose that scope in SQL function comments. Correct the interaction with pg_stat_reset and remove the obsolete fixed entry-size estimate. Validation: documentation markup and git diff --check. --- contrib/ext_vacuum_statistics/README.md | 13 ++++++++--- .../ext_vacuum_statistics--1.0.sql | 7 ++++++ doc/src/sgml/extvacuumstatistics.sgml | 22 +++++++++++++------ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index b3bbb4cb131..1461110f05d 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -50,12 +50,16 @@ Example output: mytable | 120 | 340 | 15 | 500 | 10 ``` -Reset statistics when needed: +Reset statistics only on the current node: ```sql SELECT ext_vacuum_statistics.vacuum_statistics_reset(); ``` +The reset functions act only on the current node. Calling one on the +coordinator (QD) leaves segment counters unchanged. This also applies to +`extvac_reset_entry()` and `extvac_reset_db_entry()`. + ## Configuration (GUCs) | GUC | Default | Description | @@ -66,7 +70,9 @@ SELECT ext_vacuum_statistics.vacuum_statistics_reset(); Each tracked object (table or index) uses a fixed-size shared memory entry; the exact size depends on the platform. -Example: a database with 1000 tables and 2000 indexes, all tracked, uses about **700 KB** on Ubuntu (3001 entries × 232 bytes). Per-database entries add one entry per tracked database. +Per-database aggregates add one entry per tracked database. Entry size includes +pgstat bookkeeping; memory estimates must use the structures of the actual +build rather than a fixed byte count from an earlier version. The entry of a table or an index is dropped when the relation is dropped (at commit, so a rolled back `DROP` keeps it), and a new relation that gets the OID @@ -105,4 +111,5 @@ and `pg_stat_vacuum` (and their `gp_stat_*` counterparts) for native counters. They obey `track_counts` and ordinary `pg_stat_reset*` functions. The extension retains its existing counters for compatibility and adds resource measurements such as buffers and WAL. Its counters obey `vacuum_statistics.enabled` and -its own reset functions; resetting one collection does not reset the other. +its own reset functions. Extension-specific resets do not clear native counters; +`pg_stat_reset()` also clears the extension's entries for the current database. diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index f43067992ba..ec5288559e5 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -50,6 +50,13 @@ RETURNS void AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' LANGUAGE C STRICT VOLATILE PARALLEL UNSAFE; +COMMENT ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) IS + 'Reset vacuum statistics for one table or index on the connected instance only'; +COMMENT ON FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(oid) IS + 'Reset vacuum statistics for a database and its relations on the connected instance only'; +COMMENT ON FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() IS + 'Reset vacuum statistics for all databases on the connected instance only'; + -- Reset privileges can be delegated explicitly, as for pg_stat_reset(). REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(oid) FROM PUBLIC; diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index 6cdc0678f27..d986d8bc857 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -116,8 +116,9 @@ ext_vacuum_statistics.pg_stats_get_vacuum_tables(dboid, reloid), ext_vacuum_statistics.pg_stats_get_vacuum_indexes(dboid, reloid) and ext_vacuum_statistics.pg_stats_get_vacuum_database(dboid) - functions. Missing statistics produce no rows. Collection and resets are - independent of the native VACUUM work statistics in the core. + functions. Missing statistics produce no rows. Extension-specific resets + do not clear native VACUUM work counters. pg_stat_reset() + also clears the extension's counters for the current database. @@ -133,6 +134,12 @@ the current database. + + Reset functions operate only on the current node. Calling one on the + coordinator (QD) leaves the segment counters unchanged. They can also be + used in a utility connection to an individual segment. + + @@ -141,8 +148,8 @@ - Resets all vacuum statistics of the module, those of relations and of - databases. + Resets all vacuum statistics of the module only on the current node, + including relation and database statistics for all databases. @@ -154,7 +161,8 @@ - Resets the vacuum statistics of one table or index. Use + Resets the vacuum statistics of one table or index only on the current + node. Use 0 as dboid for a shared catalog. The database aggregate and other relations are unchanged. @@ -168,8 +176,8 @@ - Resets the vacuum statistics of a database: its aggregate and the - statistics of all its tables and indexes. + Resets the vacuum statistics of a database only on the current node: + its aggregate and the statistics of all its tables and indexes. From 46a3f04ddbb1c82fb950763557d4e8536409eaf4 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 19:09:25 +0300 Subject: [PATCH 45/50] Report core VACUUM resources under VERBOSE without an extension hook Print buffer, WAL and I/O measurements for heap work, index passes, and each AO vacuum phase. Exclude index resources from the enclosing table or phase report, include local buffer writes, and keep ordinary VACUUM quiet. Remove the extension hook declaration, definition and collection gates from the instrumentation prerequisite. The extension patch introduces that interface together with its consumer. Drop unused AO aggregate reporting from the core-only patch. Add core TAP coverage without an extension and document VERBOSE output. --- doc/src/sgml/ref/vacuum.sgml | 7 ++ src/backend/access/heap/vacuumlazy.c | 60 +++++++++++--- src/backend/commands/vacuum_ao.c | 79 +++++++++---------- src/backend/commands/vacuumparallel.c | 7 +- src/backend/utils/activity/pgstat_relation.c | 6 -- src/include/access/appendonly_compaction.h | 11 +-- src/include/commands/vacuum.h | 7 +- src/include/pgstat.h | 6 +- src/test/modules/test_misc/meson.build | 1 + .../t/029_vacuum_resource_verbose.pl | 75 ++++++++++++++++++ 10 files changed, 187 insertions(+), 72 deletions(-) create mode 100644 src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 2b85c3d385d..93d31af2543 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -140,6 +140,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ is enabled. Heap table resources + exclude the separately reported index passes. For append-optimized + tables, resources are reported separately for pre-cleanup, compaction + and post-cleanup, also excluding index work. These reports do not + require a statistics extension to be loaded. diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 5da380c2989..fcd77591fb6 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -331,16 +331,16 @@ static void restore_vacuum_error_info(LVRelState *vacrel, /* * extvac_stats_start - Allocate and snapshot extended instrumentation. * - * Return NULL without allocating anything when there is no consumer. The - * caller frees a non-NULL result after reporting the phase, or its vacuum - * memory context cleans it up on error. + * Return NULL without allocating anything unless VERBOSE is requested. + * The caller frees a non-NULL result after reporting the phase, or its + * vacuum memory context cleans it up on error. */ LVExtStatCounters * -extvac_stats_start(Relation rel) +extvac_stats_start(Relation rel, bool verbose) { LVExtStatCounters *counters; - if (set_report_vacuum_hook == NULL) + if (!verbose) return NULL; counters = palloc0(sizeof(LVExtStatCounters)); @@ -373,7 +373,7 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, report->total_blks_read = bufusage.local_blks_read + bufusage.shared_blks_read; report->total_blks_hit = bufusage.local_blks_hit + bufusage.shared_blks_hit; report->total_blks_dirtied = bufusage.local_blks_dirtied + bufusage.shared_blks_dirtied; - report->total_blks_written = bufusage.shared_blks_written; + report->total_blks_written = bufusage.local_blks_written + bufusage.shared_blks_written; /* PostgreSQL 16 keeps a single timer for shared and local blocks */ report->blk_read_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_read_time); report->blk_write_time = INSTR_TIME_GET_MILLISEC(bufusage.blk_write_time); @@ -388,6 +388,41 @@ extvac_stats_end(Relation rel, LVExtStatCounters * counters, } } +/* + * Print resource measurements for VACUUM VERBOSE. + */ +void +extvac_stats_log(Relation rel, const char *phase, const PgStat_CommonCounts *report) +{ + StringInfoData buf; + + initStringInfo(&buf); + appendStringInfo(&buf, _("vacuum resource usage for \"%s.%s\" (%s):\n"), + get_namespace_name(RelationGetNamespace(rel)), + RelationGetRelationName(rel), phase); + appendStringInfo(&buf, + _("buffer usage: %lld hits, %lld misses, %lld dirtied, %lld written\n"), + (long long) report->total_blks_hit, + (long long) report->total_blks_read, + (long long) report->total_blks_dirtied, + (long long) report->total_blks_written); + if (pgstat_track_counts) + appendStringInfo(&buf, _("relation buffer usage: %lld hits, %lld misses\n"), + (long long) report->blks_hit, + (long long) (report->blks_fetched - report->blks_hit)); + appendStringInfo(&buf, + _("WAL usage: %lld records, %lld full page images, %llu bytes\n"), + (long long) report->wal_records, + (long long) report->wal_fpi, + (unsigned long long) report->wal_bytes); + if (track_io_timing) + appendStringInfo(&buf, _("I/O timings: read: %.3f ms, write: %.3f ms\n"), + report->blk_read_time, report->blk_write_time); + + ereport(INFO, (errmsg_internal("%s", buf.data))); + pfree(buf.data); +} + /* * Build the heap-specific part of the extended vacuum report from the * counters gathered in vacrel. @@ -481,7 +516,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } - extVacCounters = extvac_stats_start(rel); + extVacCounters = extvac_stats_start(rel, verbose); pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -748,6 +783,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, extvac_stats_end(rel, extVacCounters, &extVacReport->common); accumulate_heap_vacuum_statistics(vacrel, extVacReport); + if (verbose) + extvac_stats_log(rel, _("heap (excluding indexes)"), + &extVacReport->common); pfree(extVacCounters); pfree(vacrel->extVacReportIdx); @@ -2873,7 +2911,7 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; LVExtStatCounters *extVacCounters; - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2904,6 +2942,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index bulk delete"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); @@ -2935,7 +2975,7 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, LVSavedErrInfo saved_err_info; LVExtStatCounters *extVacCounters; - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, vacrel->verbose); ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; ivinfo.analyze_only = false; @@ -2966,6 +3006,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (vacrel->verbose) + extvac_stats_log(indrel, _("index cleanup"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 38a83204700..fe61bd37760 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -161,15 +161,13 @@ static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); -static AOVacuumRelStats *init_vacrelstats(void); +static AOVacuumRelStats *init_vacrelstats(bool instrument); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, bool subtract); static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats); -static void ao_measure_table_resources(Relation rel, - AOVacuumRelStats *vacrelstats); + AOVacuumRelStats *vacrelstats, bool verbose); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -414,14 +412,14 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg } static AOVacuumRelStats * -init_vacrelstats() +init_vacrelstats(bool instrument) { AOVacuumRelStats *vacrelstats; MemoryContext old_context; old_context = MemoryContextSwitchTo(TopMemoryContext); vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); - if (set_report_vacuum_hook != NULL) + if (instrument) vacrelstats->extstats = palloc0(sizeof(AOVacuumExtStats)); MemoryContextSwitchTo(old_context); @@ -438,7 +436,8 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy { static AOVacuumRelStats *vacrelstats = NULL; LVExtStatCounters *extcounters; - bool extstats = (set_report_vacuum_hook != NULL); + bool verbose = (params->options & VACOPT_VERBOSE) != 0; + PgStat_CommonCounts index_start = {0}; Assert(RelationStorageIsAO(rel)); Assert(params != NULL); @@ -473,12 +472,14 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy } pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); - vacrelstats = init_vacrelstats(); + vacrelstats = init_vacrelstats(verbose); vacrelstats->relid = RelationGetRelid(rel); } /* Sample the resource usage of the phase for the extended statistics */ - extcounters = extvac_stats_start(rel); + extcounters = extvac_stats_start(rel, verbose); + if (verbose) + index_start = vacrelstats->extstats->indexes; /* * Do the actual work --- either FULL or "lazy" vacuum @@ -493,17 +494,27 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); - if (extstats && ao_vacuum_phase != 0) + if (verbose && ao_vacuum_phase != 0) { extvac_stats_end(rel, extcounters, &extcounters->report.common); - ao_accum_resources(&vacrelstats->extstats->phases, - &extcounters->report.common, false); + if (Gp_role != GP_ROLE_DISPATCH) + { + PgStat_CommonCounts phase_usage = extcounters->report.common; + PgStat_CommonCounts index_usage = vacrelstats->extstats->indexes; + const char *phase_name = + ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE ? _("AO pre-cleanup (excluding indexes)") : + ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE ? _("AO compaction (excluding indexes)") : + _("AO post-cleanup (excluding indexes)"); + + /* Only index passes executed during this phase are excluded. */ + ao_accum_resources(&index_usage, &index_start, true); + ao_accum_resources(&phase_usage, &index_usage, true); + extvac_stats_log(rel, phase_name, &phase_usage); + } } if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) { - if (extstats) - ao_measure_table_resources(rel, vacrelstats); pgstat_progress_end_command(); cleanup_vacrelstats(&vacrelstats); } @@ -549,40 +560,24 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, } /* - * Report one pass over an index of an append-optimized table to - * set_report_vacuum_hook, as lazy_vacuum_one_index() does for the heap, and - * remember its resource usage, which the table's report must not count - * again. + * Log one index pass and remember its resource usage so the enclosing AO + * phase's VERBOSE report does not count the same work again. */ static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - AOVacuumRelStats *vacrelstats) + AOVacuumRelStats *vacrelstats, bool verbose) { PgStat_VacuumRelationCounts *report = &counters->report; extvac_stats_end(indrel, counters, &report->common); + if (verbose) + extvac_stats_log(indrel, _("AO index vacuum"), &report->common); report->type = PGSTAT_EXTVAC_INDEX; ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); pfree(counters); } -/* - * Report the vacuum of an append-optimized table, all of its phases, to - * set_report_vacuum_hook. - */ -static void -ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) -{ - PgStat_VacuumRelationCounts report; - - memset(&report, 0, sizeof(report)); - report.type = PGSTAT_EXTVAC_TABLE; - report.common = vacrelstats->extstats->phases; - ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); - -} - /* * Recycling AWAITING_DROP segments. */ @@ -689,14 +684,15 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { LVExtStatCounters *extcounters; - extcounters = extvac_stats_start(Irel[i]); + extcounters = extvac_stats_start(Irel[i], elevel == INFO); scan_index(Irel[i], aoRelation, elevel, bstrategy); - if (set_report_vacuum_hook) + if (extcounters != NULL) ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } else @@ -705,16 +701,17 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { LVExtStatCounters *extcounters; - extcounters = extvac_stats_start(Irel[i]); + extcounters = extvac_stats_start(Irel[i], elevel == INFO); vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); - if (set_report_vacuum_hook) + if (extcounters != NULL) ao_measure_index_resources(Irel[i], extcounters, - vacrelstats); + vacrelstats, + elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 9ec6d8fdd31..0f66ea6b825 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -845,7 +845,7 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, if (indstats->istat_updated) istat = &(indstats->istat); - extVacCounters = extvac_stats_start(indrel); + extVacCounters = extvac_stats_start(indrel, pvs->shared->elevel == INFO); ivinfo.index = indrel; ivinfo.heaprel = pvs->heaprel; ivinfo.analyze_only = false; @@ -878,6 +878,11 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, PgStat_VacuumRelationCounts *extVacReport = &extVacCounters->report; extvac_stats_end(indrel, extVacCounters, &extVacReport->common); + if (pvs->shared->elevel == INFO) + extvac_stats_log(indrel, + indstats->status == PARALLEL_INDVAC_STATUS_NEED_CLEANUP ? + _("index cleanup") : _("index bulk delete"), + &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; pfree(extVacCounters); } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 41b528e6ccf..45bbf9abcb7 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -273,12 +273,6 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } -/* - * Hook for extensions to receive extended vacuum statistics. - * NULL when no extension has registered. - */ -set_report_vacuum_hook_type set_report_vacuum_hook = NULL; - /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 9320a6131b2..92111c89005 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -22,10 +22,9 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) -/* Optional resource counters for the extended-statistics hook. */ +/* Optional resource counters for VACUUM VERBOSE. */ typedef struct AOVacuumExtStats { - PgStat_CommonCounts phases; PgStat_CommonCounts indexes; } AOVacuumExtStats; @@ -41,12 +40,8 @@ typedef struct AOVacuumRelStats int num_index_vacuumed; /* current # of indexes been vacuumed */ /* the relation these stats were started for */ Oid relid; - /* - * Resource usage for set_report_vacuum_hook, accumulated over the phases: - * of the phases as a whole, and of the index passes among them, which are - * reported per index and subtracted from the table's report. - */ - AOVacuumExtStats *extstats; /* allocated only when the hook is installed */ + /* Index resources are excluded from each phase's VERBOSE report. */ + AOVacuumExtStats *extstats; /* allocated only for VERBOSE */ } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 1dba80b2d84..f19205c584a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -442,7 +442,7 @@ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; extern PGDLLIMPORT pg_atomic_uint32 *VacuumActiveNWorkers; extern PGDLLIMPORT int VacuumCostBalanceLocal; -/* Allocated only when a hook consumes the extended statistics. */ +/* Allocated only for VERBOSE. */ typedef struct LVExtStatCounters { WalUsage walusage; @@ -452,10 +452,13 @@ typedef struct LVExtStatCounters PgStat_VacuumRelationCounts report; } LVExtStatCounters; -extern LVExtStatCounters *extvac_stats_start(Relation rel); +extern LVExtStatCounters *extvac_stats_start(Relation rel, bool verbose); extern void extvac_stats_end(Relation rel, LVExtStatCounters *counters, PgStat_CommonCounts *report); +extern void extvac_stats_log(Relation rel, const char *phase, + const PgStat_CommonCounts *report); + extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index d0e43e30b8d..a23eaf95d6a 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -118,7 +118,7 @@ typedef struct PgStat_FunctionCounts * Working state needed to accumulate per-function-call timing statistics. */ /* - * Extended vacuum statistics - passed to extensions via set_report_vacuum_hook. + * Resource measurements collected during VACUUM. * Type of entry: table (heap), index, or database aggregate. */ typedef enum ExtVacReportType @@ -685,10 +685,6 @@ extern void pgstat_unlink_relation(Relation rel); extern void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples); -/* Hook for extensions to receive extended vacuum statistics */ -typedef void (*set_report_vacuum_hook_type) (Oid tableoid, bool shared, - PgStat_VacuumRelationCounts * params); -extern PGDLLIMPORT set_report_vacuum_hook_type set_report_vacuum_hook; extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..51c8a091119 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,7 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/029_vacuum_resource_verbose.pl', ], }, } diff --git a/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl new file mode 100644 index 00000000000..94e5a413553 --- /dev/null +++ b/src/test/modules/test_misc/t/029_vacuum_resource_verbose.pl @@ -0,0 +1,75 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Resource instrumentation has a core consumer even without an extension. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacuum_resource_verbose'); +$node->init; +$node->append_conf('postgresql.conf', q{ +autovacuum = off +track_io_timing = on +gp_appendonly_compaction_threshold = 1 +}); +$node->start; +is($node->safe_psql('postgres', 'SHOW shared_preload_libraries'), '', + 'VERBOSE resource reporting requires no preloaded extension'); + +for my $storage ('heap', 'row', 'column') +{ + my $table = "verbose_$storage"; + my $options = $storage eq 'heap' ? '' : + "WITH (appendonly = true, orientation = $storage)"; + $node->safe_psql('postgres', qq{ +CREATE TABLE $table (id int, padding text) $options; +CREATE INDEX ${table}_idx ON $table (id); +INSERT INTO $table SELECT g, repeat('x', 100) FROM generate_series(1, 4000) g; +DELETE FROM $table WHERE id <= 1000; +}); + my ($stdout, $stderr); + is($node->psql('postgres', "VACUUM $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage plain vacuum succeeds"); + unlike($stderr, qr/vacuum resource usage for/, + "$storage plain vacuum does not emit resource reports"); + $node->safe_psql('postgres', "DELETE FROM $table WHERE id <= 2000"); + is($node->psql('postgres', "VACUUM (VERBOSE, INDEX_CLEANUP ON) $table", + stdout => \$stdout, stderr => \$stderr), 0, + "$storage verbose vacuum succeeds"); + my @phases = $storage eq 'heap' ? ('heap (excluding indexes)') : + ('AO pre-cleanup (excluding indexes)', 'AO compaction (excluding indexes)', + 'AO post-cleanup (excluding indexes)'); + for my $phase (@phases) + { + my ($report) = $stderr =~ + /vacuum resource usage for "public\.\Q$table\E" \(\Q$phase\E\):\n(.*?)(?=\n(?:INFO|CONTEXT|WARNING):|\z)/s; + ok(defined $report, "$storage reports $phase without an extension"); + $report //= ''; + like($report, qr/buffer usage: \d+ hits, \d+ misses, \d+ dirtied, \d+ written/, + "$phase reports nonnegative buffer counters"); + like($report, qr/WAL usage: \d+ records, \d+ full page images, \d+ bytes/, + "$phase reports WAL counters"); + like($report, qr/I\/O timings: read: \d+\.\d+ ms, write: \d+\.\d+ ms/, + "$phase reports I/O times in milliseconds"); + like($report, qr/WAL usage: [1-9]\d* records/, + 'compaction produces WAL') if $phase eq 'AO compaction (excluding indexes)'; + } + my $index_phase = $storage eq 'heap' ? 'index bulk delete' : 'AO index vacuum'; + like($stderr, qr/vacuum resource usage for "public\.\Q${table}_idx\E" \(\Q$index_phase\E\):/, + "$storage reports index resources separately"); +} + +# Timing output is conditional, while collection of buffer/WAL counters is not. +my ($stdout, $stderr); +is($node->psql('postgres', + 'SET track_io_timing = off; VACUUM (VERBOSE) verbose_heap', + stdout => \$stdout, stderr => \$stderr), 0, 'verbose vacuum with I/O timing disabled'); +like($stderr, qr/vacuum resource usage for "public\.verbose_heap"/, + 'resource report remains available without I/O timing'); +unlike($stderr, qr/I\/O timings:/, 'disabled I/O timing is not printed'); + +$node->stop; +done_testing(); From d8c19f53aa8bdfd3e38d2934977291484dc448f9 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 19:36:50 +0300 Subject: [PATCH 46/50] Fetch both visibility-map counts in one segment query ANALYZE reads relallvisible and relallfrozen together instead of dispatching two queries. Preserve replicated-table normalization and local visibility-map counting. Check both coordinator catalog counts against the segment totals in the distributed and replicated regression cases. --- src/backend/commands/analyze.c | 42 +++++++++++++++++----- src/test/regress/expected/vacuum_stats.out | 8 +++++ src/test/regress/sql/vacuum_stats.sql | 8 +++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 08110428258..1059586cede 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -2444,18 +2444,42 @@ AcquireVisibilityMapCounts(Relation onerel, BlockNumber *all_visible, if (Gp_role == GP_ROLE_DISPATCH && onerel->rd_cdbpolicy && !GpPolicyIsEntry(onerel->rd_cdbpolicy)) { - char relvm_sql[100]; - int64 visible; - int64 frozen; + char relvm_sql[128]; + int64 visible = 0; + int64 frozen = 0; + CdbPgResults cdb_pgresults = {NULL, 0}; + /* Read both counters from the same catalog row in one dispatch. */ snprintf(relvm_sql, sizeof(relvm_sql), - "select relallvisible from pg_catalog.pg_class where oid = %u", + "select relallvisible, relallfrozen from pg_catalog.pg_class where oid = %u", RelationGetRelid(onerel)); - visible = get_size_from_segDBs(relvm_sql); - snprintf(relvm_sql, sizeof(relvm_sql), - "select relallfrozen from pg_catalog.pg_class where oid = %u", - RelationGetRelid(onerel)); - frozen = get_size_from_segDBs(relvm_sql); + CdbDispatchCommand(relvm_sql, DF_WITH_SNAPSHOT, &cdb_pgresults); + PG_TRY(); + { + for (int i = 0; i < cdb_pgresults.numResults; i++) + { + PGresult *result = cdb_pgresults.pg_results[i]; + ExecStatusType status = PQresultStatus(result); + int ntuples = PQntuples(result); + int nfields = PQnfields(result); + + if (status != PGRES_TUPLES_OK) + elog(ERROR, "unexpected result from segment: %d", status); + if (ntuples != 1 || nfields != 2) + elog(ERROR, "unexpected shape of visibility-map result from segment (%d rows, %d cols)", + ntuples, nfields); + if (PQgetisnull(result, 0, 0) || PQgetisnull(result, 0, 1)) + elog(ERROR, "unexpected NULL visibility-map count from segment"); + + visible += pg_strtoint64(PQgetvalue(result, 0, 0)); + frozen += pg_strtoint64(PQgetvalue(result, 0, 1)); + } + } + PG_FINALLY(); + { + cdbdisp_clearCdbPgResults(&cdb_pgresults); + } + PG_END_TRY(); if (GpPolicyIsReplicated(onerel->rd_cdbpolicy)) { diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index 8cb79eaae45..ea677f0c3d6 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -37,6 +37,8 @@ VACUUM (FREEZE) frozen_stats_dist; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; frozen_stats_match @@ -49,6 +51,8 @@ ANALYZE frozen_stats_dist; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; frozen_stats_match @@ -73,6 +77,8 @@ VACUUM (FREEZE) frozen_stats_repl; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; frozen_stats_match @@ -85,6 +91,8 @@ ANALYZE frozen_stats_repl; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; frozen_stats_match diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index 99cf075cc84..084ff4d0dbf 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -29,6 +29,8 @@ VACUUM (FREEZE) frozen_stats_dist; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; UPDATE frozen_stats_dist SET a = a + 10000 WHERE a = 1; @@ -36,6 +38,8 @@ ANALYZE frozen_stats_dist; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; VACUUM (FULL) frozen_stats_dist; @@ -51,6 +55,8 @@ VACUUM (FREEZE) frozen_stats_repl; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; UPDATE frozen_stats_repl SET a = a + 10000 WHERE a = 1; @@ -58,6 +64,8 @@ ANALYZE frozen_stats_repl; SELECT relallfrozen > 0 AND relallfrozen <= relallvisible AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) AS frozen_stats_match FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; VACUUM (FULL) frozen_stats_repl; From ddbf2f3e005e473cacd754f4eee18884ddd9fe93 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 20:15:34 +0300 Subject: [PATCH 47/50] Add a native reset for VACUUM work counters Reset work for one relation or the current database without clearing timing, invocation counts, other native counters, or extension metrics. Add a privileged gp_stat_reset_vacuum_stats wrapper for all primary segments and cover scope, permissions, pending work, and restart behavior. --- doc/src/sgml/monitoring.sgml | 38 +++++ src/backend/catalog/system_functions.sql | 7 + src/backend/catalog/system_views.sql | 9 ++ src/backend/utils/activity/pgstat_shmem.c | 72 ++++++++++ src/backend/utils/adt/pgstatfuncs.c | 19 +++ src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_proc.dat | 5 + src/include/pgstat.h | 1 + .../test_misc/t/030_vacuum_work_reset.pl | 132 ++++++++++++++++++ src/test/regress/expected/vacuum_stats.out | 82 +++++++++++ src/test/regress/sql/vacuum_stats.sql | 37 +++++ 11 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 src/test/modules/test_misc/t/030_vacuum_work_reset.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 78f47575cef..200dd58f6fa 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4808,6 +4808,14 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i compatibility counters and resource statistics in ext_vacuum_statistics. + + pg_stat_reset_vacuum_stats() resets just native work + counters, preserving other cumulative statistics. Supply a relation OID + to reset one table or index, or omit it to reset the current database and + its non-shared relations. On Cloudberry, the corresponding + gp_stat_reset_vacuum_stats() wrapper applies the same + reset on the coordinator and every primary segment. + @@ -6060,6 +6068,36 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + pg_stat_reset_vacuum_stats + pg_stat_reset_vacuum_stats ( + relid oid DEFAULT NULL ) + void + + + Resets only native VACUUM work counters for one table or index, including + a shared catalog. With no argument, or NULL, resets + those counters for the current database and all its non-shared relations. + A relation reset leaves its database aggregate and indexes unchanged. + Missing relation entries are not created. OID zero is a no-op. + + + Timing, last-vacuum timestamps, vacuum and analyze invocation counts, + other native counters, and extension resource statistics are preserved. + Cached snapshots follow the ordinary statistics reset rules; concurrently + running backends can subsequently publish pending work. + + + The function acts only on the connected instance. From a Cloudberry + coordinator, use gp_stat_reset_vacuum_stats with + the same argument to reset the coordinator and every primary segment. + Both functions require superuser privileges by default; access can be + delegated with GRANT EXECUTE. A delegated cluster + wrapper also requires access to its local function. + + + diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 39c4c309a0b..89222ff88d6 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -659,6 +659,13 @@ REVOKE EXECUTE ON FUNCTION pg_current_logfile(text) FROM public; REVOKE EXECUTE ON FUNCTION pg_promote(boolean, integer) FROM public; +CREATE OR REPLACE FUNCTION pg_stat_reset_vacuum_stats(relid oid DEFAULT NULL) +RETURNS void +LANGUAGE internal VOLATILE PARALLEL UNSAFE +AS 'pg_stat_reset_vacuum_stats'; + +REVOKE EXECUTE ON FUNCTION pg_stat_reset_vacuum_stats(oid) FROM public; + REVOKE EXECUTE ON FUNCTION pg_stat_reset() FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_shared(text) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ca40335c5f8..4a3f8852609 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1935,6 +1935,15 @@ $$ SELECT pg_stat_reset_shared(target) FROM gp_dist_random('gp_id'); $$ LANGUAGE SQL; +-- Reset native VACUUM work on the coordinator and every primary segment. +CREATE FUNCTION gp_stat_reset_vacuum_stats(relid oid DEFAULT NULL) RETURNS void AS +$$ + SELECT pg_stat_reset_vacuum_stats(relid) FROM gp_dist_random('gp_id'); + SELECT pg_stat_reset_vacuum_stats(relid); +$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + +REVOKE EXECUTE ON FUNCTION gp_stat_reset_vacuum_stats(oid) FROM public; + CREATE FUNCTION gp_stat_reset_single_table_counters(tableoid oid) RETURNS VOID AS $$ WITH table_info AS ( diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index 0225c794774..38d2286dcf2 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -1053,6 +1053,78 @@ pgstat_reset_matching_entries(bool (*do_reset) (PgStatShared_HashEntry *, Datum) dshash_seq_term(&hstat); } +/* Reset just the work payload while the caller holds the entry lock. */ +static void +shared_vacuum_stat_reset(PgStat_Kind kind, PgStatShared_Common *header) +{ + PgStat_VacuumStats *stats; + + Assert(kind == PGSTAT_KIND_RELATION || kind == PGSTAT_KIND_DATABASE); + if (kind == PGSTAT_KIND_RELATION) + stats = &((PgStatShared_Relation *) header)->stats.vacuum_stats; + else + stats = &((PgStatShared_Database *) header)->stats.vacuum_stats; + memset(stats, 0, sizeof(*stats)); +} + +/* + * Reset native VACUUM work for a relation, or for a database and all its + * relations when relid is InvalidOid. Shared catalogs require dboid InvalidOid + * and a valid relid; a database-wide reset leaves shared entries alone. + * Do not reset timing, vacuum/analyze counts, timestamps, other native + * counters, or custom statistics kinds. Cached snapshots follow the usual + * pgstat reset rules; concurrent backends can later flush pending work. + */ +void +pgstat_reset_vacuum_counters(Oid dboid, Oid relid) +{ + PgStat_EntryRef *entry_ref; + dshash_seq_status hstat; + PgStatShared_HashEntry *p; + + Assert(OidIsValid(dboid) || OidIsValid(relid)); + + if (OidIsValid(relid)) + { + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_RELATION, dboid, relid, + false, NULL); + if (!entry_ref || entry_ref->shared_entry->dropped) + return; + (void) pgstat_lock_entry(entry_ref, false); + shared_vacuum_stat_reset(PGSTAT_KIND_RELATION, entry_ref->shared_stats); + pgstat_unlock_entry(entry_ref); + return; + } + + /* Discard this backend's unflushed database work too. */ + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_DATABASE, dboid, InvalidOid, + false, NULL); + if (entry_ref && entry_ref->pending) + { + PgStat_StatDBEntry *pending = entry_ref->pending; + + memset(&pending->vacuum_stats, 0, sizeof(pending->vacuum_stats)); + } + + /* The hash keys are read-only; each payload has its own exclusive lock. */ + dshash_seq_init(&hstat, pgStatLocal.shared_hash, false); + while ((p = dshash_seq_next(&hstat)) != NULL) + { + PgStatShared_Common *header; + + if (p->dropped || p->key.dboid != dboid || + (p->key.kind != PGSTAT_KIND_RELATION && + p->key.kind != PGSTAT_KIND_DATABASE)) + continue; + + header = dsa_get_address(pgStatLocal.dsa, p->body); + LWLockAcquire(&header->lock, LW_EXCLUSIVE); + shared_vacuum_stat_reset(p->key.kind, header); + LWLockRelease(&header->lock); + } + dshash_seq_term(&hstat); +} + static bool match_kind(PgStatShared_HashEntry *p, Datum match_data) { diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 5fa97736091..d0105df866f 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1858,6 +1858,25 @@ pg_stat_reset_single_table_counters(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* Reset native VACUUM work, without affecting other cumulative statistics. */ +Datum +pg_stat_reset_vacuum_stats(PG_FUNCTION_ARGS) +{ + if (PG_ARGISNULL(0)) + pgstat_reset_vacuum_counters(MyDatabaseId, InvalidOid); + else + { + Oid relid = PG_GETARG_OID(0); + Oid dboid = IsSharedRelation(relid) ? InvalidOid : MyDatabaseId; + + /* OID zero is a missing relation, not a request to reset a database. */ + if (OidIsValid(relid)) + pgstat_reset_vacuum_counters(dboid, relid); + } + + PG_RETURN_VOID(); +} + Datum pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) { diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index cf6355bcc46..0e1d1e17f7f 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302610082 +#define CATALOG_VERSION_NO 302610091 #endif diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 9c170fc383c..066f4d007b6 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -13020,6 +13020,11 @@ proargnames => '{queueid,queueid,queries_submitted,queries_admitted,queries_rejected,queries_completed,elapsed_wait_secs,max_wait_secs,elapsed_exec_secs,max_exec_secs,total_cost,total_memory_kb,stat_reset_timestamp,have_stats}', prosrc => 'pg_stat_get_resqueue_stats' }, +{ oid => '9222', descr => 'statistics: reset native vacuum work counters', + proname => 'pg_stat_reset_vacuum_stats', proisstrict => 'f', provolatile => 'v', + prorettype => 'void', proargtypes => 'oid', proargnames => '{relid}', + prosrc => 'pg_stat_reset_vacuum_stats' }, + { oid => '9220', descr => 'statistics: vacuum work counters', proname => 'pg_stat_get_vacuum_stats', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'oid', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 0fe91d31d07..d77a640c0c9 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -569,6 +569,7 @@ extern void pgstat_force_next_flush(void); extern void pgstat_reset_counters(void); extern void pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid); extern void pgstat_reset_of_kind(PgStat_Kind kind); +extern void pgstat_reset_vacuum_counters(Oid dboid, Oid relid); /* stats accessors */ extern void pgstat_clear_snapshot(void); diff --git a/src/test/modules/test_misc/t/030_vacuum_work_reset.pl b/src/test/modules/test_misc/t/030_vacuum_work_reset.pl new file mode 100644 index 00000000000..a5b19ac3e1b --- /dev/null +++ b/src/test/modules/test_misc/t/030_vacuum_work_reset.pl @@ -0,0 +1,132 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# VACUUM work has a native reset that does not clear other statistics. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacuum_work_reset'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->start; +$node->safe_psql('postgres', 'CREATE DATABASE other_work'); +for my $db ('postgres', 'other_work') +{ + $node->safe_psql($db, q{ +CREATE TABLE reset_work (id int PRIMARY KEY); +INSERT INTO reset_work SELECT generate_series(1, 1000); +DELETE FROM reset_work WHERE id <= 100; +ANALYZE reset_work; +VACUUM (FREEZE) reset_work; +SELECT pg_stat_force_next_flush(); +}); +} +my $work_sql = q{ +SELECT row_to_json(s) FROM pg_stat_get_vacuum_stats('reset_work'::regclass) s +}; +my $db_work_sql = q{ +SELECT row_to_json(s) FROM pg_stat_get_vacuum_database_stats( + (SELECT oid FROM pg_database WHERE datname = current_database())) s +}; +my $other_sql = q{ +SELECT n_tup_ins, n_tup_del, n_mod_since_analyze, vacuum_count, analyze_count, + last_vacuum, last_analyze, total_vacuum_time, total_vacuum_delay_time + FROM pg_stat_all_tables WHERE relname = 'reset_work' +}; +my $zero_sql = q{ +SELECT bool_and(value::bigint = 0) + FROM pg_stat_get_vacuum_stats('reset_work'::regclass) s, + LATERAL jsonb_each_text(to_jsonb(s)) +}; +my $before = $node->safe_psql('postgres', $work_sql); +my $other_db_before = $node->safe_psql('other_work', $work_sql); +my $db_before = $node->safe_psql('postgres', $db_work_sql); +my $other_before = $node->safe_psql('postgres', $other_sql); + +$node->safe_psql('postgres', 'CREATE ROLE work_reader'); +for my $call ('pg_stat_reset_vacuum_stats()', + q[pg_stat_reset_vacuum_stats('reset_work'::regclass)], + 'gp_stat_reset_vacuum_stats()') +{ + my ($stdout, $stderr); + my $ret = $node->psql('postgres', "SET ROLE work_reader; SELECT $call", + stdout => \$stdout, stderr => \$stderr); + isnt($ret, 0, "$call denied without reset privilege"); + like($stderr, qr/permission denied for function/, 'function ACL protects reset'); +} +is($node->safe_psql('postgres', $work_sql), $before, 'denied reset preserves work'); +$node->safe_psql('postgres', q{ +SELECT pg_stat_reset_vacuum_stats(0); +SELECT pg_stat_reset_vacuum_stats(4294967294); +}); +is($node->safe_psql('postgres', $work_sql), $before, + 'missing relation and OID zero do not reset the database'); + +$node->safe_psql('postgres', q{ +GRANT EXECUTE ON FUNCTION pg_stat_reset_vacuum_stats(oid) TO work_reader; +SET ROLE work_reader; +SELECT pg_stat_reset_vacuum_stats('reset_work'::regclass); +}); +is($node->safe_psql('postgres', $zero_sql), 't', 'relation reset clears every work field'); +is($node->safe_psql('postgres', $other_sql), $other_before, + 'relation reset preserves timing, invocation counts and other statistics'); +is($node->safe_psql('postgres', $db_work_sql), $db_before, + 'relation reset preserves database work aggregate'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted FROM pg_stat_get_vacuum_stats('reset_work_pkey'::regclass) +}), '100', 'relation reset leaves index work untouched'); +is($node->safe_psql('other_work', $work_sql), $other_db_before, + 'relation reset leaves other databases untouched'); + +$node->safe_psql('postgres', q{ +SELECT pg_stat_reset_vacuum_stats('reset_work_pkey'::regclass); +VACUUM (DISABLE_PAGE_SKIPPING) reset_work; +VACUUM pg_shdepend; +SELECT pg_stat_force_next_flush(); +}); +is($node->safe_psql('postgres', q{ +SELECT pages_scanned > 0 FROM pg_stat_get_vacuum_stats('reset_work'::regclass) +}), 't', 'work accumulation resumes after reset'); +my $shared_sql = q{ +SELECT row_to_json(s) FROM pg_stat_get_vacuum_stats('pg_shdepend'::regclass) s +}; +my $shared_before = $node->safe_psql('postgres', $shared_sql); +$other_before = $node->safe_psql('postgres', $other_sql); +$node->safe_psql('postgres', 'SELECT pg_stat_reset_vacuum_stats()'); +is($node->safe_psql('postgres', $zero_sql), 't', 'database reset clears relation work'); +is($node->safe_psql('postgres', q{ +SELECT bool_and(value::bigint = 0) + FROM pg_stat_get_vacuum_database_stats( + (SELECT oid FROM pg_database WHERE datname = current_database())) s, + LATERAL jsonb_each_text(to_jsonb(s)) +}), 't', 'database reset clears every database work field'); +is($node->safe_psql('postgres', $other_sql), $other_before, + 'database reset preserves other relation statistics'); +is($node->safe_psql('postgres', $shared_sql), $shared_before, + 'database reset leaves shared catalogs untouched'); +is($node->safe_psql('other_work', $work_sql), $other_db_before, + 'database reset leaves another database untouched'); +$node->safe_psql('postgres', + q[SELECT pg_stat_reset_vacuum_stats('pg_shdepend'::regclass)]); +is($node->safe_psql('other_work', q{ +SELECT pages_scanned FROM pg_stat_get_vacuum_stats('pg_shdepend'::regclass) +}), '0', 'explicit shared-catalog reset is visible from another database'); + +# A database reset must not republish this backend's pre-reset pending work. +$node->safe_psql('postgres', q{ +VACUUM (DISABLE_PAGE_SKIPPING) reset_work; +SELECT pg_stat_reset_vacuum_stats(NULL); +SELECT pg_stat_force_next_flush(); +}); +is($node->safe_psql('postgres', q{ +SELECT pages_scanned FROM pg_stat_get_vacuum_database_stats( + (SELECT oid FROM pg_database WHERE datname = current_database())) +}), '0', 'reset discards local pending database work before a later flush'); +$node->restart; +is($node->safe_psql('postgres', $zero_sql), 't', 'work stays reset after a clean restart'); +is($node->safe_psql('other_work', $work_sql), $other_db_before, + 'untouched database history survives restart'); +$node->stop; +done_testing(); diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index fe323292d8b..d7dcba4bd73 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -80,6 +80,57 @@ SELECT tuples_deleted = 100 AS index_work_matches t (1 row) +-- Native work-only resets preserve other counters and relation scope. +CREATE TEMP TABLE work_other_before AS + SELECT relid, sum(n_tup_ins) AS n_tup_ins, sum(n_tup_del) AS n_tup_del, + sum(vacuum_count) AS vacuum_count, sum(analyze_count) AS analyze_count, + sum(total_vacuum_time) AS total_vacuum_time + FROM gp_stat_all_tables WHERE relname = 'work_stats_dist' GROUP BY relid + DISTRIBUTED BY (relid); +SELECT gp_stat_reset_vacuum_stats('work_stats_dist'::regclass); + gp_stat_reset_vacuum_stats +---------------------------- + +(1 row) + +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p') + AND bool_and(tuples_deleted = 0 AND pages_scanned = 0 + AND pages_frozen = 0 AND freeze_age_vacuum_count = 0) + AS work_reset_on_all_instances + FROM gp_stat_vacuum_tables WHERE relname = 'work_stats_dist'; + work_reset_on_all_instances +----------------------------- + t +(1 row) + +SELECT tuples_deleted = 100 AS other_relation_preserved + FROM gp_stat_vacuum_tables_summary WHERE relname = 'work_stats_repl'; + other_relation_preserved +-------------------------- + t +(1 row) + +SELECT tuples_deleted = 100 AS index_preserved + FROM gp_stat_vacuum_indexes_summary WHERE indexrelname = 'work_stats_dist_pkey'; + index_preserved +----------------- + t +(1 row) + +SELECT s.n_tup_ins = b.n_tup_ins AND s.n_tup_del = b.n_tup_del + AND s.vacuum_count = b.vacuum_count AND s.analyze_count = b.analyze_count + AND s.total_vacuum_time = b.total_vacuum_time AS other_counters_preserved + FROM (SELECT relid, sum(n_tup_ins) AS n_tup_ins, sum(n_tup_del) AS n_tup_del, + sum(vacuum_count) AS vacuum_count, sum(analyze_count) AS analyze_count, + sum(total_vacuum_time) AS total_vacuum_time + FROM gp_stat_all_tables WHERE relname = 'work_stats_dist' GROUP BY relid) s + JOIN work_other_before b USING (relid); + other_counters_preserved +-------------------------- + t +(1 row) + +DROP TABLE work_other_before; DROP TABLE work_stats_dist, work_stats_repl; CREATE TABLE work_stats_ao (a int) WITH (appendonly = true) DISTRIBUTED BY (a); CREATE TABLE work_stats_aoco (a int) WITH (appendonly = true, orientation = column) DISTRIBUTED BY (a); @@ -100,4 +151,35 @@ SELECT relname, tuples_deleted = 1000 AND tuples_moved = 1000 work_stats_aoco | t (2 rows) +-- The database form clears AO work and aggregates on all instances. +SELECT gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT gp_stat_reset_vacuum_stats(); + gp_stat_reset_vacuum_stats +---------------------------- + +(1 row) + +SELECT bool_and(tuples_deleted = 0 AND tuples_moved = 0 + AND compacted_segments = 0 AND pages_scanned = 0 + AND total_file_segs = 0) AS ao_work_reset + FROM gp_stat_vacuum_tables + WHERE relname IN ('work_stats_ao', 'work_stats_aoco'); + ao_work_reset +--------------- + t +(1 row) + +SELECT bool_and(tuples_deleted = 0 AND pages_scanned = 0 + AND pages_frozen = 0 AND compacted_segments = 0) AS database_work_reset + FROM gp_stat_vacuum WHERE datname = current_database(); + database_work_reset +--------------------- + t +(1 row) + DROP TABLE work_stats_ao, work_stats_aoco; diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index e4f0298096d..fd3d817182a 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -49,6 +49,32 @@ SELECT relname, tuples_deleted = 100 AS deleted_matches WHERE relname IN ('work_stats_dist', 'work_stats_repl') ORDER BY relname; SELECT tuples_deleted = 100 AS index_work_matches FROM gp_stat_vacuum_indexes_summary WHERE indexrelname = 'work_stats_dist_pkey'; +-- Native work-only resets preserve other counters and relation scope. +CREATE TEMP TABLE work_other_before AS + SELECT relid, sum(n_tup_ins) AS n_tup_ins, sum(n_tup_del) AS n_tup_del, + sum(vacuum_count) AS vacuum_count, sum(analyze_count) AS analyze_count, + sum(total_vacuum_time) AS total_vacuum_time + FROM gp_stat_all_tables WHERE relname = 'work_stats_dist' GROUP BY relid + DISTRIBUTED BY (relid); +SELECT gp_stat_reset_vacuum_stats('work_stats_dist'::regclass); +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p') + AND bool_and(tuples_deleted = 0 AND pages_scanned = 0 + AND pages_frozen = 0 AND freeze_age_vacuum_count = 0) + AS work_reset_on_all_instances + FROM gp_stat_vacuum_tables WHERE relname = 'work_stats_dist'; +SELECT tuples_deleted = 100 AS other_relation_preserved + FROM gp_stat_vacuum_tables_summary WHERE relname = 'work_stats_repl'; +SELECT tuples_deleted = 100 AS index_preserved + FROM gp_stat_vacuum_indexes_summary WHERE indexrelname = 'work_stats_dist_pkey'; +SELECT s.n_tup_ins = b.n_tup_ins AND s.n_tup_del = b.n_tup_del + AND s.vacuum_count = b.vacuum_count AND s.analyze_count = b.analyze_count + AND s.total_vacuum_time = b.total_vacuum_time AS other_counters_preserved + FROM (SELECT relid, sum(n_tup_ins) AS n_tup_ins, sum(n_tup_del) AS n_tup_del, + sum(vacuum_count) AS vacuum_count, sum(analyze_count) AS analyze_count, + sum(total_vacuum_time) AS total_vacuum_time + FROM gp_stat_all_tables WHERE relname = 'work_stats_dist' GROUP BY relid) s + JOIN work_other_before b USING (relid); +DROP TABLE work_other_before; DROP TABLE work_stats_dist, work_stats_repl; CREATE TABLE work_stats_ao (a int) WITH (appendonly = true) DISTRIBUTED BY (a); @@ -64,4 +90,15 @@ SELECT relname, tuples_deleted = 1000 AND tuples_moved = 1000 AND compacted_segments > 0 AND pages_scanned > 0 AS compaction_matches FROM gp_stat_vacuum_tables_summary WHERE relname IN ('work_stats_ao', 'work_stats_aoco') ORDER BY relname; +-- The database form clears AO work and aggregates on all instances. +SELECT gp_stat_force_next_flush(); +SELECT gp_stat_reset_vacuum_stats(); +SELECT bool_and(tuples_deleted = 0 AND tuples_moved = 0 + AND compacted_segments = 0 AND pages_scanned = 0 + AND total_file_segs = 0) AS ao_work_reset + FROM gp_stat_vacuum_tables + WHERE relname IN ('work_stats_ao', 'work_stats_aoco'); +SELECT bool_and(tuples_deleted = 0 AND pages_scanned = 0 + AND pages_frozen = 0 AND compacted_segments = 0) AS database_work_reset + FROM gp_stat_vacuum WHERE datname = current_database(); DROP TABLE work_stats_ao, work_stats_aoco; From 43b1a72e10d257ece2dafef39bed8ed2f262b745 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 20:16:47 +0300 Subject: [PATCH 48/50] Read native VACUUM work counters from extension views Remove duplicate work fields from the hook payload and extension storage. Read native work alongside separately collected resources, including when resource collection is disabled. Keep extension resets limited to their own metrics and document the independent collection and reset scopes. --- contrib/ext_vacuum_statistics/README.md | 4 +- .../ext_vacuum_statistics--1.0.sql | 101 ++++++------ .../t/054_vacuum_extending_gucs_test.pl | 14 +- .../t/060_vacuum_stats_reset.pl | 4 +- .../t/064_native_work_source.pl | 147 ++++++++++++++++++ .../ext_vacuum_statistics/vacuum_statistics.c | 28 +--- doc/src/sgml/extvacuumstatistics.sgml | 25 ++- doc/src/sgml/monitoring.sgml | 9 +- src/backend/access/heap/vacuumlazy.c | 31 ---- src/backend/commands/vacuum_ao.c | 35 +---- src/backend/commands/vacuumparallel.c | 7 - src/include/pgstat.h | 22 --- 12 files changed, 244 insertions(+), 183 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/t/064_native_work_source.pl diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index 1461110f05d..3c344448a2f 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -64,7 +64,7 @@ coordinator (QD) leaves segment counters unchanged. This also applies to | GUC | Default | Description | |-----|---------|-------------| -| `vacuum_statistics.enabled` | on | Enable extended vacuum statistics collection | +| `vacuum_statistics.enabled` | on | Collect extension resource metrics; native work obeys `track_counts` | ## Memory usage @@ -81,7 +81,7 @@ of an old one starts from zero. The module does that with an ## Recipes -**Disable statistics collection temporarily:** +**Disable extension resource collection temporarily (native work continues):** ```sql SET vacuum_statistics.enabled = off; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index ec5288559e5..b6023498412 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -51,18 +51,18 @@ AS 'MODULE_PATHNAME', 'vacuum_statistics_reset' LANGUAGE C STRICT VOLATILE PARALLEL UNSAFE; COMMENT ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) IS - 'Reset vacuum statistics for one table or index on the connected instance only'; + 'Reset extension-owned vacuum metrics for one table or index on the connected instance only'; COMMENT ON FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(oid) IS - 'Reset vacuum statistics for a database and its relations on the connected instance only'; + 'Reset extension-owned vacuum metrics for a database and its relations on the connected instance only'; COMMENT ON FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() IS - 'Reset vacuum statistics for all databases on the connected instance only'; + 'Reset extension-owned vacuum metrics for all databases on the connected instance only'; -- Reset privileges can be delegated explicitly, as for pg_stat_reset(). REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_entry(oid, oid) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.extvac_reset_db_entry(oid) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.vacuum_statistics_reset() FROM PUBLIC; --- Internal C function to fetch table vacuum stats +-- Internal C function to fetch table VACUUM resources CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( IN dboid oid, IN reloid oid, @@ -77,16 +77,13 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_tables( OUT blk_read_time double precision, OUT blk_write_time double precision, OUT rel_blks_read bigint, - OUT rel_blks_hit bigint, - OUT tuples_deleted bigint, - OUT pages_scanned bigint, - OUT pages_removed bigint + OUT rel_blks_hit bigint ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_tables' LANGUAGE C STRICT STABLE; --- Internal C function to fetch index vacuum stats +-- Internal C function to fetch index VACUUM resources CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( IN dboid oid, IN reloid oid, @@ -101,9 +98,7 @@ CREATE OR REPLACE FUNCTION ext_vacuum_statistics.pg_stats_get_vacuum_indexes( OUT blk_read_time double precision, OUT blk_write_time double precision, OUT rel_blks_read bigint, - OUT rel_blks_hit bigint, - OUT tuples_deleted bigint, - OUT pages_deleted bigint + OUT rel_blks_hit bigint ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'pg_stats_get_vacuum_indexes' @@ -134,28 +129,26 @@ SELECT ns.nspname AS schema, rel.relname AS relname, db.datname AS dbname, - stats.total_blks_read, - stats.total_blks_hit, - stats.total_blks_dirtied, - stats.total_blks_written, - stats.wal_records, - stats.wal_fpi, - stats.wal_bytes, - stats.blk_read_time, - stats.blk_write_time, - stats.rel_blks_read, - stats.rel_blks_hit, - stats.tuples_deleted, - stats.pages_scanned, - stats.pages_removed -FROM pg_database db, - pg_class rel, - pg_namespace ns, - LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats -WHERE db.datname = current_database() - AND rel.relkind IN ('r', 'm', 't') - AND rel.relnamespace = ns.oid - AND rel.oid = stats.relid; + COALESCE(stats.total_blks_read, 0) AS total_blks_read, + COALESCE(stats.total_blks_hit, 0) AS total_blks_hit, + COALESCE(stats.total_blks_dirtied, 0) AS total_blks_dirtied, + COALESCE(stats.total_blks_written, 0) AS total_blks_written, + COALESCE(stats.wal_records, 0) AS wal_records, + COALESCE(stats.wal_fpi, 0) AS wal_fpi, + COALESCE(stats.wal_bytes, 0) AS wal_bytes, + COALESCE(stats.blk_read_time, 0) AS blk_read_time, + COALESCE(stats.blk_write_time, 0) AS blk_write_time, + COALESCE(stats.rel_blks_read, 0) AS rel_blks_read, + COALESCE(stats.rel_blks_hit, 0) AS rel_blks_hit, + work.tuples_deleted, + work.pages_scanned, + work.pages_removed +FROM pg_class rel +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +JOIN pg_database db ON db.datname = current_database() +CROSS JOIN LATERAL pg_catalog.pg_stat_get_vacuum_stats(rel.oid) work +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(db.oid, rel.oid) stats ON true +WHERE rel.relkind IN ('r', 'm', 't', 'o', 'b', 'M'); COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_tables IS 'Extended vacuum statistics per table (heap)'; @@ -167,27 +160,25 @@ SELECT ns.nspname AS schema, rel.relname AS indexrelname, db.datname AS dbname, - stats.total_blks_read, - stats.total_blks_hit, - stats.total_blks_dirtied, - stats.total_blks_written, - stats.wal_records, - stats.wal_fpi, - stats.wal_bytes, - stats.blk_read_time, - stats.blk_write_time, - stats.rel_blks_read, - stats.rel_blks_hit, - stats.tuples_deleted, - stats.pages_deleted -FROM pg_database db, - pg_class rel, - pg_namespace ns, - LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats -WHERE db.datname = current_database() - AND rel.relkind = 'i' - AND rel.relnamespace = ns.oid - AND rel.oid = stats.relid; + COALESCE(stats.total_blks_read, 0) AS total_blks_read, + COALESCE(stats.total_blks_hit, 0) AS total_blks_hit, + COALESCE(stats.total_blks_dirtied, 0) AS total_blks_dirtied, + COALESCE(stats.total_blks_written, 0) AS total_blks_written, + COALESCE(stats.wal_records, 0) AS wal_records, + COALESCE(stats.wal_fpi, 0) AS wal_fpi, + COALESCE(stats.wal_bytes, 0) AS wal_bytes, + COALESCE(stats.blk_read_time, 0) AS blk_read_time, + COALESCE(stats.blk_write_time, 0) AS blk_write_time, + COALESCE(stats.rel_blks_read, 0) AS rel_blks_read, + COALESCE(stats.rel_blks_hit, 0) AS rel_blks_hit, + work.tuples_deleted, + work.pages_deleted +FROM pg_class rel +JOIN pg_namespace ns ON ns.oid = rel.relnamespace +JOIN pg_database db ON db.datname = current_database() +CROSS JOIN LATERAL pg_catalog.pg_stat_get_vacuum_stats(rel.oid) work +LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_indexes(db.oid, rel.oid) stats ON true +WHERE rel.relkind = 'i'; COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_indexes IS 'Extended vacuum statistics per index'; diff --git a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl index 238d3e5634b..b22f4f74a0b 100644 --- a/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl +++ b/contrib/ext_vacuum_statistics/t/054_vacuum_extending_gucs_test.pl @@ -78,7 +78,7 @@ sub reset_and_vacuum { ok($count > 0, 'stats collected when enabled'); # Reset preserves existing rows. With collection disabled, their counters - # must stay zero even after another vacuum. + # keep resources at zero while native work continues. reset_and_vacuum($dbname, 'guc_test', { gucs => ['vacuum_statistics.enabled = off'] }); $count = $node->safe_psql($dbname, @@ -86,12 +86,18 @@ sub reset_and_vacuum { is($count, 1, 'reset preserves the row when collection is disabled'); my $sums = $node->safe_psql($dbname, q{ - SELECT COALESCE(SUM(tuples_deleted), 0) - + COALESCE(SUM(pages_scanned), 0) + SELECT COALESCE(SUM(total_blks_read), 0) + + COALESCE(SUM(total_blks_hit), 0) FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'guc_test' }); - is($sums, '0', 'no counters accumulated when disabled'); + is($sums, '0', 'no resource counters accumulated when disabled'); + is($node->safe_psql($dbname, q{ + SELECT e.pages_scanned > 0 AND e.pages_scanned = n.pages_scanned + FROM ext_vacuum_statistics.pg_stats_vacuum_tables e + JOIN pg_stat_vacuum_tables n USING (relid) + WHERE e.relname = 'guc_test' + }), 't', 'disabled resource collection still displays native work'); }; $node->stop; diff --git a/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl index ea28cb5d66d..83068de1992 100644 --- a/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl +++ b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl @@ -30,8 +30,8 @@ q[SELECT oid FROM pg_database WHERE datname = current_database()]); my $relid = $node->safe_psql('postgres', q[SELECT 'reset_t'::regclass::oid]); my $idxid = $node->safe_psql('postgres', q[SELECT 'reset_t_pkey'::regclass::oid]); -my $table = q[SELECT pages_scanned FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'reset_t']; -my $index = q[SELECT tuples_deleted FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'reset_t_pkey']; +my $table = q[SELECT total_blks_read + total_blks_hit FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'reset_t']; +my $index = q[SELECT total_blks_read + total_blks_hit FROM ext_vacuum_statistics.pg_stats_vacuum_indexes WHERE indexrelname = 'reset_t_pkey']; my $database = q[SELECT db_blks_read + db_blks_hit FROM ext_vacuum_statistics.pg_stats_vacuum_database WHERE dbname = current_database()]; my $table_before = $node->safe_psql('postgres', $table); my $index_before = $node->safe_psql('postgres', $index); diff --git a/contrib/ext_vacuum_statistics/t/064_native_work_source.pl b/contrib/ext_vacuum_statistics/t/064_native_work_source.pl new file mode 100644 index 00000000000..02cdd2a4286 --- /dev/null +++ b/contrib/ext_vacuum_statistics/t/064_native_work_source.pl @@ -0,0 +1,147 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Extension views reuse native work history; collection and resets of +# extension resources must never change native counters. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('vacstat_native_source'); +$node->init; +$node->append_conf('postgresql.conf', q{ +shared_preload_libraries = 'ext_vacuum_statistics' +autovacuum = off +}); +$node->start; +$node->safe_psql('postgres', q{ +CREATE EXTENSION ext_vacuum_statistics; +CREATE TABLE native_heap (id int PRIMARY KEY); +INSERT INTO native_heap SELECT generate_series(1, 1000); +DELETE FROM native_heap WHERE id <= 100; +SET vacuum_statistics.enabled = off; +VACUUM (FREEZE) native_heap; +SELECT pg_stat_force_next_flush(); +}); + +is($node->safe_psql('postgres', q{ +SELECT count(*) FROM ext_vacuum_statistics.pg_stats_get_vacuum_tables( + (SELECT oid FROM pg_database WHERE datname = current_database()), + 'native_heap'::regclass) +}), '0', 'disabled first vacuum creates no extension resource entry'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted = 100 AND pages_scanned > 0 AND tuples_frozen = 900 + AND total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0 + FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'native_heap' +}), 't', 'view exposes native work even without a resource entry'); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted = 100 AND total_blks_read = 0 AND total_blks_hit = 0 + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes + WHERE indexrelname = 'native_heap_pkey' +}), 't', 'index view exposes native work without a resource entry'); + +for my $orientation ('row', 'column') +{ + $node->safe_psql('postgres', qq{ +CREATE TABLE native_$orientation (id int) + WITH (appendonly = true, orientation = $orientation); +INSERT INTO native_$orientation SELECT generate_series(1, 1000); +DELETE FROM native_$orientation WHERE id <= 100; +SET gp_appendonly_compaction = off; +VACUUM native_$orientation; +VACUUM native_$orientation; +SELECT pg_stat_force_next_flush(); +}); + is($node->safe_psql('postgres', qq{ +SELECT recently_dead_tuples FROM ext_vacuum_statistics.pg_stats_vacuum_tables + WHERE relname = 'native_$orientation' +}), '200', "$orientation AO shows the native cumulative hidden-row count"); +} + +my @table_fields = qw(tuples_deleted pages_scanned pages_removed tuples_frozen + recently_dead_tuples missed_dead_pages missed_dead_tuples pages_frozen + pages_all_visible total_file_segs compacted_segments tuples_moved + dead_pages freeze_age_vacuum_count); +my @index_fields = qw(tuples_deleted pages_deleted dead_pages); +for my $consistency ('none', 'cache', 'snapshot') +{ + my $tables_match = join(' AND ', map { "e.$_ = n.$_" } @table_fields); + my $indexes_match = join(' AND ', map { "e.$_ = n.$_" } @index_fields); + is($node->safe_psql('postgres', qq{ +SET stats_fetch_consistency = '$consistency'; +SELECT count(*) = 3 AND bool_and($tables_match) + FROM ext_vacuum_statistics.pg_stats_vacuum_tables e + CROSS JOIN LATERAL pg_stat_get_vacuum_stats(e.relid) n + WHERE e.relname IN ('native_heap', 'native_row', 'native_column'); +}), 't', "$consistency: all heap and AO work fields match native counters"); + is($node->safe_psql('postgres', qq{ +SET stats_fetch_consistency = '$consistency'; +SELECT $indexes_match + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes e + CROSS JOIN LATERAL pg_stat_get_vacuum_stats(e.indexrelid) n + WHERE e.indexrelname = 'native_heap_pkey'; +}), 't', "$consistency: all index work fields match native counters"); +} + +my $work = q{ +SELECT row_to_json(n) FROM pg_stat_get_vacuum_stats('native_heap'::regclass) n +}; +my $resources = q{ +SELECT total_blks_read + total_blks_hit + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'native_heap' +}; +my $work_before = $node->safe_psql('postgres', $work); +$node->safe_psql('postgres', q{ +SET track_counts = off; +VACUUM (DISABLE_PAGE_SKIPPING) native_heap; +}); +is($node->safe_psql('postgres', $work), $work_before, + 'track_counts disables native work collection independently'); +cmp_ok($node->safe_psql('postgres', $resources), '>', 0, + 'resource collection continues with track_counts off'); + +my $dboid = $node->safe_psql('postgres', + q[SELECT oid FROM pg_database WHERE datname = current_database()]); +for my $call ("extvac_reset_entry($dboid, 'native_heap'::regclass)", + "extvac_reset_db_entry($dboid)", 'vacuum_statistics_reset()') +{ + $node->safe_psql('postgres', q{ +VACUUM (DISABLE_PAGE_SKIPPING) native_heap; +SELECT pg_stat_force_next_flush(); +}); + my $native = $node->safe_psql('postgres', $work); + my $db_native_sql = qq{ +SELECT row_to_json(n) FROM pg_stat_get_vacuum_database_stats($dboid) n +}; + my $db_native = $node->safe_psql('postgres', $db_native_sql); + my $other_sql = q{ +SELECT n_tup_ins, n_tup_del, vacuum_count, last_vacuum + FROM pg_stat_all_tables WHERE relname = 'native_heap' +}; + my $other = $node->safe_psql('postgres', $other_sql); + cmp_ok($node->safe_psql('postgres', $resources), '>', 0, + "$call: resources collected before reset"); + $node->safe_psql('postgres', "SELECT ext_vacuum_statistics.$call"); + is($node->safe_psql('postgres', $resources), '0', "$call clears resources"); + is($node->safe_psql('postgres', $work), $native, "$call preserves native relation work"); + is($node->safe_psql('postgres', $db_native_sql), $db_native, + "$call preserves native database work"); + is($node->safe_psql('postgres', $other_sql), $other, + "$call preserves other native relation statistics"); +} + +$node->safe_psql('postgres', 'VACUUM (DISABLE_PAGE_SKIPPING) native_heap'); +my $resources_before = $node->safe_psql('postgres', $resources); +$node->safe_psql('postgres', + q[SELECT pg_stat_reset_vacuum_stats('native_heap'::regclass)]); +is($node->safe_psql('postgres', q{ +SELECT tuples_deleted = 0 AND pages_scanned = 0 AND tuples_frozen = 0 + FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'native_heap' +}), 't', 'native reset is visible in the extension work columns'); +is($node->safe_psql('postgres', $resources), $resources_before, + 'native relation reset preserves extension resources'); + +$node->stop; +done_testing(); diff --git a/contrib/ext_vacuum_statistics/vacuum_statistics.c b/contrib/ext_vacuum_statistics/vacuum_statistics.c index 17ed45743b6..1731260aa32 100644 --- a/contrib/ext_vacuum_statistics/vacuum_statistics.c +++ b/contrib/ext_vacuum_statistics/vacuum_statistics.c @@ -122,7 +122,6 @@ pgstat_accumulate_common(PgStat_CommonCounts * dst, const PgStat_CommonCounts * dst->wal_records += src->wal_records; dst->wal_fpi += src->wal_fpi; dst->wal_bytes += src->wal_bytes; - dst->tuples_deleted += src->tuples_deleted; } static inline void @@ -136,16 +135,6 @@ pgstat_accumulate_extvac_stats(PgStat_VacuumRelationCounts * dst, Assert(src->type == dst->type); pgstat_accumulate_common(&dst->common, &src->common); - - if (dst->type == PGSTAT_EXTVAC_TABLE) - { - dst->table.pages_scanned += src->table.pages_scanned; - dst->table.pages_removed += src->table.pages_removed; - } - else if (dst->type == PGSTAT_EXTVAC_INDEX) - { - dst->pages_deleted += src->pages_deleted; - } } void @@ -158,7 +147,7 @@ _PG_init(void) errdetail("Add 'ext_vacuum_statistics' into the shared_preload_libraries list."))); DefineCustomBoolVariable("vacuum_statistics.enabled", - "Enable extended vacuum statistics collection.", + "Collect VACUUM resources and extension-only metrics; work counters obey track_counts.", NULL, &evs_enabled, true, PGC_SUSET, GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); @@ -399,8 +388,8 @@ tuplestore_put_common(PgStat_CommonCounts * vacuum_ext, Assert((*i - base) == EXTVAC_COMMON_STAT_COLS); } -#define EXTVAC_HEAP_STAT_COLS 15 -#define EXTVAC_IDX_STAT_COLS 14 +#define EXTVAC_HEAP_STAT_COLS 12 +#define EXTVAC_IDX_STAT_COLS 12 #define EXTVAC_MAX_STAT_COLS Max(EXTVAC_HEAP_STAT_COLS, EXTVAC_IDX_STAT_COLS) static void @@ -418,17 +407,6 @@ tuplestore_put_for_relation(Oid relid, Tuplestorestate *tupstore, values[i++] = Int64GetDatum(vacuum_ext->common.blks_fetched - vacuum_ext->common.blks_hit); values[i++] = Int64GetDatum(vacuum_ext->common.blks_hit); - if (vacuum_ext->type == PGSTAT_EXTVAC_TABLE) - { - values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); - values[i++] = Int64GetDatum(vacuum_ext->table.pages_scanned); - values[i++] = Int64GetDatum(vacuum_ext->table.pages_removed); - } - else if (vacuum_ext->type == PGSTAT_EXTVAC_INDEX) - { - values[i++] = Int64GetDatum(vacuum_ext->common.tuples_deleted); - values[i++] = Int64GetDatum(vacuum_ext->pages_deleted); - } Assert(i == ((vacuum_ext->type == PGSTAT_EXTVAC_TABLE) ? EXTVAC_HEAP_STAT_COLS : EXTVAC_IDX_STAT_COLS)); tuplestore_putvalues(tupstore, tupdesc, values, nulls); diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index d986d8bc857..8201af36894 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -32,7 +32,7 @@ is not registered, so the server reports the file as corrupted and resets every statistics kind, built-in ones such as those shown in pg_stat_user_tables included. To stop collecting - vacuum statistics without that side effect, keep the module loaded and + extension resource statistics without that side effect, keep the module loaded and set vacuum_statistics.enabled to off instead. @@ -125,10 +125,24 @@ Functions + + These functions reset only the extension's own metrics. Native VACUUM + work counters and other core statistics are never reset by the module. + Use pg_stat_reset_vacuum_stats(relid) for one relation + or pg_stat_reset_vacuum_stats() for the current database + and its non-shared relations. These preserve timing, invocation counts and + other native statistics. On Cloudberry, use the corresponding + gp_stat_reset_vacuum_stats() wrapper from the coordinator + to include every primary segment. A native relation reset leaves extension resources intact; + pg_stat_reset() clears both for the current database. + The counters can cover different periods after a reset or a change in + collection settings. + + Reset functions require superuser privileges by default. Access can be delegated with GRANT EXECUTE. Existing entries remain - visible with zero counters after a reset; their table, index, or database + visible with zero extension counters after a reset; native work is unchanged; their table, index, or database type is preserved. Missing entries are not created by a reset. pg_stat_reset() also resets the module's counters for the current database. @@ -161,7 +175,7 @@ - Resets the vacuum statistics of one table or index only on the current + Resets the module's own vacuum metrics for one table or index only on the current node. Use 0 as dboid for a shared catalog. The database aggregate and other relations are unchanged. @@ -176,7 +190,7 @@ - Resets the vacuum statistics of a database only on the current node: + Resets the module's own vacuum metrics for a database only on the current node: its aggregate and the statistics of all its tables and indexes. @@ -192,7 +206,8 @@ vacuum_statistics.enabled (boolean) - Enables extended vacuum statistics collection. Default: on. + Enables collection of extension resource metrics. Native work counters + obey track_counts independently. Default: on. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index a0e87878d6b..c57b6ece3c7 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -4941,9 +4941,12 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i statistics snapshot rules, and are removed when their relation or database is dropped. pg_stat_reset_single_table_counters(oid) resets a relation's counters; pg_stat_reset() resets - the current database statistics. These resets are independent of the - compatibility counters and resource statistics in - ext_vacuum_statistics. + the current database statistics. The table and index views of + ext_vacuum_statistics read these same work counters. + A native relation reset leaves extension resource counters intact; + pg_stat_reset() resets both for the current database. + Extension reset functions clear only extension metrics and never modify + native work or other core statistics. pg_stat_reset_vacuum_stats() resets just native work diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 7a9e8279a4e..5c82196d853 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -435,17 +435,6 @@ static void accumulate_heap_vacuum_statistics(LVRelState *vacrel, PgStat_VacuumRelationCounts * extVacStats) { extVacStats->type = PGSTAT_EXTVAC_TABLE; - extVacStats->table.pages_scanned = vacrel->scanned_pages; - extVacStats->table.pages_removed = vacrel->removed_pages; - extVacStats->table.pages_frozen = vacrel->frozen_pages; - extVacStats->table.pages_all_visible = vacrel->all_visible_pages; - extVacStats->common.tuples_deleted = vacrel->tuples_deleted; - extVacStats->table.tuples_frozen = vacrel->tuples_frozen; - extVacStats->table.recently_dead_tuples = vacrel->recently_dead_tuples; - extVacStats->table.missed_dead_tuples = vacrel->missed_dead_tuples; - extVacStats->table.missed_dead_pages = vacrel->missed_dead_pages; - extVacStats->dead_pages = vacrel->dead_pages; - extVacStats->table.freeze_age_vacuum_count = vacrel->freeze_age_vacuum ? 1 : 0; /* * Subtract the resource usage of the index passes this process ran: they @@ -3056,15 +3045,6 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (vacrel->verbose) extvac_stats_log(indrel, _("index bulk delete"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; - if (istat != NULL) - { - extVacReport->common.tuples_deleted = - istat->tuples_removed - prev_tuples_removed; - extVacReport->pages_deleted = - (istat->pages_newly_deleted >= prev_pages_newly_deleted) ? - istat->pages_newly_deleted - prev_pages_newly_deleted : - istat->pages_newly_deleted; - } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); @@ -3153,17 +3133,6 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, if (vacrel->verbose) extvac_stats_log(indrel, _("index cleanup"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; - if (istat != NULL) - { - extVacReport->common.tuples_deleted = - istat->tuples_removed - prev_tuples_removed; - extVacReport->pages_deleted = - (istat->pages_newly_deleted >= prev_pages_newly_deleted) ? - istat->pages_newly_deleted - prev_pages_newly_deleted : - istat->pages_newly_deleted; - if (istat->pages_deleted > istat->pages_free) - extVacReport->dead_pages = istat->pages_deleted - istat->pages_free; - } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); accumulate_idxs_vacuum_statistics(vacrel, extVacReport); pfree(extVacCounters); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 49a5e6e5f45..cb7964d8799 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -173,8 +173,7 @@ static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, bool subtract); static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - IndexBulkDeleteResult *result, - AOVacuumRelStats *vacrelstats, bool final_cleanup, bool verbose); + AOVacuumRelStats *vacrelstats, bool verbose); static void ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats); @@ -616,7 +615,6 @@ ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, /* * Add the resource usage counters of src to dst, or subtract them from it. - * The tuple counter is left alone. */ static void ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, @@ -659,8 +657,7 @@ ao_accum_resources(PgStat_CommonCounts *dst, const PgStat_CommonCounts *src, */ static void ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, - IndexBulkDeleteResult *result, - AOVacuumRelStats *vacrelstats, bool final_cleanup, bool verbose) + AOVacuumRelStats *vacrelstats, bool verbose) { PgStat_VacuumRelationCounts *report = &counters->report; @@ -668,10 +665,6 @@ ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, if (verbose) extvac_stats_log(indrel, _("AO index vacuum"), &report->common); report->type = PGSTAT_EXTVAC_INDEX; - report->common.tuples_deleted = (int64) result->tuples_removed; - report->pages_deleted = result->pages_newly_deleted; - if (final_cleanup && result->pages_deleted > result->pages_free) - report->dead_pages = result->pages_deleted - result->pages_free; ao_accum_resources(&vacrelstats->extstats->indexes, &report->common, false); @@ -682,11 +675,8 @@ ao_measure_index_resources(Relation indrel, LVExtStatCounters *counters, * Report the vacuum of an append-optimized table, all of its phases, to * set_report_vacuum_hook. * - * The counters that mean something for append-optimized storage are set: - * the tuples discarded, moved or left hidden, the segments actually - * compacted, the segment count at completion, and the space scanned or - * released in heap-equivalent pages. Per-tuple freezing, heap pruning and - * heap visibility-map counters stay zero. + * Work counters are reported separately to native pgstat. The hook carries + * resources and the extension-only awaiting-drop segment snapshot. */ static void ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) @@ -697,15 +687,6 @@ ao_measure_table_resources(Relation rel, AOVacuumRelStats *vacrelstats) report.type = PGSTAT_EXTVAC_TABLE; report.common = vacrelstats->extstats->phases; ao_accum_resources(&report.common, &vacrelstats->extstats->indexes, true); - report.common.tuples_deleted = vacrelstats->num_dead_tuples; - report.table.recently_dead_tuples = vacrelstats->dead_tuples; - report.table.pages_scanned = vacrelstats->pages_scanned; - report.table.total_file_segs = vacrelstats->total_file_segs; - report.table.compacted_segments = vacrelstats->compacted_segments; - report.table.tuples_moved = vacrelstats->tuples_moved; - report.table.pages_removed = - vacrelstats->nbytes_truncated / BLCKSZ + - (vacrelstats->nbytes_truncated % BLCKSZ != 0); } @@ -829,8 +810,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); if (extcounters != NULL) - ao_measure_index_resources(Irel[i], extcounters, &result, - vacrelstats, final_cleanup, + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats, elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } @@ -854,8 +835,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs vacuum_report_index_stats(Irel[i], &result, 0, 0, final_cleanup); ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); if (extcounters != NULL) - ao_measure_index_resources(Irel[i], extcounters, &result, - vacrelstats, final_cleanup, + ao_measure_index_resources(Irel[i], extcounters, + vacrelstats, elevel == INFO && Gp_role != GP_ROLE_DISPATCH); } } diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index de0993c5926..dd7fda5e66c 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -908,13 +908,6 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, _("index cleanup") : _("index bulk delete"), &extVacReport->common); extVacReport->type = PGSTAT_EXTVAC_INDEX; - if (istat_res != NULL) - { - extVacReport->common.tuples_deleted = - istat_res->tuples_removed - prev_tuples_removed; - extVacReport->pages_deleted = - istat_res->pages_newly_deleted - prev_pages_newly_deleted; - } pgstat_report_vacuum_ext(indrel, -1, -1, 0, 0, false, extVacReport); pfree(extVacCounters); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index bfe0214e918..16a1044681e 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -142,34 +142,12 @@ typedef struct PgStat_CommonCounts int64 wal_records; int64 wal_fpi; uint64 wal_bytes; - int64 tuples_deleted; } PgStat_CommonCounts; typedef struct PgStat_VacuumRelationCounts { PgStat_CommonCounts common; ExtVacReportType type; - int64 dead_pages; /* remaining dead pages observed at each vacuum */ - union - { - struct - { - int64 tuples_frozen; - int64 recently_dead_tuples; - int64 missed_dead_tuples; - int64 pages_scanned; - int64 pages_removed; - int64 missed_dead_pages; - int64 pages_frozen; - int64 pages_all_visible; - int64 freeze_age_vacuum_count; - /* AO-only: last segment count and cumulative compaction work. */ - int64 total_file_segs; - int64 compacted_segments; - int64 tuples_moved; - } table; - int64 pages_deleted; /* index-only counter */ - }; } PgStat_VacuumRelationCounts; typedef struct PgStat_FunctionCallUsage From f9d6bbedeaabe1a8820bd90a21c893a0cc805a99 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 20:40:10 +0300 Subject: [PATCH 49/50] Include cluster VACUUM resource views and resets with the extension Introduce gp_stats_vacuum_tables, indexes and database views, cluster summaries, and gp_ reset wrappers with the initial extension. Carry over replicated-table normalization, synchronized collection control coverage, transactional DROP tests, and cluster installation and failover documentation. --- contrib/ext_vacuum_statistics/Makefile | 9 + contrib/ext_vacuum_statistics/README.md | 179 +++++++++- .../expected/gp_vacuum_statistics.out | 337 ++++++++++++++++++ .../ext_vacuum_statistics--1.0.sql | 220 +++++++++++- .../sql/gp_vacuum_statistics.sql | 206 +++++++++++ .../t/060_vacuum_stats_reset.pl | 3 +- .../t/064_native_work_source.pl | 35 +- doc/src/sgml/extvacuumstatistics.sgml | 219 +++++++++++- 8 files changed, 1155 insertions(+), 53 deletions(-) create mode 100644 contrib/ext_vacuum_statistics/expected/gp_vacuum_statistics.out create mode 100644 contrib/ext_vacuum_statistics/sql/gp_vacuum_statistics.sql diff --git a/contrib/ext_vacuum_statistics/Makefile b/contrib/ext_vacuum_statistics/Makefile index 1e53a7dbda4..4d90451588a 100644 --- a/contrib/ext_vacuum_statistics/Makefile +++ b/contrib/ext_vacuum_statistics/Makefile @@ -20,3 +20,12 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif + +# Cloudberry: the test of the cluster-wide views needs a running cluster +# with the module in shared_preload_libraries of every instance, which the +# temporary instances of "make check" are not, so it has a target of its +# own, run against such a cluster. +.PHONY: installcheck-cluster + +installcheck-cluster: + $(pg_regress_installcheck) --dbname=contrib_regression_vacstat gp_vacuum_statistics diff --git a/contrib/ext_vacuum_statistics/README.md b/contrib/ext_vacuum_statistics/README.md index 3c344448a2f..f8c7bb950bc 100644 --- a/contrib/ext_vacuum_statistics/README.md +++ b/contrib/ext_vacuum_statistics/README.md @@ -29,20 +29,24 @@ CREATE EXTENSION ext_vacuum_statistics; ## Usage -Query vacuum statistics via the provided views: +From a Cloudberry coordinator, query statistics across the cluster: ```sql --- Per-table heap vacuum statistics -SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_tables; +-- Per-table heap vacuum statistics, with gp_segment_id +SELECT * FROM ext_vacuum_statistics.gp_stats_vacuum_tables; -- Per-index vacuum statistics -SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; +SELECT * FROM ext_vacuum_statistics.gp_stats_vacuum_indexes; -- Per-database aggregate vacuum statistics -SELECT * FROM ext_vacuum_statistics.pg_stats_vacuum_database; +SELECT * FROM ext_vacuum_statistics.gp_stats_vacuum_database; ``` -Example output: +The corresponding `pg_stats_vacuum_*` views read only the connected instance. +Use them for a standalone server or a utility connection to a single segment. +Use the `gp_stats_vacuum_*_summary` views for cluster totals. + +Example projection of table statistics: ``` relname | total_blks_read | total_blks_hit | wal_records | tuples_deleted | pages_removed @@ -50,15 +54,36 @@ Example output: mytable | 120 | 340 | 15 | 500 | 10 ``` -Reset statistics only on the current node: +Reset statistics across the cluster from the coordinator: ```sql -SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); ``` -The reset functions act only on the current node. Calling one on the -coordinator (QD) leaves segment counters unchanged. This also applies to -`extvac_reset_entry()` and `extvac_reset_db_entry()`. +These functions reset only extension-owned metrics; native work is unchanged. +Use the `gp_` wrappers from a normal coordinator connection to reset them +on the coordinator and all primary segments. + +**Local functions reset only the current node.** Calling +`ext_vacuum_statistics.vacuum_statistics_reset()` on the coordinator (QD) +resets only the coordinator's counters; segment counters remain unchanged. +The same applies to `extvac_reset_entry()` and `extvac_reset_db_entry()`. +Use the local functions for a utility connection to an individual segment. + +All functions below belong to the `ext_vacuum_statistics` schema: + +| Reset scope | Cluster wrapper (call on the coordinator) | Local function (current node only) | +|-------------|------------------------------------------|-----------------------------------| +| One table or index | `gp_extvac_reset_entry(dboid, relid)` | `extvac_reset_entry(dboid, relid)` | +| One database and all its relations | `gp_extvac_reset_db_entry(dboid)` | `extvac_reset_db_entry(dboid)` | +| All databases | `gp_vacuum_statistics_reset()` | `vacuum_statistics_reset()` | + +For example, reset only the current database across the cluster: + +```sql +SELECT ext_vacuum_statistics.gp_extvac_reset_db_entry(oid) +FROM pg_database WHERE datname = current_database(); +``` ## Configuration (GUCs) @@ -105,11 +130,127 @@ SET vacuum_statistics.enabled = off; ## Native work counters -Heap, index and AO work is also collected by the built-in statistics system -without this module. Use `pg_stat_vacuum_tables`, `pg_stat_vacuum_indexes` -and `pg_stat_vacuum` (and their `gp_stat_*` counterparts) for native counters. -They obey `track_counts` and ordinary `pg_stat_reset*` functions. The extension -retains its existing counters for compatibility and adds resource measurements -such as buffers and WAL. Its counters obey `vacuum_statistics.enabled` and -its own reset functions. Extension-specific resets do not clear native counters; -`pg_stat_reset()` also clears the extension's entries for the current database. +Heap, index and AO work counters are accumulated once, by native pgstat. +The extension's table and index views read those same counters through +`pg_stat_get_vacuum_stats()`, alongside separately stored buffer, WAL and I/O +measurements. They have the same work-counter history as `pg_stat_vacuum_tables`, +`pg_stat_vacuum_indexes` and their `gp_stat_*` counterparts; enabling the +extension does not start a second history. + +Work counters obey `track_counts`. `vacuum_statistics.enabled` controls only +extension resource metrics. +With extension collection disabled, native work remains visible in these views. +Relations without a resource entry show zero resource counters. + +Extension reset functions clear only the extension's own metrics, without +touching native work or any other core statistics. To reset native work, use +`pg_stat_reset_vacuum_stats(relid)` for one table or index, or +`pg_stat_reset_vacuum_stats()` for the current database and its non-shared +relations. These functions preserve native timing, vacuum/analyze invocation +counts, and other statistics. On Cloudberry, call the corresponding +`gp_stat_reset_vacuum_stats(relid)` or `gp_stat_reset_vacuum_stats()` on the +coordinator to reach all primary segments. The broader native reset +`pg_stat_reset_single_table_counters()` +clears native work but leaves extension resources intact; `pg_stat_reset()` +clears both for the current database. The two sets of metrics can therefore +cover different periods after a reset or a change in collection settings. + +The resource getter signatures and shared payload changed in this development +series. Rebuild the server and extension together and recreate the extension's +SQL objects when updating an earlier 1.0 installation. The new statistics-file +format discards previously saved counters; the new core function also requires +the matching catalog version. + +## Cloudberry + +Each instance (the coordinator and every segment) keeps the statistics of the +vacuums it runs itself. The local `pg_stats_vacuum_*` views show only that +instance; use `gp_stats_vacuum_*` to read the coordinator and primary segments. + +Install the matching extension library, control and SQL files in the Cloudberry +installation on every host, including mirror and standby coordinator hosts. +`gpconfig` changes configuration files; it does not distribute extension files. +Load the module on every instance so it is also available after promotion. + +Inspect both the active setting and the configuration files first: + +```sh +gpconfig -s shared_preload_libraries +gpconfig -s shared_preload_libraries --file +``` + +When the existing list is empty, configure it with: + +```sh +gpconfig -c shared_preload_libraries -v 'ext_vacuum_statistics' +``` + +Otherwise pass the complete existing list with `ext_vacuum_statistics` appended. +Preserve any intentional differences between coordinator, primary and mirror +settings; do not replace their other preloaded libraries. With `-v` and no +role restriction, `gpconfig` updates the coordinator, standby, primaries and +mirrors. Resolve any reported unreachable hosts before relying on their setup. +Check the resulting files, then restart the cluster to load the library: + +```sh +gpconfig -s shared_preload_libraries --file +gpstop -ar +gpconfig -s shared_preload_libraries --file-compare +``` + +A configuration reload (`gpstop -u`) alone cannot load a new preload library. +Check that mirror and standby instances also restarted successfully. After the +restart, run the following on the coordinator in each database where the views +are needed; `CREATE EXTENSION` is dispatched to the segments: + +```sql +CREATE EXTENSION IF NOT EXISTS ext_vacuum_statistics; +``` + +Cluster-wide views, like the `gp_stat_*` views of the core: + +| View | Description | +|------|-------------| +| `ext_vacuum_statistics.gp_stats_vacuum_tables` | `pg_stats_vacuum_tables` of every instance, with `gp_segment_id` (-1 for the coordinator) | +| `ext_vacuum_statistics.gp_stats_vacuum_indexes` | the same for indexes | +| `ext_vacuum_statistics.gp_stats_vacuum_database` | the same for databases | +| `ext_vacuum_statistics.gp_stats_vacuum_tables_summary` | one row per table: summed over the segments (divided by their number for replicated tables); catalogs as on the coordinator | +| `ext_vacuum_statistics.gp_stats_vacuum_indexes_summary` | the same for indexes | +| `ext_vacuum_statistics.gp_stats_vacuum_database_summary` | one row per database, summed over all instances | + +The reset functions without the `gp_` prefix act only on the connected instance; +`gp_vacuum_statistics_reset()`, `gp_extvac_reset_entry(dboid, relid)` and +`gp_extvac_reset_db_entry(dboid)` dispatch the resets from the coordinator to +all primary segments and also reset the coordinator. A `SET` of +`vacuum_statistics.enabled` on the coordinator is passed on to the segments. + +The counters are local pgstat state and are not replicated through WAL. +Preloading on a mirror enables collection after promotion; it does not copy +its primary's accumulated counters. A promoted instance reports its own local +statistics, so it cannot continue the former primary's counter history. This +also applies when the standby coordinator is promoted. A compatible clean +restart can preserve an instance's own counters; crash recovery resets them. + +Treat promotion as a new measurement interval for the affected instance, even +if its `gp_segment_id` is unchanged. Other instances keep their own counters, +but cluster summaries can decrease when a primary is replaced. Monitoring +should retain previous samples externally and begin a new baseline after +failover, without interpreting the change as negative vacuum work. + +The test of the cluster-wide views runs against such a cluster: + +``` +make -C contrib/ext_vacuum_statistics installcheck-cluster +``` + +Reset functions return `void` and require superuser privileges by default. +An administrator can delegate access with `GRANT EXECUTE`; cluster wrappers +also require permission to execute the corresponding local reset function. +An extension relation reset clears only that relation's resource counters, leaving its indexes and +the database aggregate unchanged. A database reset clears its aggregate and +all its relation entries. The global reset affects all databases on the current +instance; the `gp_` wrappers apply these operations on the coordinator and all +primary segments. +Existing rows remain visible with zero extension counters after a reset. +Native work counters remain unchanged. `pg_stat_reset()` also resets the +extension's entries for the current database. Resetting does not remove the type of a table, index, or database entry. diff --git a/contrib/ext_vacuum_statistics/expected/gp_vacuum_statistics.out b/contrib/ext_vacuum_statistics/expected/gp_vacuum_statistics.out new file mode 100644 index 00000000000..5fedd63d316 --- /dev/null +++ b/contrib/ext_vacuum_statistics/expected/gp_vacuum_statistics.out @@ -0,0 +1,337 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Cluster-wide vacuum statistics: aggregation, collection control, reset, +-- and cleanup after DROP. The final two cases cover core summary views +-- and local scope of track_cost_delay_timing. +-- +-- This test runs against a Cloudberry cluster that has ext_vacuum_statistics +-- in shared_preload_libraries of every instance ("make installcheck-cluster"). +-- +CREATE EXTENSION IF NOT EXISTS ext_vacuum_statistics; +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); + gp_vacuum_statistics_reset +---------------------------- + +(1 row) + +SELECT oid AS dboid FROM pg_database WHERE datname = current_database() \gset +-- Prepare dead tuples: 10000 in a distributed table and 300 copies on +-- each segment in a replicated table. VACUUM also cleans the index. +CREATE TABLE gpvs_dist (id int, v text) WITH (autovacuum_enabled = off) + DISTRIBUTED BY (id); +CREATE INDEX gpvs_dist_idx ON gpvs_dist (id); +INSERT INTO gpvs_dist SELECT g, 'x' FROM generate_series(1, 10000) g; +CREATE TABLE gpvs_repl (id int) WITH (autovacuum_enabled = off) + DISTRIBUTED REPLICATED; +INSERT INTO gpvs_repl SELECT generate_series(1, 300); +DELETE FROM gpvs_dist; +DELETE FROM gpvs_repl; +VACUUM gpvs_dist, gpvs_repl; +-- Collection: expect an entry on every primary segment and the coordinator. +-- role = 'p' includes the coordinator and excludes mirrors. +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration + WHERE role = 'p') AS one_row_per_instance + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + one_row_per_instance +---------------------- + t +(1 row) + +-- Expect 10000 removed tuples across the segments and 0 on the coordinator. +SELECT gp_segment_id = -1 AS coordinator, sum(tuples_deleted) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' + GROUP BY 1 ORDER BY 1; + coordinator | tuples_deleted +-------------+---------------- + f | 10000 + t | 0 +(2 rows) + +-- Aggregation: expect 10000 for the distributed table and 300 for the +-- replicated table. Replicated counts must be averaged across segments. +SELECT relname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + relname | tuples_deleted +-----------+---------------- + gpvs_dist | 10000 + gpvs_repl | 300 +(2 rows) + +-- The index summary must also report 10000 removed tuples. +SELECT indexrelname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_indexes_summary + WHERE indexrelname = 'gpvs_dist_idx'; + indexrelname | tuples_deleted +---------------+---------------- + gpvs_dist_idx | 10000 +(1 row) + +-- WAL volume varies, so compare the summary with the segment counters. +SELECT s.wal_records = (SELECT sum(wal_records) + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' AND gp_segment_id >= 0) + AS summary_matches_segments + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary s + WHERE s.relname = 'gpvs_dist'; + summary_matches_segments +-------------------------- + t +(1 row) + +-- Database WAL includes these tables plus indexes and other relations; +-- it must be positive and at least as large as the table WAL total. +SELECT db_wal_records > 0 AS database_has_wal, + db_wal_records >= (SELECT sum(wal_records) + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname LIKE 'gpvs%') AS database_covers_tables + FROM ext_vacuum_statistics.gp_stats_vacuum_database_summary + WHERE dbname = current_database(); + database_has_wal | database_covers_tables +------------------+------------------------ + t | t +(1 row) + +-- Core table summaries must also record a positive vacuum duration. +SELECT relname, total_vacuum_time > 0 AS vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + relname | vacuum_timed +-----------+-------------- + gpvs_dist | t + gpvs_repl | t +(2 rows) + +-- Collection control: disable collection on the coordinator, then vacuum +-- another 1000 dead tuples. Native work grows; resource counters stay unchanged. +CREATE TEMP TABLE gpvs_resources_before AS + SELECT gp_segment_id AS segid, total_blks_read, total_blks_hit, wal_records + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' + DISTRIBUTED BY (segid); +INSERT INTO gpvs_dist SELECT g, 'y' FROM generate_series(1, 1000) g; +DELETE FROM gpvs_dist; +SET vacuum_statistics.enabled = off; +VACUUM gpvs_dist; +RESET vacuum_statistics.enabled; +SELECT bool_and(s.total_blks_read = b.total_blks_read + AND s.total_blks_hit = b.total_blks_hit + AND s.wal_records = b.wal_records) AS resources_unchanged + FROM ext_vacuum_statistics.gp_stats_vacuum_tables s + JOIN gpvs_resources_before b ON b.segid = s.gp_segment_id + WHERE s.relname = 'gpvs_dist'; + resources_unchanged +--------------------- + t +(1 row) + +DROP TABLE gpvs_resources_before; +SELECT bool_and(e.tuples_deleted = n.tuples_deleted) AS native_work_matches + FROM ext_vacuum_statistics.gp_stats_vacuum_tables e + JOIN gp_stat_vacuum_tables n USING (gp_segment_id, relid) + WHERE e.relname = 'gpvs_dist'; + native_work_matches +--------------------- + t +(1 row) + +SELECT sum(tuples_deleted) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + tuples_deleted +---------------- + 11000 +(1 row) + +-- Relation reset clears extension resources across the cluster, preserving +-- native work for gpvs_dist and gpvs_repl. +SELECT ext_vacuum_statistics.gp_extvac_reset_entry(:dboid, 'gpvs_dist'::regclass); + gp_extvac_reset_entry +----------------------- + +(1 row) + +SELECT bool_and(total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0) + AS resources_reset + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + resources_reset +----------------- + t +(1 row) + +SELECT coalesce(sum(tuples_deleted), 0) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + tuples_deleted +---------------- + 11000 +(1 row) + +SELECT relname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + relname | tuples_deleted +-----------+---------------- + gpvs_dist | 11000 + gpvs_repl | 300 +(2 rows) + +-- DROP cleanup: save the OID and read the statistics directly. The public +-- views join pg_class, so they would hide an orphaned entry after DROP. +-- gp_id makes this helper view executable on each segment via gp_dist_random. +-- Expect 3 segment entries and 1 coordinator entry in the regression cluster. +SELECT 'gpvs_repl'::regclass::oid AS repl_oid \gset +CREATE VIEW gpvs_repl_entry AS + SELECT s.relid + FROM gp_id, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(:dboid, :repl_oid) s; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); + segment_entries +----------------- + 3 +(1 row) + +SELECT count(*) AS coordinator_entries FROM gpvs_repl_entry; + coordinator_entries +--------------------- + 1 +(1 row) + +-- Rolling back DROP must preserve the 3 segment entries. +BEGIN; +DROP TABLE gpvs_repl; +ROLLBACK; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); + segment_entries +----------------- + 3 +(1 row) + +-- Committing DROP must remove entries from both segments and coordinator. +DROP TABLE gpvs_repl; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); + segment_entries +----------------- + 0 +(1 row) + +SELECT count(*) AS coordinator_entries FROM gpvs_repl_entry; + coordinator_entries +--------------------- + 0 +(1 row) + +-- Core database timing is published asynchronously. Allow pending stats +-- to be flushed before checking that the vacuum duration is positive. +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); + database_vacuum_timed +----------------------- + t +(1 row) + +-- Cluster reset must preserve the remaining table's entries and zero +-- its resources on every instance, preserving native work. +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); + gp_vacuum_statistics_reset +---------------------------- + +(1 row) + +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p') AS rows_preserved, + bool_and(total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0) AS resources_reset, + sum(tuples_deleted) = 11000 AS work_preserved + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + rows_preserved | resources_reset | work_preserved +----------------+-----------------+---------------- + t | t | t +(1 row) + +DROP VIEW gpvs_repl_entry; +DROP TABLE gpvs_dist; +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +RESET track_cost_delay_timing; +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +VACUUM core_summary_test; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; + summary_indexes | distinct_indexes +-----------------+------------------ + 2 | 2 +(1 row) + +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; + index_totals_match +-------------------- + t +(1 row) + +DROP TABLE core_summary_test; diff --git a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql index b6023498412..802bda5684c 100644 --- a/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql +++ b/contrib/ext_vacuum_statistics/ext_vacuum_statistics--1.0.sql @@ -31,7 +31,8 @@ CREATE SCHEMA IF NOT EXISTS ext_vacuum_statistics; COMMENT ON SCHEMA ext_vacuum_statistics IS 'Extended vacuum statistics (heap, index, database)'; --- Reset functions +-- These resets leave native VACUUM work counters untouched. +-- Local reset functions; use the gp_ wrappers below for a cluster-wide reset. CREATE OR REPLACE FUNCTION ext_vacuum_statistics.extvac_reset_entry( dboid oid, relid oid @@ -202,3 +203,220 @@ LEFT JOIN LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_database(db.oid) sta COMMENT ON VIEW ext_vacuum_statistics.pg_stats_vacuum_database IS 'Extended vacuum statistics per database (aggregate)'; + +-- +-- Cloudberry: cluster-wide views. +-- +-- Every instance of the cluster keeps the statistics of the vacuums it runs +-- itself, and the pg_stats_vacuum_* views above only show those of the +-- instance they are queried on. The gp_stats_vacuum_* views show the rows +-- of all instances, with the gp_segment_id of each (-1 for the +-- coordinator), like the gp_stat_* views of the core. +-- +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_tables AS +SELECT gp_execution_segment() AS gp_segment_id, * + FROM gp_dist_random('ext_vacuum_statistics.pg_stats_vacuum_tables') +UNION ALL +SELECT -1 AS gp_segment_id, * + FROM ext_vacuum_statistics.pg_stats_vacuum_tables; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_tables IS + 'Extended vacuum statistics per table, on every instance of the cluster'; + +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_indexes AS +SELECT gp_execution_segment() AS gp_segment_id, * + FROM gp_dist_random('ext_vacuum_statistics.pg_stats_vacuum_indexes') +UNION ALL +SELECT -1 AS gp_segment_id, * + FROM ext_vacuum_statistics.pg_stats_vacuum_indexes; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_indexes IS + 'Extended vacuum statistics per index, on every instance of the cluster'; + +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_database AS +SELECT gp_execution_segment() AS gp_segment_id, * + FROM gp_dist_random('ext_vacuum_statistics.pg_stats_vacuum_database') +UNION ALL +SELECT -1 AS gp_segment_id, * + FROM ext_vacuum_statistics.pg_stats_vacuum_database; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_database IS + 'Extended vacuum statistics per database, on every instance of the cluster'; + +-- +-- The *_summary views add the numbers up over the cluster, the way the +-- gp_stat_*_summary views of the core do: user relations are summed over +-- the segments (a replicated table is stored and vacuumed in full on every +-- segment, so its sums are divided by the number of segments), and the +-- system catalogs, which every instance keeps its own copy of, are shown as +-- the coordinator counts them. +-- +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_tables_summary AS +SELECT + s.relid, + s.schema, + s.relname, + s.dbname, + (sum(s.total_blks_read) / s.divisor)::bigint AS total_blks_read, + (sum(s.total_blks_hit) / s.divisor)::bigint AS total_blks_hit, + (sum(s.total_blks_dirtied) / s.divisor)::bigint AS total_blks_dirtied, + (sum(s.total_blks_written) / s.divisor)::bigint AS total_blks_written, + (sum(s.wal_records) / s.divisor)::bigint AS wal_records, + (sum(s.wal_fpi) / s.divisor)::bigint AS wal_fpi, + (sum(s.wal_bytes) / s.divisor) AS wal_bytes, + (sum(s.blk_read_time) / s.divisor) AS blk_read_time, + (sum(s.blk_write_time) / s.divisor) AS blk_write_time, + (sum(s.rel_blks_read) / s.divisor)::bigint AS rel_blks_read, + (sum(s.rel_blks_hit) / s.divisor)::bigint AS rel_blks_hit, + (sum(s.tuples_deleted) / s.divisor)::bigint AS tuples_deleted, + (sum(s.pages_scanned) / s.divisor)::bigint AS pages_scanned, + (sum(s.pages_removed) / s.divisor)::bigint AS pages_removed +FROM ( + SELECT v.*, + CASE WHEN d.policytype = 'r' THEN d.numsegments ELSE 1 END AS divisor + FROM gp_dist_random('ext_vacuum_statistics.pg_stats_vacuum_tables') v + LEFT JOIN (SELECT relid, unnest(ARRAY[segrelid, blkdirrelid, visimaprelid]) AS auxrelid + FROM pg_appendonly) a ON a.auxrelid = v.relid + LEFT JOIN gp_distribution_policy d ON d.localoid = coalesce(a.relid, v.relid) + WHERE v.relid >= 16384 +) s +GROUP BY s.relid, s.schema, s.relname, s.dbname, s.divisor +UNION ALL +SELECT + relid, + schema, + relname, + dbname, + total_blks_read, + total_blks_hit, + total_blks_dirtied, + total_blks_written, + wal_records, + wal_fpi, + wal_bytes, + blk_read_time, + blk_write_time, + rel_blks_read, + rel_blks_hit, + tuples_deleted, + pages_scanned, + pages_removed +FROM ext_vacuum_statistics.pg_stats_vacuum_tables +WHERE relid < 16384; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_tables_summary IS + 'Extended vacuum statistics per table, summed over the cluster'; + +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_indexes_summary AS +SELECT + s.indexrelid, + s.schema, + s.indexrelname, + s.dbname, + (sum(s.total_blks_read) / s.divisor)::bigint AS total_blks_read, + (sum(s.total_blks_hit) / s.divisor)::bigint AS total_blks_hit, + (sum(s.total_blks_dirtied) / s.divisor)::bigint AS total_blks_dirtied, + (sum(s.total_blks_written) / s.divisor)::bigint AS total_blks_written, + (sum(s.wal_records) / s.divisor)::bigint AS wal_records, + (sum(s.wal_fpi) / s.divisor)::bigint AS wal_fpi, + (sum(s.wal_bytes) / s.divisor) AS wal_bytes, + (sum(s.blk_read_time) / s.divisor) AS blk_read_time, + (sum(s.blk_write_time) / s.divisor) AS blk_write_time, + (sum(s.rel_blks_read) / s.divisor)::bigint AS rel_blks_read, + (sum(s.rel_blks_hit) / s.divisor)::bigint AS rel_blks_hit, + (sum(s.tuples_deleted) / s.divisor)::bigint AS tuples_deleted, + (sum(s.pages_deleted) / s.divisor)::bigint AS pages_deleted +FROM ( + SELECT v.*, + CASE WHEN d.policytype = 'r' THEN d.numsegments ELSE 1 END AS divisor + FROM gp_dist_random('ext_vacuum_statistics.pg_stats_vacuum_indexes') v + JOIN pg_index i ON i.indexrelid = v.indexrelid + LEFT JOIN (SELECT relid, unnest(ARRAY[segrelid, blkdirrelid, visimaprelid]) AS auxrelid + FROM pg_appendonly) a ON a.auxrelid = i.indrelid + LEFT JOIN gp_distribution_policy d ON d.localoid = coalesce(a.relid, i.indrelid) + WHERE v.indexrelid >= 16384 +) s +GROUP BY s.indexrelid, s.schema, s.indexrelname, s.dbname, s.divisor +UNION ALL +SELECT + indexrelid, + schema, + indexrelname, + dbname, + total_blks_read, + total_blks_hit, + total_blks_dirtied, + total_blks_written, + wal_records, + wal_fpi, + wal_bytes, + blk_read_time, + blk_write_time, + rel_blks_read, + rel_blks_hit, + tuples_deleted, + pages_deleted +FROM ext_vacuum_statistics.pg_stats_vacuum_indexes +WHERE indexrelid < 16384; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_indexes_summary IS + 'Extended vacuum statistics per index, summed over the cluster'; + +-- The database aggregates are summed over all instances, the coordinator +-- included: they are the vacuum work done in the database cluster-wide. +CREATE VIEW ext_vacuum_statistics.gp_stats_vacuum_database_summary AS +SELECT + dboid, + dbname, + sum(db_blks_read)::bigint AS db_blks_read, + sum(db_blks_hit)::bigint AS db_blks_hit, + sum(db_blks_dirtied)::bigint AS db_blks_dirtied, + sum(db_blks_written)::bigint AS db_blks_written, + sum(db_wal_records)::bigint AS db_wal_records, + sum(db_wal_fpi)::bigint AS db_wal_fpi, + sum(db_wal_bytes) AS db_wal_bytes, + sum(db_blk_read_time) AS db_blk_read_time, + sum(db_blk_write_time) AS db_blk_write_time +FROM ext_vacuum_statistics.gp_stats_vacuum_database +GROUP BY dboid, dbname; + +COMMENT ON VIEW ext_vacuum_statistics.gp_stats_vacuum_database_summary IS + 'Extended vacuum statistics per database, summed over the cluster'; + +-- +-- Cloudberry: resetting on the whole cluster. The reset functions above act +-- on the instance they are called on; these run them on the coordinator and +-- on every primary segment. Call these wrappers from a normal coordinator +-- connection; use the local functions in utility mode. +-- +CREATE FUNCTION ext_vacuum_statistics.gp_vacuum_statistics_reset() +RETURNS void +AS $$ + SELECT ext_vacuum_statistics.vacuum_statistics_reset() FROM gp_dist_random('gp_id'); + SELECT ext_vacuum_statistics.vacuum_statistics_reset(); +$$ LANGUAGE sql VOLATILE; + +CREATE FUNCTION ext_vacuum_statistics.gp_extvac_reset_entry(dboid oid, relid oid) +RETURNS void +AS $$ + SELECT ext_vacuum_statistics.extvac_reset_entry(dboid, relid) FROM gp_dist_random('gp_id'); + SELECT ext_vacuum_statistics.extvac_reset_entry(dboid, relid); +$$ LANGUAGE sql STRICT VOLATILE; + +CREATE FUNCTION ext_vacuum_statistics.gp_extvac_reset_db_entry(dboid oid) +RETURNS void +AS $$ + SELECT ext_vacuum_statistics.extvac_reset_db_entry(dboid) FROM gp_dist_random('gp_id'); + SELECT ext_vacuum_statistics.extvac_reset_db_entry(dboid); +$$ LANGUAGE sql STRICT VOLATILE; + +COMMENT ON FUNCTION ext_vacuum_statistics.gp_extvac_reset_entry(oid, oid) IS + 'Reset extension-owned vacuum metrics for one table or index on the coordinator and all primary segments; call on the coordinator'; +COMMENT ON FUNCTION ext_vacuum_statistics.gp_extvac_reset_db_entry(oid) IS + 'Reset extension-owned vacuum metrics for a database and its relations on the coordinator and all primary segments; call on the coordinator'; +COMMENT ON FUNCTION ext_vacuum_statistics.gp_vacuum_statistics_reset() IS + 'Reset extension-owned vacuum metrics for all databases on the coordinator and all primary segments; call on the coordinator'; + +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.gp_extvac_reset_entry(oid, oid) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.gp_extvac_reset_db_entry(oid) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION ext_vacuum_statistics.gp_vacuum_statistics_reset() FROM PUBLIC; diff --git a/contrib/ext_vacuum_statistics/sql/gp_vacuum_statistics.sql b/contrib/ext_vacuum_statistics/sql/gp_vacuum_statistics.sql new file mode 100644 index 00000000000..3ab6a0f58df --- /dev/null +++ b/contrib/ext_vacuum_statistics/sql/gp_vacuum_statistics.sql @@ -0,0 +1,206 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Cluster-wide vacuum statistics: aggregation, collection control, reset, +-- and cleanup after DROP. The final two cases cover core summary views +-- and local scope of track_cost_delay_timing. +-- +-- This test runs against a Cloudberry cluster that has ext_vacuum_statistics +-- in shared_preload_libraries of every instance ("make installcheck-cluster"). +-- +CREATE EXTENSION IF NOT EXISTS ext_vacuum_statistics; +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); + +SELECT oid AS dboid FROM pg_database WHERE datname = current_database() \gset + +-- Prepare dead tuples: 10000 in a distributed table and 300 copies on +-- each segment in a replicated table. VACUUM also cleans the index. +CREATE TABLE gpvs_dist (id int, v text) WITH (autovacuum_enabled = off) + DISTRIBUTED BY (id); +CREATE INDEX gpvs_dist_idx ON gpvs_dist (id); +INSERT INTO gpvs_dist SELECT g, 'x' FROM generate_series(1, 10000) g; +CREATE TABLE gpvs_repl (id int) WITH (autovacuum_enabled = off) + DISTRIBUTED REPLICATED; +INSERT INTO gpvs_repl SELECT generate_series(1, 300); +DELETE FROM gpvs_dist; +DELETE FROM gpvs_repl; +VACUUM gpvs_dist, gpvs_repl; + +-- Collection: expect an entry on every primary segment and the coordinator. +-- role = 'p' includes the coordinator and excludes mirrors. +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration + WHERE role = 'p') AS one_row_per_instance + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + +-- Expect 10000 removed tuples across the segments and 0 on the coordinator. +SELECT gp_segment_id = -1 AS coordinator, sum(tuples_deleted) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' + GROUP BY 1 ORDER BY 1; + +-- Aggregation: expect 10000 for the distributed table and 300 for the +-- replicated table. Replicated counts must be averaged across segments. +SELECT relname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + +-- The index summary must also report 10000 removed tuples. +SELECT indexrelname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_indexes_summary + WHERE indexrelname = 'gpvs_dist_idx'; + +-- WAL volume varies, so compare the summary with the segment counters. +SELECT s.wal_records = (SELECT sum(wal_records) + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' AND gp_segment_id >= 0) + AS summary_matches_segments + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary s + WHERE s.relname = 'gpvs_dist'; + +-- Database WAL includes these tables plus indexes and other relations; +-- it must be positive and at least as large as the table WAL total. +SELECT db_wal_records > 0 AS database_has_wal, + db_wal_records >= (SELECT sum(wal_records) + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname LIKE 'gpvs%') AS database_covers_tables + FROM ext_vacuum_statistics.gp_stats_vacuum_database_summary + WHERE dbname = current_database(); + +-- Core table summaries must also record a positive vacuum duration. +SELECT relname, total_vacuum_time > 0 AS vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + +-- Collection control: disable collection on the coordinator, then vacuum +-- another 1000 dead tuples. Native work grows; resource counters stay unchanged. +CREATE TEMP TABLE gpvs_resources_before AS + SELECT gp_segment_id AS segid, total_blks_read, total_blks_hit, wal_records + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist' + DISTRIBUTED BY (segid); +INSERT INTO gpvs_dist SELECT g, 'y' FROM generate_series(1, 1000) g; +DELETE FROM gpvs_dist; +SET vacuum_statistics.enabled = off; +VACUUM gpvs_dist; +RESET vacuum_statistics.enabled; +SELECT bool_and(s.total_blks_read = b.total_blks_read + AND s.total_blks_hit = b.total_blks_hit + AND s.wal_records = b.wal_records) AS resources_unchanged + FROM ext_vacuum_statistics.gp_stats_vacuum_tables s + JOIN gpvs_resources_before b ON b.segid = s.gp_segment_id + WHERE s.relname = 'gpvs_dist'; +DROP TABLE gpvs_resources_before; +SELECT bool_and(e.tuples_deleted = n.tuples_deleted) AS native_work_matches + FROM ext_vacuum_statistics.gp_stats_vacuum_tables e + JOIN gp_stat_vacuum_tables n USING (gp_segment_id, relid) + WHERE e.relname = 'gpvs_dist'; +SELECT sum(tuples_deleted) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + +-- Relation reset clears extension resources across the cluster, preserving +-- native work for gpvs_dist and gpvs_repl. +SELECT ext_vacuum_statistics.gp_extvac_reset_entry(:dboid, 'gpvs_dist'::regclass); +SELECT bool_and(total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0) + AS resources_reset + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; +SELECT coalesce(sum(tuples_deleted), 0) AS tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; +SELECT relname, tuples_deleted + FROM ext_vacuum_statistics.gp_stats_vacuum_tables_summary + WHERE relname LIKE 'gpvs%' + ORDER BY relname; + +-- DROP cleanup: save the OID and read the statistics directly. The public +-- views join pg_class, so they would hide an orphaned entry after DROP. +-- gp_id makes this helper view executable on each segment via gp_dist_random. +-- Expect 3 segment entries and 1 coordinator entry in the regression cluster. +SELECT 'gpvs_repl'::regclass::oid AS repl_oid \gset +CREATE VIEW gpvs_repl_entry AS + SELECT s.relid + FROM gp_id, + LATERAL ext_vacuum_statistics.pg_stats_get_vacuum_tables(:dboid, :repl_oid) s; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); +SELECT count(*) AS coordinator_entries FROM gpvs_repl_entry; +-- Rolling back DROP must preserve the 3 segment entries. +BEGIN; +DROP TABLE gpvs_repl; +ROLLBACK; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); +-- Committing DROP must remove entries from both segments and coordinator. +DROP TABLE gpvs_repl; +SELECT count(*) AS segment_entries FROM gp_dist_random('gpvs_repl_entry'); +SELECT count(*) AS coordinator_entries FROM gpvs_repl_entry; + +-- Core database timing is published asynchronously. Allow pending stats +-- to be flushed before checking that the vacuum duration is positive. +SELECT pg_sleep(2); +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); + +-- Cluster reset must preserve the remaining table's entries and zero +-- its resources on every instance, preserving native work. +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); +SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p') AS rows_preserved, + bool_and(total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0) AS resources_reset, + sum(tuples_deleted) = 11000 AS work_preserved + FROM ext_vacuum_statistics.gp_stats_vacuum_tables + WHERE relname = 'gpvs_dist'; + +DROP VIEW gpvs_repl_entry; +DROP TABLE gpvs_dist; + +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +RESET track_cost_delay_timing; + +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +VACUUM core_summary_test; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; +DROP TABLE core_summary_test; diff --git a/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl index 83068de1992..0158bc436fd 100644 --- a/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl +++ b/contrib/ext_vacuum_statistics/t/060_vacuum_stats_reset.pl @@ -45,7 +45,8 @@ GRANT USAGE ON SCHEMA ext_vacuum_statistics TO reset_reader; }); for my $call ('extvac_reset_entry(0, 0)', 'extvac_reset_db_entry(0)', - 'vacuum_statistics_reset()') + 'vacuum_statistics_reset()', 'gp_extvac_reset_entry(0, 0)', + 'gp_extvac_reset_db_entry(0)', 'gp_vacuum_statistics_reset()') { my ($stdout, $stderr); my $ret = $node->psql('postgres', diff --git a/contrib/ext_vacuum_statistics/t/064_native_work_source.pl b/contrib/ext_vacuum_statistics/t/064_native_work_source.pl index 02cdd2a4286..3c5f9b013cb 100644 --- a/contrib/ext_vacuum_statistics/t/064_native_work_source.pl +++ b/contrib/ext_vacuum_statistics/t/064_native_work_source.pl @@ -31,7 +31,7 @@ 'native_heap'::regclass) }), '0', 'disabled first vacuum creates no extension resource entry'); is($node->safe_psql('postgres', q{ -SELECT tuples_deleted = 100 AND pages_scanned > 0 AND tuples_frozen = 900 +SELECT tuples_deleted = 100 AND pages_scanned > 0 AND total_blks_read = 0 AND total_blks_hit = 0 AND wal_records = 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'native_heap' @@ -42,40 +42,19 @@ WHERE indexrelname = 'native_heap_pkey' }), 't', 'index view exposes native work without a resource entry'); -for my $orientation ('row', 'column') -{ - $node->safe_psql('postgres', qq{ -CREATE TABLE native_$orientation (id int) - WITH (appendonly = true, orientation = $orientation); -INSERT INTO native_$orientation SELECT generate_series(1, 1000); -DELETE FROM native_$orientation WHERE id <= 100; -SET gp_appendonly_compaction = off; -VACUUM native_$orientation; -VACUUM native_$orientation; -SELECT pg_stat_force_next_flush(); -}); - is($node->safe_psql('postgres', qq{ -SELECT recently_dead_tuples FROM ext_vacuum_statistics.pg_stats_vacuum_tables - WHERE relname = 'native_$orientation' -}), '200', "$orientation AO shows the native cumulative hidden-row count"); -} - -my @table_fields = qw(tuples_deleted pages_scanned pages_removed tuples_frozen - recently_dead_tuples missed_dead_pages missed_dead_tuples pages_frozen - pages_all_visible total_file_segs compacted_segments tuples_moved - dead_pages freeze_age_vacuum_count); -my @index_fields = qw(tuples_deleted pages_deleted dead_pages); +my @table_fields = qw(tuples_deleted pages_scanned pages_removed); +my @index_fields = qw(tuples_deleted pages_deleted); for my $consistency ('none', 'cache', 'snapshot') { my $tables_match = join(' AND ', map { "e.$_ = n.$_" } @table_fields); my $indexes_match = join(' AND ', map { "e.$_ = n.$_" } @index_fields); is($node->safe_psql('postgres', qq{ SET stats_fetch_consistency = '$consistency'; -SELECT count(*) = 3 AND bool_and($tables_match) +SELECT count(*) = 1 AND bool_and($tables_match) FROM ext_vacuum_statistics.pg_stats_vacuum_tables e CROSS JOIN LATERAL pg_stat_get_vacuum_stats(e.relid) n - WHERE e.relname IN ('native_heap', 'native_row', 'native_column'); -}), 't', "$consistency: all heap and AO work fields match native counters"); + WHERE e.relname = 'native_heap'; +}), 't', "$consistency: all heap work fields match native counters"); is($node->safe_psql('postgres', qq{ SET stats_fetch_consistency = '$consistency'; SELECT $indexes_match @@ -137,7 +116,7 @@ $node->safe_psql('postgres', q[SELECT pg_stat_reset_vacuum_stats('native_heap'::regclass)]); is($node->safe_psql('postgres', q{ -SELECT tuples_deleted = 0 AND pages_scanned = 0 AND tuples_frozen = 0 +SELECT tuples_deleted = 0 AND pages_scanned = 0 FROM ext_vacuum_statistics.pg_stats_vacuum_tables WHERE relname = 'native_heap' }), 't', 'native reset is visible in the extension work columns'); is($node->safe_psql('postgres', $resources), $resources_before, diff --git a/doc/src/sgml/extvacuumstatistics.sgml b/doc/src/sgml/extvacuumstatistics.sgml index 8201af36894..c8ca317414e 100644 --- a/doc/src/sgml/extvacuumstatistics.sgml +++ b/doc/src/sgml/extvacuumstatistics.sgml @@ -116,7 +116,10 @@ ext_vacuum_statistics.pg_stats_get_vacuum_tables(dboid, reloid), ext_vacuum_statistics.pg_stats_get_vacuum_indexes(dboid, reloid) and ext_vacuum_statistics.pg_stats_get_vacuum_database(dboid) - functions. Missing statistics produce no rows. Extension-specific resets + functions for resource metrics. Missing resource entries produce zero + resource counters in the table and index views. Work counters are read + from pg_stat_get_vacuum_stats() and remain visible + when resource collection is disabled. Extension-specific resets do not clear native VACUUM work counters. pg_stat_reset() also clears the extension's counters for the current database. @@ -149,11 +152,30 @@ - Reset functions operate only on the current node. Calling one on the - coordinator (QD) leaves the segment counters unchanged. They can also be - used in a utility connection to an individual segment. + For Cloudberry, use the gp_ wrappers from a normal + coordinator connection to reset statistics across the cluster. These + wrappers reset the coordinator and all primary segments. For example, + to reset vacuum statistics for all databases across the cluster: + +SELECT ext_vacuum_statistics.gp_vacuum_statistics_reset(); + + Delegating access to a cluster wrapper also requires granting access to + its corresponding local reset function. + + + Functions without the gp_ prefix reset statistics + only on the current node. Calling + ext_vacuum_statistics.vacuum_statistics_reset() + on the coordinator (QD) resets only the coordinator's counters; + segment counters remain unchanged. The same applies to + ext_vacuum_statistics.extvac_reset_entry() and + ext_vacuum_statistics.extvac_reset_db_entry(). + Use the local functions in a utility connection to an individual segment. + + + @@ -195,7 +217,196 @@ + + + + ext_vacuum_statistics.gp_vacuum_statistics_reset() + void + + + + Calls vacuum_statistics_reset() on the coordinator + and all primary segments, resetting statistics for all databases. + + + + + + + ext_vacuum_statistics.gp_extvac_reset_entry(dboid oid, relid oid) + void + + + + Calls extvac_reset_entry(dboid, relid) + on the coordinator and all primary segments, resetting statistics for + one table or index. The database aggregate and other relations are + unchanged. + + + + + + + ext_vacuum_statistics.gp_extvac_reset_db_entry(dboid oid) + void + + + + Calls extvac_reset_db_entry(dboid) + on the coordinator and all primary segments, resetting the database + aggregate and the statistics of all its tables and indexes. + + + + + + To reset the current database across the cluster, run on the coordinator: + +SELECT ext_vacuum_statistics.gp_extvac_reset_db_entry(oid) +FROM pg_database WHERE datname = current_database(); + + + + + + Using the Module in a Cloudberry Cluster + + + Every instance of the cluster, the coordinator and each segment, keeps + the statistics of the vacuums it runs itself: a VACUUM + issued on the coordinator is dispatched to the segments, and each segment + vacuums and counts its own part of the tables. The + pg_stats_vacuum_* views show the statistics of + the instance they are queried on, so on the coordinator they show only + what the coordinator itself vacuumed, which for distributed tables is next + to nothing. The module therefore also provides cluster-wide views, which + query every instance: + + + + + + ext_vacuum_statistics.gp_stats_vacuum_tables, + ext_vacuum_statistics.gp_stats_vacuum_indexes and + ext_vacuum_statistics.gp_stats_vacuum_database + have the columns of the corresponding pg_stats_vacuum_* + view, preceded by gp_segment_id: one row per + instance, -1 being the coordinator, like the + gp_stat_* views of the core. + + + + + ext_vacuum_statistics.gp_stats_vacuum_tables_summary + and ext_vacuum_statistics.gp_stats_vacuum_indexes_summary + have one row per relation, like the + gp_stat_*_summary views of the core: the + counters of user relations are summed over the segments, divided by the + number of segments for replicated tables, which every segment stores and + vacuums in full; system catalogs are shown as the coordinator counts + them. + + + + + ext_vacuum_statistics.gp_stats_vacuum_database_summary + sums the database aggregates over all instances, the coordinator + included. + + + + + + Like the other gp_* views, the cluster-wide views + must not be queried in utility mode, where they read the local instance + twice. + + + + The reset functions without the gp_ prefix act only on + the connected instance. + ext_vacuum_statistics.gp_vacuum_statistics_reset(), + ext_vacuum_statistics.gp_extvac_reset_entry(dboid, relid) + and + ext_vacuum_statistics.gp_extvac_reset_db_entry(dboid) + call them on the coordinator and on every primary segment. Call these + wrappers from a normal coordinator connection; use the local functions + in utility mode. See . + + + + Install the matching extension library, control and SQL files in the + Cloudberry installation on every host, including mirror and standby + coordinator hosts. gpconfig updates configuration files; + it does not distribute the extension files. The module must be preloaded + on every instance so that it is available after promotion. + + + + Inspect the active setting and the configuration files: + +gpconfig -s shared_preload_libraries +gpconfig -s shared_preload_libraries --file + + If the existing list is empty, set it as follows: + +gpconfig -c shared_preload_libraries -v 'ext_vacuum_statistics' + + Otherwise pass the complete existing list with + ext_vacuum_statistics appended. Preserve other preloaded + libraries and any intentional differences between coordinator, primary + and mirror settings. With and no role restriction, + gpconfig updates the coordinator, standby, primaries and + mirrors. Resolve any reported unreachable hosts before relying on their + configuration. Check the resulting files and restart the cluster: + +gpconfig -s shared_preload_libraries --file +gpstop -ar +gpconfig -s shared_preload_libraries --file-compare + + A reload with gpstop -u alone cannot load a new preload + library. Check that the mirror and standby instances also restarted + successfully. Then run + CREATE EXTENSION IF NOT EXISTS ext_vacuum_statistics + on the coordinator in each database where the views are needed; this + command is dispatched to the segments. + + + + The warning above about starting without the module applies separately + to each instance that has saved statistics of its custom kinds. Keep the + module preloaded when disabling collection to preserve that instance's + cumulative statistics. + + + + The counters are local pgstat state, not replicated + through WAL. Preloading the module on a mirror enables collection after + promotion but does not copy its primary's accumulated counters. A promoted + instance reports its own local statistics and cannot continue the former + primary's counter history. This also applies to standby coordinator + promotion. A compatible clean restart can preserve an instance's own + statistics; crash recovery resets them. + + + + Start a new measurement interval for an instance after promotion, even + when its gp_segment_id is unchanged. Other + instances keep their counters, but cluster summaries can decrease when + a primary is replaced. Monitoring should retain earlier samples externally + and establish a new baseline after failover; the discontinuity does not + represent negative vacuum work. + + + + vacuum_statistics.enabled is read by the instance that + runs the vacuum. A SET on the coordinator is passed on + to the segments of the session; to stop collecting extension resources in the whole cluster, + set it with gpconfig and reload the configuration + (gpstop -u). + From 3011f9650605d40616f5f2b782917d206ebbaaef Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Fri, 9 Oct 2026 20:48:37 +0300 Subject: [PATCH 50/50] Test native VACUUM cluster timing without the extension Move cluster timing, index-summary cardinality, and track_cost_delay_timing scope checks from the extension regression into the core vacuum_stats test. Cover distributed and replicated tables, and explicitly flush database statistics before checking their timing aggregate. --- src/test/regress/expected/vacuum_stats.out | 86 ++++++++++++++++++++++ src/test/regress/sql/vacuum_stats.sql | 48 ++++++++++++ 2 files changed, 134 insertions(+) diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index ea677f0c3d6..8b302003e8a 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -109,3 +109,89 @@ SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map (1 row) DROP TABLE frozen_stats_repl; +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +RESET track_cost_delay_timing; +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +CREATE TABLE core_timing_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO core_timing_repl SELECT generate_series(1, 1000); +DELETE FROM core_timing_repl; +VACUUM core_summary_test, core_timing_repl; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; + summary_indexes | distinct_indexes +-----------------+------------------ + 2 | 2 +(1 row) + +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; + index_totals_match +-------------------- + t +(1 row) + +-- Native timing summaries work without a statistics extension. +SELECT count(*) = 2 AND bool_and(total_vacuum_time > 0) AS tables_vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname IN ('core_summary_test', 'core_timing_repl'); + tables_vacuum_timed +--------------------- + t +(1 row) + +-- Flush database timing before checking its cluster aggregate. +SELECT gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); + database_vacuum_timed +----------------------- + t +(1 row) + +DROP TABLE core_summary_test, core_timing_repl; diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index 084ff4d0dbf..83620a49294 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -72,3 +72,51 @@ VACUUM (FULL) frozen_stats_repl; SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; DROP TABLE frozen_stats_repl; + +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +RESET track_cost_delay_timing; + +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +CREATE TABLE core_timing_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO core_timing_repl SELECT generate_series(1, 1000); +DELETE FROM core_timing_repl; +VACUUM core_summary_test, core_timing_repl; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; +-- Native timing summaries work without a statistics extension. +SELECT count(*) = 2 AND bool_and(total_vacuum_time > 0) AS tables_vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname IN ('core_summary_test', 'core_timing_repl'); +-- Flush database timing before checking its cluster aggregate. +SELECT gp_stat_force_next_flush(); +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); +DROP TABLE core_summary_test, core_timing_repl;