diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c index 2340d49e00a..6a6609be16f 100644 --- a/contrib/bloom/blvacuum.c +++ b/contrib/bloom/blvacuum.c @@ -61,7 +61,7 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, *itupPtr, *itupEnd; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); @@ -191,7 +191,7 @@ blvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) Buffer buffer; Page page; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index b5ffde4131b..59fcc8ae729 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -1197,7 +1197,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, for (;;) { /* Check for user-requested abort or sleep */ - vacuum_delay_point(); + vacuum_delay_point(true); /* Fetch next row */ MemoryContextReset(tupcontext); diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 9b498953f45..45102c5d654 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -2061,6 +2061,26 @@ SCRAM-SHA-256$<iteration count>:&l + + + relallfrozen int4 + + + Number of pages that are marked all-frozen in the table's visibility + map. This is only an estimate and can be used along with + relallvisible for scheduling vacuums and + tuning vacuum's freezing + behavior. + + It is updated by + VACUUM, + ANALYZE, + and a few DDL commands such as + CREATE INDEX. + + + + reltoastrelid oid diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 3b8b60d9347..1873cf1f972 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -8155,6 +8155,30 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + track_cost_delay_timing (boolean) + + track_cost_delay_timing configuration parameter + + + + + Enables timing of cost-based vacuum delay (see + ). This parameter + is off by default, as it will repeatedly query the operating system for + the current time, which may cause significant overhead on some + platforms. You can use the tool to + measure the overhead of timing on your system. Cost-based vacuum delay + timing information is displayed in + pg_stat_progress_vacuum + and + pg_stat_progress_analyze. + Only superusers and users with the appropriate SET + privilege can change this setting. + + + + track_io_timing (boolean) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f9ceead292f..6c36323b843 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -459,6 +459,12 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_vacuum + One row per database, plus one for shared objects, showing cumulative + vacuum statistics. See . + + pg_stat_database_conflictspg_stat_database_conflicts @@ -4697,6 +4703,137 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + <structname>pg_stat_vacuum</structname> + pg_stat_vacuum + + The pg_stat_vacuum view contains one row for each + database, plus one for shared objects, showing cumulative vacuum statistics + on the current instance. The current database's counters are reset + together with its other statistics by pg_stat_reset(). + + + In Cloudberry, gp_stat_vacuum shows these columns + for every instance, with gp_segment_id identifying + the instance. gp_stat_vacuum_summary sums the times + across instances and takes the maximum failsafe and interrupt counts and + the latest reset time for each database. + + + <structname>pg_stat_vacuum</structname> View + + + + + Column Type + + + Description + + + + + + + datid oid + + + OID of this database, or 0 for objects belonging to a shared + relation + + + + + + datname name + + + Name of this database, or NULL for shared + objects. + + + + + + total_vacuum_time double precision + + + Total time manually invoked vacuums spent processing tables of this + database, in milliseconds (index processing is included in the owning + table's run) + + + + + + total_autovacuum_time double precision + + + Total time the autovacuum daemon spent processing tables of this + database, in milliseconds + + + + + + total_vacuum_delay_time double precision + + + Total time manually invoked vacuums of this database spent sleeping in + cost-based delay points, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time autovacuums of this database spent sleeping in cost-based + delay points, in milliseconds (counted only when + is enabled) + + + + + + vacuum_failsafe_count bigint + + + Number of vacuums in this database that entered the wraparound + failsafe mode (see ) + + + + + + vacuum_interrupt_count bigint + + + Number of times a vacuum in this database was interrupted by an error, + the vacuum of an append-optimized table included. In a Cloudberry + cluster the count is kept by the instance on which the vacuum failed: + a vacuum dispatched to the segments that fails there is counted by + those segments, not by the coordinator. A segment reports the count + once its part of the failed distributed transaction has ended, which + may be as late as the next command of the session. + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset + + + + +
+ + <structname>pg_stat_database_conflicts</structname> @@ -4926,6 +5063,62 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + total_vacuum_time double precision + + + Total time this index has been processed by manually invoked vacuums, + in milliseconds. (This includes the time spent sleeping due to + cost-based delays.) + + + + + + total_autovacuum_time double precision + + + Total time this index has been processed by the autovacuum daemon, + in milliseconds. (This includes the time spent sleeping due to + cost-based delays.) + + + + + + total_vacuum_delay_time double precision + + + Total time manually invoked vacuums spent sleeping in cost-based delay + points while processing this index, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time the autovacuum daemon spent sleeping in cost-based delay + points while processing this index, in milliseconds (counted only when + is enabled) + + + + + + vacuum_failsafe_count bigint + + + Number of vacuums of this table that entered the wraparound failsafe + mode (see ). Always zero for + append-optimized tables, which have no tuple transaction IDs to + freeze. + + + n_tup_ins bigint @@ -5017,6 +5210,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + + + + visible_page_marks_cleared bigint + + + Number of times the all-visible mark in the + visibility map was cleared for + pages of this table. The all-visible mark of a heap page is + cleared whenever a backend process modifies a page that was + previously marked all-visible by vacuum activity (whether manual + VACUUM or autovacuum). The page must then be + processed again by vacuum on a subsequent run. A high rate of + change in this counter means that vacuum has to repeatedly + re-process pages of this table. Always zero for append-optimized + tables, which have no visibility map. + + + + + + frozen_page_marks_cleared bigint + + + Number of times the all-frozen mark in the + visibility map was cleared for + pages of this table. The all-frozen mark of a heap page is cleared + whenever a backend process modifies a page that was previously + marked all-frozen by vacuum activity (manual VACUUM + or autovacuum). The page must then be processed again by vacuum on + the next freeze run for this table. Always zero for append-optimized + tables. + + + last_vacuum timestamp with time zone @@ -5094,6 +5323,66 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i daemon + + + + total_vacuum_time double precision + + + Total time this table has been manually vacuumed, in milliseconds + + + + + + total_autovacuum_time double precision + + + Total time this table has been vacuumed by the autovacuum daemon, + in milliseconds + + + + + + total_analyze_time double precision + + + Total time this table has been manually analyzed, in milliseconds + + + + + + total_autoanalyze_time double precision + + + Total time this table has been analyzed by the autovacuum daemon, + in milliseconds + + + + + + total_vacuum_delay_time double precision + + + Total time this table's vacuums spent sleeping in cost-based delay + points, in milliseconds (counted only when + is enabled) + + + + + + total_autovacuum_delay_time double precision + + + Total time autovacuums of this table spent sleeping in cost-based + delay points, in milliseconds (counted only when + is enabled) + + @@ -6496,6 +6785,18 @@ FROM pg_stat_get_backend_idset() AS backendid; acquiring inherited sample rows. + + + + delay_time double precision + + + Total time spent sleeping due to cost-based delay (see + , in milliseconds + (if is enabled, otherwise + zero). + + @@ -7377,6 +7678,18 @@ FROM pg_stat_get_backend_idset() AS backendid; Number of dead tuples collected since the last index vacuum cycle. + + + + delay_time double precision + + + Total time spent sleeping due to cost-based delay (see + ), in milliseconds + (if is enabled, otherwise + zero). + + diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index 12df08f3d67..17ddd8e060d 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -323,7 +323,7 @@ AOCSSegmentFileFullCompaction(Relation aorel, tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { - vacuum_delay_point(); + vacuum_delay_point(false); } /* diff --git a/src/backend/access/aocs/aocsam_handler.c b/src/backend/access/aocs/aocsam_handler.c index 4dc7d541c5f..0256c1f4e00 100644 --- a/src/backend/access/aocs/aocsam_handler.c +++ b/src/backend/access/aocs/aocsam_handler.c @@ -1661,7 +1661,7 @@ aoco_acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, { aocoscan->targrow = RowSampler_Next(&rs); - vacuum_delay_point(); + vacuum_delay_point(true); if (aocs_get_target_tuple(aocoscan, aocoscan->targrow, slot)) { diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index b8cae6ebf6c..c37120be0c3 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -506,7 +506,7 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { - vacuum_delay_point(); + vacuum_delay_point(false); } } diff --git a/src/backend/access/appendonly/appendonlyam_handler.c b/src/backend/access/appendonly/appendonlyam_handler.c index cae12e110ff..7a7942c5e24 100644 --- a/src/backend/access/appendonly/appendonlyam_handler.c +++ b/src/backend/access/appendonly/appendonlyam_handler.c @@ -1544,7 +1544,7 @@ appendonly_acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, { aoscan->targrow = RowSampler_Next(&rs); - vacuum_delay_point(); + vacuum_delay_point(true); if (appendonly_get_target_tuple(aoscan, aoscan->targrow, slot)) { diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c index 2c23afe3a03..65b7f7ef3b7 100644 --- a/src/backend/access/gin/ginfast.c +++ b/src/backend/access/gin/ginfast.c @@ -891,7 +891,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, */ processPendingPage(&accum, &datums, page, FirstOffsetNumber); - vacuum_delay_point(); + vacuum_delay_point(false); /* * Is it time to flush memory to disk? Flush if we are at the end of @@ -928,7 +928,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, { ginEntryInsert(ginstate, attnum, key, category, list, nlist, NULL); - vacuum_delay_point(); + vacuum_delay_point(false); } /* @@ -1001,7 +1001,7 @@ ginInsertCleanup(GinState *ginstate, bool full_clean, /* * Read next page in pending list */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBuffer(index, blkno); LockBuffer(buffer, GIN_SHARE); page = BufferGetPage(buffer); diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index e5d310d8362..067224ab475 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -663,12 +663,12 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, UnlockReleaseBuffer(buffer); } - vacuum_delay_point(); + vacuum_delay_point(false); for (i = 0; i < nRoot; i++) { ginVacuumPostingTree(&gvs, rootOfPostingTree[i]); - vacuum_delay_point(); + vacuum_delay_point(false); } if (blkno == InvalidBlockNumber) /* rightmost page */ @@ -749,7 +749,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) Buffer buffer; Page page; - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 3f60d3274d2..f6608b9c3cf 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -283,7 +283,7 @@ gistvacuumpage(GistVacState *vstate, BlockNumber blkno, BlockNumber orig_blkno) recurse_to = InvalidBlockNumber; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, info->strategy); diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index e50f117bcb9..bb5fd890d76 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -732,7 +732,7 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, bool retain_pin = false; bool clear_dead_marking = false; - vacuum_delay_point(); + vacuum_delay_point(false); page = BufferGetPage(buf); opaque = HashPageGetOpaque(page); diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 19c69270371..7fd1244ed66 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1974,7 +1974,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, + visibilitymap_clear_rel(relation, ItemPointerGetBlockNumber(&(heaptup->t_self)), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -2350,7 +2350,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -2935,7 +2935,7 @@ heap_delete(Relation relation, ItemPointer tid, { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -3764,7 +3764,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * worthwhile. */ if (PageIsAllVisible(page) && - visibilitymap_clear(relation, block, vmbuffer, + visibilitymap_clear_rel(relation, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; @@ -3998,14 +3998,14 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(buffer), vmbuffer, VISIBILITYMAP_VALID_BITS); } if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf))) { all_visible_cleared_new = true; PageClearAllVisible(BufferGetPage(newbuf)); - visibilitymap_clear(relation, BufferGetBlockNumber(newbuf), + visibilitymap_clear_rel(relation, BufferGetBlockNumber(newbuf), vmbuffer_new, VISIBILITYMAP_VALID_BITS); } @@ -5095,7 +5095,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, /* Clear only the all-frozen bit on visibility map if needed */ if (PageIsAllVisible(page) && - visibilitymap_clear(relation, block, vmbuffer, + visibilitymap_clear_rel(relation, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; @@ -5852,7 +5852,7 @@ heap_lock_updated_tuple_rec(Relation rel, ItemPointer tid, TransactionId xid, &new_xmax, &new_infomask, &new_infomask2); if (PageIsAllVisible(BufferGetPage(buf)) && - visibilitymap_clear(rel, block, vmbuffer, + visibilitymap_clear_rel(rel, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN)) cleared_all_frozen = true; diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e3261b49a6c..0cffad86150 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -335,11 +335,13 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, minmulti_updated; BlockNumber orig_rel_pages, new_rel_pages, - new_rel_allvisible; + new_rel_allvisible, + new_rel_allfrozen; PGRUsage ru0; TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, startwritetime = 0; + double startdelaytime; WalUsage startwalusage = pgWalUsage; BufferUsage startbufferusage = pgBufferUsage; ErrorContextCallback errcallback; @@ -351,7 +353,6 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, if (instrument) { pg_rusage_init(&ru0); - starttime = GetCurrentTimestamp(); if (track_io_timing) { startreadtime = pgStatBlockReadTime; @@ -359,6 +360,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } + /* Used for instrumentation and stats report */ + starttime = GetCurrentTimestamp(); + startdelaytime = VacuumDelayTime; + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); @@ -589,10 +594,18 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * pg_class.relpages to */ new_rel_pages = vacrel->rel_pages; /* After possible rel truncation */ - visibilitymap_count(rel, &new_rel_allvisible, NULL); + visibilitymap_count(rel, &new_rel_allvisible, &new_rel_allfrozen); if (new_rel_allvisible > new_rel_pages) new_rel_allvisible = new_rel_pages; + /* + * An all-frozen block _must_ be all-visible. As such, clamp the count of + * all-frozen blocks to the count of all-visible blocks. This matches the + * clamping of relallvisible above. + */ + if (new_rel_allfrozen > new_rel_allvisible) + new_rel_allfrozen = new_rel_allvisible; + /* * Now actually update rel's pg_class entry. * @@ -601,7 +614,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * scan every page that isn't skipped using the visibility map. */ vac_update_relstats(rel, new_rel_pages, vacrel->new_live_tuples, - new_rel_allvisible, vacrel->nindexes > 0, + new_rel_allvisible, new_rel_allfrozen, + vacrel->nindexes > 0, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false, true); @@ -615,11 +629,20 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * soon in cases where the failsafe prevented significant amounts of heap * vacuuming. */ + /* + * The delay counter covers the whole heap_vacuum_rel() run, matching the + * scope of total_vacuum_time. In a parallel vacuum it covers the + * leader's sleeps only; parallel workers account their own sleeps to the + * indexes they process. + */ pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + - vacrel->missed_dead_tuples); + vacrel->missed_dead_tuples, + starttime, + (PgStat_Counter) rint(VacuumDelayTime - startdelaytime), + VacuumFailsafeActive); pgstat_progress_end_command(); if (instrument) @@ -754,8 +777,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, continue; appendStringInfo(&buf, - _("index \"%s\": pages: %u in total, %u newly deleted, %u currently deleted, %u reusable\n"), + _("index \"%s\": tuples: %.0f removed; pages: %u in total, %u newly deleted, %u currently deleted, %u reusable\n"), indnames[i], + istat->tuples_removed, istat->num_pages, istat->pages_newly_deleted, istat->pages_deleted, @@ -911,7 +935,7 @@ lazy_scan_heap(LVRelState *vacrel) update_vacuum_error_info(vacrel, NULL, VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, InvalidOffsetNumber); - vacuum_delay_point(); + vacuum_delay_point(false); /* * Regularly check if wraparound failsafe should trigger. @@ -1151,7 +1175,7 @@ lazy_scan_heap(LVRelState *vacrel) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", vacrel->relname, blkno); - visibilitymap_clear(vacrel->rel, blkno, vmbuffer, + visibilitymap_clear_rel(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1176,7 +1200,7 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); - visibilitymap_clear(vacrel->rel, blkno, vmbuffer, + visibilitymap_clear_rel(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1374,7 +1398,7 @@ lazy_scan_skip(LVRelState *vacrel, Buffer *vmbuffer, BlockNumber next_block, skipsallvis = true; } - vacuum_delay_point(); + vacuum_delay_point(false); next_unskippable_block++; nskippable_blocks++; } @@ -2460,7 +2484,7 @@ lazy_vacuum_heap_rel(LVRelState *vacrel) Page page; Size freespace; - vacuum_delay_point(); + vacuum_delay_point(false); blkno = ItemPointerGetBlockNumber(&vacrel->dead_items->items[index]); vacrel->blkno = blkno; @@ -2734,6 +2758,8 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2759,6 +2785,14 @@ lazy_vacuum_one_index(Relation indrel, IndexBulkDeleteResult *istat, /* Do bulk deletion */ istat = vac_bulkdel_one_index(&ivinfo, istat, (void *) vacrel->dead_items); + /* Accumulate this pass into the index's cumulative vacuum times */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -2783,6 +2817,8 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, { IndexVacuumInfo ivinfo; LVSavedErrInfo saved_err_info; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; ivinfo.index = indrel; ivinfo.heaprel = vacrel->rel; @@ -2808,6 +2844,14 @@ lazy_cleanup_one_index(Relation indrel, IndexBulkDeleteResult *istat, istat = vac_cleanup_one_index(&ivinfo, istat); + /* Accumulate this pass into the index's cumulative vacuum times */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); + /* Revert to the previous phase information for error traceback */ restore_vacuum_error_info(vacrel, &saved_err_info); pfree(vacrel->indname); @@ -3400,7 +3444,7 @@ update_relstats_all_indexes(LVRelState *vacrel) vac_update_relstats(indrel, istat->num_pages, istat->num_index_tuples, - 0, + 0, 0, false, InvalidTransactionId, InvalidMultiXactId, @@ -3419,6 +3463,10 @@ vacuum_error_callback(void *arg) { LVRelState *errinfo = arg; + /* Ignore non-error reports that also invoke this callback. */ + if (errinfo->rel != NULL && geterrlevel() == ERROR) + pgstat_count_vacuum_error(errinfo->rel->rd_rel->relisshared); + switch (errinfo->phase) { case VACUUM_ERRCB_PHASE_SCAN_HEAP: diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 2e18cd88bcf..562c7fdc10f 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -12,6 +12,8 @@ * * INTERFACE ROUTINES * visibilitymap_clear - clear bits for one page in the visibility map + * visibilitymap_clear_rel - clear bits for an open relation, counting the + * clears in its statistics * visibilitymap_pin - pin a map page for setting a bit * visibilitymap_pin_ok - check whether correct map page is already pinned * visibilitymap_set - set a bit in a previously pinned page @@ -91,6 +93,7 @@ #include "access/xloginsert.h" #include "access/xlogutils.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" @@ -125,6 +128,8 @@ * bit pair */ /* prototypes for internal routines */ +static bool vm_do_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, + uint8 flags, bool count_stats); static Buffer vm_readbuf(Relation rel, BlockNumber blkno, bool extend); static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks); @@ -135,9 +140,38 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks); * You must pass a buffer containing the correct map page to this function. * Call visibilitymap_pin first to pin the right one. This function doesn't do * any I/O. Returns true if any bits have been cleared and false otherwise. + * + * This variant is for callers such as recovery, which use a fake relcache + * entry and cannot count the clears in the relation's statistics; callers + * holding an open relation should use visibilitymap_clear_rel() instead. */ bool visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags) +{ + return vm_do_clear(rel, heapBlk, vmbuf, flags, false); +} + +/* + * visibilitymap_clear_rel - visibilitymap_clear() for an open relation + * + * Same as visibilitymap_clear(), but additionally counts the cleared + * all-visible and all-frozen marks in the relation's statistics, which + * makes the stability of its visibility map observable. + */ +bool +visibilitymap_clear_rel(Relation rel, BlockNumber heapBlk, Buffer vmbuf, + uint8 flags) +{ + return vm_do_clear(rel, heapBlk, vmbuf, flags, true); +} + +/* + * Workhorse of visibilitymap_clear() and visibilitymap_clear_rel(). + * count_stats is false when the caller has no real relcache entry at hand. + */ +static bool +vm_do_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags, + bool count_stats) { BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); @@ -162,6 +196,18 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags if (map[mapByte] & mask) { + /* + * Track how often all-visible or all-frozen bits are cleared in the + * visibility map. + */ + if (count_stats) + { + if (map[mapByte] & ((flags & VISIBILITYMAP_ALL_VISIBLE) << mapOffset)) + pgstat_count_visible_page_marks_cleared(rel); + if (map[mapByte] & ((flags & VISIBILITYMAP_ALL_FROZEN) << mapOffset)) + pgstat_count_frozen_page_marks_cleared(rel); + } + map[mapByte] &= ~mask; MarkBufferDirty(vmbuf); diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 3ad1c7ea0d8..2345a5bcb02 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -1171,7 +1171,7 @@ btvacuumpage(BTVacState *vstate, BlockNumber scanblkno) backtrack_to = P_NONE; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); /* * We can't use _bt_getbuf() here because it always applies diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index 85bc02b92c8..1f0e98648b8 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -626,7 +626,7 @@ spgvacuumpage(spgBulkDeleteState *bds, BlockNumber blkno) Page page; /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); buffer = ReadBufferExtended(index, MAIN_FORKNUM, blkno, RBM_NORMAL, bds->info->strategy); @@ -705,7 +705,7 @@ spgprocesspending(spgBulkDeleteState *bds) continue; /* ignore already-done items */ /* call vacuum_delay_point while not holding any buffer lock */ - vacuum_delay_point(); + vacuum_delay_point(false); /* examine the referenced page */ blkno = ItemPointerGetBlockNumber(&pitem->tid); diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 0b62b174afa..52adec9f666 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1264,6 +1264,7 @@ InsertPgClassTuple(Relation pg_class_desc, values[Anum_pg_class_relpages - 1] = Int32GetDatum(rd_rel->relpages); values[Anum_pg_class_reltuples - 1] = Float4GetDatum(rd_rel->reltuples); values[Anum_pg_class_relallvisible - 1] = Int32GetDatum(rd_rel->relallvisible); + values[Anum_pg_class_relallfrozen - 1] = Int32GetDatum(rd_rel->relallfrozen); values[Anum_pg_class_reltoastrelid - 1] = ObjectIdGetDatum(rd_rel->reltoastrelid); values[Anum_pg_class_relhasindex - 1] = BoolGetDatum(rd_rel->relhasindex); values[Anum_pg_class_relisshared - 1] = BoolGetDatum(rd_rel->relisshared); @@ -1337,6 +1338,7 @@ AddNewRelationTuple(Relation pg_class_desc, new_rel_reltup->relpages = 0; new_rel_reltup->reltuples = -1; new_rel_reltup->relallvisible = 0; + new_rel_reltup->relallfrozen = 0; /* Sequences always have a known size */ if (relkind == RELKIND_SEQUENCE) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index fadd32fde9a..7c7df5d4c58 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2945,8 +2945,8 @@ FormIndexDatum(struct IndexInfo *indexInfo, * hasindex: set relhasindex to this value * reltuples: if >= 0, set reltuples to this value; else no change * - * If reltuples >= 0, relpages and relallvisible are also updated (using - * RelationGetNumberOfBlocks() and visibilitymap_count()). + * If reltuples >= 0, relpages, relallvisible, and relallfrozen are also + * updated (using RelationGetNumberOfBlocks() and visibilitymap_count()). * * NOTE: an important side-effect of this operation is that an SI invalidation * message is sent out to all backends --- including me --- causing relcache @@ -2964,6 +2964,7 @@ index_update_stats(Relation rel, bool update_stats; BlockNumber relpages = 0; /* keep compiler quiet */ BlockNumber relallvisible = 0; + BlockNumber relallfrozen = 0; Oid relid = RelationGetRelid(rel); Relation pg_class; ScanKeyData key[1]; @@ -2985,7 +2986,7 @@ index_update_stats(Relation rel, if (reltuples == 0 && rel->rd_rel->reltuples < 0) reltuples = -1; - update_stats = reltuples >= 0; + update_stats = reltuples >= 0 && Gp_role != GP_ROLE_DISPATCH; /* * Finish I/O and visibility map buffer locks before @@ -2998,8 +2999,8 @@ index_update_stats(Relation rel, { relpages = RelationGetNumberOfBlocks(rel); - if (rel->rd_rel->relkind != RELKIND_INDEX) - visibilitymap_count(rel, &relallvisible, NULL); + if (rel->rd_rel->relkind != RELKIND_INDEX && !RelationStorageIsAO(rel)) + visibilitymap_count(rel, &relallvisible, &relallfrozen); } /* @@ -3055,16 +3056,8 @@ index_update_stats(Relation rel, dirty = true; } - if (reltuples >= 0 && Gp_role != GP_ROLE_DISPATCH) + if (update_stats) { - /* - * GPDB: We don't maintain relallvisible for AO/CO tables. - */ - if (rd_rel->relkind != RELKIND_INDEX && !RelationStorageIsAO(rel)) - visibilitymap_count(rel, &relallvisible, NULL); - else /* don't bother for indexes */ - relallvisible = 0; - if (rd_rel->relpages != (int32) relpages) { rd_rel->relpages = (int32) relpages; @@ -3080,6 +3073,11 @@ index_update_stats(Relation rel, rd_rel->relallvisible = (int32) relallvisible; dirty = true; } + if (rd_rel->relallfrozen != (int32) relallfrozen) + { + rd_rel->relallfrozen = (int32) relallfrozen; + dirty = true; + } } /* diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 24b7fc1d33b..f2b72af85f4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -701,7 +701,16 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_vacuum_count(C.oid) AS vacuum_count, pg_stat_get_autovacuum_count(C.oid) AS autovacuum_count, pg_stat_get_analyze_count(C.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(C.oid) AS autoanalyze_count, + pg_stat_get_total_vacuum_time(C.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(C.oid) AS total_autovacuum_time, + pg_stat_get_total_analyze_time(C.oid) AS total_analyze_time, + pg_stat_get_total_autoanalyze_time(C.oid) AS total_autoanalyze_time, + pg_stat_get_total_vacuum_delay_time(C.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(C.oid) AS total_autovacuum_delay_time, + pg_stat_get_vacuum_failsafe_count(C.oid) AS vacuum_failsafe_count, + pg_stat_get_visible_page_marks_cleared(C.oid) AS visible_page_marks_cleared, + pg_stat_get_frozen_page_marks_cleared(C.oid) AS frozen_page_marks_cleared FROM pg_class C LEFT JOIN pg_index I ON C.oid = I.indrelid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) @@ -821,7 +830,11 @@ CREATE VIEW pg_stat_all_indexes AS pg_stat_get_numscans(I.oid) AS idx_scan, pg_stat_get_lastscan(I.oid) AS last_idx_scan, pg_stat_get_tuples_returned(I.oid) AS idx_tup_read, - pg_stat_get_tuples_fetched(I.oid) AS idx_tup_fetch + pg_stat_get_tuples_fetched(I.oid) AS idx_tup_fetch, + pg_stat_get_total_vacuum_time(I.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(I.oid) AS total_autovacuum_time, + pg_stat_get_total_vacuum_delay_time(I.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(I.oid) AS total_autovacuum_delay_time FROM pg_class C JOIN pg_index X ON C.oid = X.indrelid JOIN pg_class I ON I.oid = X.indexrelid @@ -1205,6 +1218,24 @@ CREATE VIEW pg_stat_database AS UNION ALL SELECT oid, datname FROM pg_database ) D; + +CREATE VIEW pg_stat_vacuum AS + SELECT + D.oid AS datid, + D.datname AS datname, + pg_stat_get_db_total_vacuum_time(D.oid) AS total_vacuum_time, + pg_stat_get_db_total_autovacuum_time(D.oid) AS total_autovacuum_time, + pg_stat_get_db_total_vacuum_delay_time(D.oid) AS total_vacuum_delay_time, + pg_stat_get_db_total_autovacuum_delay_time(D.oid) AS total_autovacuum_delay_time, + pg_stat_get_db_vacuum_failsafe_count(D.oid) AS vacuum_failsafe_count, + pg_stat_get_db_vacuum_interrupt_count(D.oid) AS vacuum_interrupt_count, + pg_stat_get_db_stat_reset_time(D.oid) AS stats_reset + FROM ( + SELECT 0 AS oid, NULL::name AS datname + UNION ALL + SELECT oid, datname FROM pg_database + ) D; + CREATE VIEW pg_resqueue_status AS SELECT q.rsqname, @@ -1564,7 +1595,8 @@ CREATE VIEW pg_stat_progress_analyze AS S.param5 AS ext_stats_computed, S.param6 AS child_tables_total, S.param7 AS child_tables_done, - CAST(S.param8 AS oid) AS current_child_table_relid + CAST(S.param8 AS oid) AS current_child_table_relid, + S.param9 / 1000000::double precision AS delay_time FROM pg_stat_get_progress_info('ANALYZE') AS S LEFT JOIN pg_database D ON S.datid = D.oid; @@ -1585,7 +1617,8 @@ CREATE VIEW pg_stat_progress_vacuum AS END AS phase, S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned, S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count, - S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples + S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples, + S.param8 / 1000000::double precision AS delay_time FROM pg_stat_get_progress_info('VACUUM') AS S LEFT JOIN pg_database D ON S.datid = D.oid; @@ -1933,4 +1966,4 @@ $$ WHERE pg_subscription.subname = sub_info.subname ) from gp_dist_random('gp_id'); -$$ LANGUAGE SQL; \ No newline at end of file +$$ LANGUAGE SQL; diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in index 41e4a2468e5..71fac72cd5f 100644 --- a/src/backend/catalog/system_views_gp.in +++ b/src/backend/catalog/system_views_gp.in @@ -14,6 +14,7 @@ pg_stat_all_tables pg_stat_archiver pg_stat_bgwriter pg_stat_database +pg_stat_vacuum pg_stat_database_conflicts pg_stat_gssapi pg_stat_operations diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql index 62bde4e3d96..8cd6d1638c2 100644 --- a/src/backend/catalog/system_views_gp_summary.sql +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -94,6 +94,23 @@ GROUP BY sdb.datname; +CREATE VIEW gp_stat_vacuum_summary AS +SELECT + sdb.datid, + sdb.datname, + sum(sdb.total_vacuum_time) as total_vacuum_time, + sum(sdb.total_autovacuum_time) as total_autovacuum_time, + sum(sdb.total_vacuum_delay_time) as total_vacuum_delay_time, + sum(sdb.total_autovacuum_delay_time) as total_autovacuum_delay_time, + max(sdb.vacuum_failsafe_count) as vacuum_failsafe_count, + max(sdb.vacuum_interrupt_count) as vacuum_interrupt_count, + max(sdb.stats_reset) as stats_reset +FROM + gp_stat_vacuum sdb +GROUP BY + sdb.datid, + sdb.datname; + -- Gather data from segments on user tables, and use data on coordinator on system tables. CREATE VIEW gp_stat_all_tables_summary AS SELECT @@ -120,7 +137,16 @@ SELECT s.vacuum_count, s.autovacuum_count, s.analyze_count, - s.autoanalyze_count + s.autoanalyze_count, + m.total_vacuum_time, + m.total_autovacuum_time, + m.total_analyze_time, + m.total_autoanalyze_time, + m.total_vacuum_delay_time, + m.total_autovacuum_delay_time, + m.vacuum_failsafe_count, + m.visible_page_marks_cleared, + m.frozen_page_marks_cleared FROM (SELECT allt.relid, @@ -148,7 +174,16 @@ FROM max(vacuum_count) as vacuum_count, max(autovacuum_count) as autovacuum_count, max(analyze_count) as analyze_count, - max(autoanalyze_count) as autoanalyze_count + max(autoanalyze_count) as autoanalyze_count, + case when d.policytype = 'r' then (sum(total_vacuum_time)/d.numsegments) else sum(total_vacuum_time) end total_vacuum_time, + case when d.policytype = 'r' then (sum(total_autovacuum_time)/d.numsegments) else sum(total_autovacuum_time) end total_autovacuum_time, + case when d.policytype = 'r' then (sum(total_analyze_time)/d.numsegments) else sum(total_analyze_time) end total_analyze_time, + case when d.policytype = 'r' then (sum(total_autoanalyze_time)/d.numsegments) else sum(total_autoanalyze_time) end total_autoanalyze_time, + case when d.policytype = 'r' then (sum(total_vacuum_delay_time)/d.numsegments) else sum(total_vacuum_delay_time) end total_vacuum_delay_time, + case when d.policytype = 'r' then (sum(total_autovacuum_delay_time)/d.numsegments) else sum(total_autovacuum_delay_time) end total_autovacuum_delay_time, + max(vacuum_failsafe_count) as vacuum_failsafe_count, + case when d.policytype = 'r' then (sum(visible_page_marks_cleared)/d.numsegments)::bigint else sum(visible_page_marks_cleared) end visible_page_marks_cleared, + case when d.policytype = 'r' then (sum(frozen_page_marks_cleared)/d.numsegments)::bigint else sum(frozen_page_marks_cleared) end frozen_page_marks_cleared FROM gp_dist_random('pg_stat_all_tables') allt inner join pg_class c @@ -229,7 +264,11 @@ SELECT m.idx_scan, m.last_idx_scan, m.idx_tup_read, - m.idx_tup_fetch + m.idx_tup_fetch, + m.total_vacuum_time, + m.total_autovacuum_time, + m.total_vacuum_delay_time, + m.total_autovacuum_delay_time FROM (SELECT alli.relid, @@ -240,7 +279,11 @@ FROM case when d.policytype = 'r' then (sum(alli.idx_scan)/d.numsegments)::bigint else sum(alli.idx_scan) end idx_scan, max(last_idx_scan) as last_idx_scan, case when d.policytype = 'r' then (sum(alli.idx_tup_read)/d.numsegments)::bigint else sum(alli.idx_tup_read) end idx_tup_read, - case when d.policytype = 'r' then (sum(alli.idx_tup_fetch)/d.numsegments)::bigint else sum(alli.idx_tup_fetch) end idx_tup_fetch + case when d.policytype = 'r' then (sum(alli.idx_tup_fetch)/d.numsegments)::bigint else sum(alli.idx_tup_fetch) end idx_tup_fetch, + case when d.policytype = 'r' then (sum(alli.total_vacuum_time)/d.numsegments) else sum(alli.total_vacuum_time) end total_vacuum_time, + case when d.policytype = 'r' then (sum(alli.total_autovacuum_time)/d.numsegments) else sum(alli.total_autovacuum_time) end total_autovacuum_time, + case when d.policytype = 'r' then (sum(alli.total_vacuum_delay_time)/d.numsegments) else sum(alli.total_vacuum_delay_time) end total_vacuum_delay_time, + case when d.policytype = 'r' then (sum(alli.total_autovacuum_delay_time)/d.numsegments) else sum(alli.total_autovacuum_delay_time) end total_autovacuum_delay_time FROM gp_dist_random('pg_stat_all_indexes') alli inner join pg_class c @@ -259,7 +302,7 @@ FROM pg_stat_all_indexes WHERE relid < 16384) m, pg_stat_all_indexes s -WHERE m.relid = s.relid; +WHERE m.relid = s.relid AND m.indexrelid = s.indexrelid; CREATE VIEW gp_stat_sys_indexes_summary AS SELECT * FROM gp_stat_all_indexes_summary diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index e7c5e68082c..1059586cede 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -541,7 +541,10 @@ do_analyze_rel(Relation onerel, VacuumParams *params, save_sec_context | SECURITY_RESTRICTED_OPERATION); save_nestlevel = NewGUCNestLevel(); - /* measure elapsed time iff autovacuum logging requires it */ + /* + * When autovacuum logging is used, initialize a resource usage snapshot + * and optionally track I/O timing. + */ if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0) { if (track_io_timing) @@ -551,9 +554,11 @@ do_analyze_rel(Relation onerel, VacuumParams *params, } pg_rusage_init(&ru0); - starttime = GetCurrentTimestamp(); } + /* Used for instrumentation and stats report */ + starttime = GetCurrentTimestamp(); + /* * Determine which columns to analyze * @@ -1100,12 +1105,12 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ if (!inh) { - BlockNumber relallvisible; + BlockNumber relallvisible = 0; + BlockNumber relallfrozen = 0; - if (RelationStorageIsAO(onerel)) - relallvisible = 0; - else - relallvisible = AcquireNumberOfAllVisibleBlocks(onerel); + if (RELKIND_HAS_STORAGE(onerel->rd_rel->relkind) && + !RelationStorageIsAO(onerel)) + AcquireVisibilityMapCounts(onerel, &relallvisible, &relallfrozen); /* * Update pg_class for table relation. CCI first, in case acquirefunc @@ -1116,6 +1121,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, relpages, totalrows, relallvisible, + relallfrozen, hasindex, InvalidTransactionId, InvalidMultiXactId, @@ -1187,7 +1193,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, vac_update_relstats(Irel[ind], estimatedIndexPages, totalindexrows, - 0, + 0, 0, false, InvalidTransactionId, InvalidMultiXactId, @@ -1204,7 +1210,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ CommandCounterIncrement(); vac_update_relstats(onerel, -1, totalrows, - 0, hasindex, InvalidTransactionId, + 0, 0, hasindex, InvalidTransactionId, InvalidMultiXactId, NULL, NULL, in_outer_xact, @@ -1222,9 +1228,9 @@ do_analyze_rel(Relation onerel, VacuumParams *params, */ if (!inh) pgstat_report_analyze(onerel, totalrows, totaldeadrows, - (va_cols == NIL)); + (va_cols == NIL), starttime); else if (onerel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - pgstat_report_analyze(onerel, 0, 0, (va_cols == NIL)); + pgstat_report_analyze(onerel, 0, 0, (va_cols == NIL), starttime); /* * If this isn't part of VACUUM ANALYZE, let index AMs do cleanup. @@ -1423,7 +1429,7 @@ compute_index_stats(Relation onerel, double totalrows, { HeapTuple heapTuple = rows[rowno]; - vacuum_delay_point(); + vacuum_delay_point(true); /* * Reset the per-tuple context each time, to reclaim any cruft @@ -1843,7 +1849,7 @@ acquire_sample_rows(Relation onerel, int elevel, prefetch_targblock = BlockSampler_Next(&prefetch_bs); #endif - vacuum_delay_point(); + vacuum_delay_point(true); block_accepted = table_scan_analyze_next_block(scan, targblock, vac_strategy); @@ -2428,46 +2434,63 @@ AcquireNumberOfBlocks(Relation onerel) } /* - * Collect visibility map of relation in dispatcher. - * - * In GPDB if we're in the dispatcher, we need to collect the number of - * visibility map in pg_class from segments. + * Collect both visibility-map counts locally or from the segment catalogs. + * Replicated relations contribute one copy, just like relpages. */ -BlockNumber -AcquireNumberOfAllVisibleBlocks(Relation onerel) +void +AcquireVisibilityMapCounts(Relation onerel, BlockNumber *all_visible, + BlockNumber *all_frozen) { - int64 totalvms; - - /* collect total vms from segments in master */ - if (Gp_role == GP_ROLE_DISPATCH && - onerel->rd_cdbpolicy && !GpPolicyIsEntry(onerel->rd_cdbpolicy)) - { - /* Query the segments pg_class. */ - char relvm_sql[80]; - - snprintf(relvm_sql, sizeof(relvm_sql), - "select relallvisible from pg_catalog.pg_class where oid = %u", RelationGetRelid(onerel)); - totalvms = get_size_from_segDBs(relvm_sql); - if (GpPolicyIsReplicated(onerel->rd_cdbpolicy)) - { - /* - * If the distribution of the relation is replicated, we will sum up - * vms much twice which we expecting only once. So we need to divide - * up totalvms by numsegments. - */ - totalvms /= onerel->rd_cdbpolicy->numsegments; - } - - return (BlockNumber)totalvms; - } - /* get vms from local in segment */ - else - { - BlockNumber all_visible = 0; - visibilitymap_count(onerel, &all_visible, NULL); - - return all_visible; - } + if (Gp_role == GP_ROLE_DISPATCH && + onerel->rd_cdbpolicy && !GpPolicyIsEntry(onerel->rd_cdbpolicy)) + { + char relvm_sql[128]; + int64 visible = 0; + int64 frozen = 0; + CdbPgResults cdb_pgresults = {NULL, 0}; + + /* Read both counters from the same catalog row in one dispatch. */ + snprintf(relvm_sql, sizeof(relvm_sql), + "select relallvisible, relallfrozen from pg_catalog.pg_class where oid = %u", + RelationGetRelid(onerel)); + CdbDispatchCommand(relvm_sql, DF_WITH_SNAPSHOT, &cdb_pgresults); + PG_TRY(); + { + for (int i = 0; i < cdb_pgresults.numResults; i++) + { + PGresult *result = cdb_pgresults.pg_results[i]; + ExecStatusType status = PQresultStatus(result); + int ntuples = PQntuples(result); + int nfields = PQnfields(result); + + if (status != PGRES_TUPLES_OK) + elog(ERROR, "unexpected result from segment: %d", status); + if (ntuples != 1 || nfields != 2) + elog(ERROR, "unexpected shape of visibility-map result from segment (%d rows, %d cols)", + ntuples, nfields); + if (PQgetisnull(result, 0, 0) || PQgetisnull(result, 0, 1)) + elog(ERROR, "unexpected NULL visibility-map count from segment"); + + visible += pg_strtoint64(PQgetvalue(result, 0, 0)); + frozen += pg_strtoint64(PQgetvalue(result, 0, 1)); + } + } + PG_FINALLY(); + { + cdbdisp_clearCdbPgResults(&cdb_pgresults); + } + PG_END_TRY(); + + if (GpPolicyIsReplicated(onerel->rd_cdbpolicy)) + { + visible /= onerel->rd_cdbpolicy->numsegments; + frozen /= onerel->rd_cdbpolicy->numsegments; + } + *all_visible = (BlockNumber) visible; + *all_frozen = (BlockNumber) frozen; + } + else + visibilitymap_count(onerel, all_visible, all_frozen); } /* @@ -3471,7 +3494,7 @@ compute_trivial_stats(VacAttrStatsP stats, Datum value; bool isnull; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); @@ -3593,7 +3616,7 @@ compute_distinct_stats(VacAttrStatsP stats, int firstcount1, j; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); @@ -3953,7 +3976,7 @@ compute_scalar_stats(VacAttrStatsP stats, Datum value; bool isnull; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, i, &isnull); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index a3d2ad87bfb..1c12029334a 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -1511,6 +1511,7 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, int32 swap_pages; float4 swap_tuples; int32 swap_allvisible; + int32 swap_allfrozen; swap_pages = relform1->relpages; relform1->relpages = relform2->relpages; @@ -1523,6 +1524,10 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, swap_allvisible = relform1->relallvisible; relform1->relallvisible = relform2->relallvisible; relform2->relallvisible = swap_allvisible; + + swap_allfrozen = relform1->relallfrozen; + relform1->relallfrozen = relform2->relallfrozen; + relform2->relallfrozen = swap_allfrozen; } /* @@ -1728,7 +1733,8 @@ swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, vac_send_relstats_to_qd(rel, relform1->relpages, relform1->reltuples, - relform1->relallvisible); + relform1->relallvisible, + relform1->relallfrozen); relation_close(rel, AccessShareLock); } diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index f4ba5615d5a..286f698b988 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -47,6 +47,7 @@ #include "catalog/pg_namespace.h" #include "commands/cluster.h" #include "commands/defrem.h" +#include "commands/progress.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" #include "miscadmin.h" @@ -106,6 +107,7 @@ int vacuum_multixact_freeze_min_age; int vacuum_multixact_freeze_table_age; int vacuum_failsafe_age; int vacuum_multixact_failsafe_age; +bool track_cost_delay_timing; /* * Variables for cost-based vacuum delay. The defaults differ between @@ -116,6 +118,16 @@ int vacuum_multixact_failsafe_age; double vacuum_cost_delay = 0; int vacuum_cost_limit = 200; +/* Variable for reporting cost-based vacuum delay from parallel workers. */ +int64 parallel_vacuum_worker_delay_ns = 0; + +/* + * Cumulative cost-based VACUUM delay time (ms) for this process, excluding + * ANALYZE. + * Callers save the starting value and report the delta. + */ +double VacuumDelayTime = 0; + /* * VacuumFailsafeActive is a defined as a global so that we can determine * whether or not to re-enable cost-based vacuum delay when vacuuming a table. @@ -1788,6 +1800,7 @@ void vac_update_relstats(Relation relation, BlockNumber num_pages, double num_tuples, BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages, bool hasindex, TransactionId frozenxid, MultiXactId minmulti, bool *frozenxid_updated, bool *minmulti_updated, @@ -1825,13 +1838,15 @@ vac_update_relstats(Relation relation, num_pages = relation->rd_rel->relpages; num_tuples = relation->rd_rel->reltuples; num_all_visible_pages = relation->rd_rel->relallvisible; + num_all_frozen_pages = relation->rd_rel->relallfrozen; } else if (Gp_role == GP_ROLE_EXECUTE) { vac_send_relstats_to_qd(relation, num_pages, num_tuples, - num_all_visible_pages); + num_all_visible_pages, + num_all_frozen_pages); } } @@ -1911,6 +1926,11 @@ vac_update_relstats(Relation relation, pgcform->relallvisible = (int32) num_all_visible_pages; dirty = true; } + if (pgcform->relallfrozen != (int32) num_all_frozen_pages) + { + pgcform->relallfrozen = (int32) num_all_frozen_pages; + dirty = true; + } elog(DEBUG2, "Vacuum oid=%u pages=%d tuples=%f", relid, pgcform->relpages, pgcform->reltuples); @@ -3179,7 +3199,7 @@ vac_close_indexes(int nindexes, Relation *Irel, LOCKMODE lockmode) * typically once per page processed. */ void -vacuum_delay_point(void) +vacuum_delay_point(bool is_analyze) { double msec = 0; @@ -3222,13 +3242,54 @@ vacuum_delay_point(void) /* Nap if appropriate */ if (msec > 0) { + instr_time delay_start; + if (msec > vacuum_cost_delay * 4) msec = vacuum_cost_delay * 4; + if (track_cost_delay_timing) + INSTR_TIME_SET_CURRENT(delay_start); + pgstat_report_wait_start(WAIT_EVENT_VACUUM_DELAY); pg_usleep(msec * 1000); pgstat_report_wait_end(); + if (track_cost_delay_timing) + { + instr_time delay_end; + instr_time delay; + + INSTR_TIME_SET_CURRENT(delay_end); + INSTR_TIME_SET_ZERO(delay); + INSTR_TIME_ACCUM_DIFF(delay, delay_end, delay_start); + if (!is_analyze) + VacuumDelayTime += INSTR_TIME_GET_MILLISEC(delay); + + /* + * For parallel workers, we only report the delay time every once + * in a while to avoid overloading the leader with messages and + * interrupts. + */ + if (IsParallelWorker()) + { + /* + * Upstream sends the sleep time of a parallel worker to the + * leader's progress entry now and then, which needs + * pgstat_progress_parallel_incr_param() of PostgreSQL 17. + * Cloudberry does not run parallel vacuum, so just + * accumulate it. + */ + Assert(!is_analyze); + parallel_vacuum_worker_delay_ns += INSTR_TIME_GET_NANOSEC(delay); + } + else if (is_analyze) + pgstat_progress_incr_param(PROGRESS_ANALYZE_DELAY_TIME, + INSTR_TIME_GET_NANOSEC(delay)); + else + pgstat_progress_incr_param(PROGRESS_VACUUM_DELAY_TIME, + INSTR_TIME_GET_NANOSEC(delay)); + } + /* * We don't want to ignore postmaster death during very long vacuums * with vacuum_cost_delay configured. We can't use the usual @@ -3563,6 +3624,7 @@ vacuum_combine_stats(VacuumStatsContext *stats_context, CdbPgResults *cdb_pgresu tmp_stats_combo->rel_pages += pgclass_stats_combo->rel_pages; tmp_stats_combo->rel_tuples += pgclass_stats_combo->rel_tuples; tmp_stats_combo->relallvisible += pgclass_stats_combo->relallvisible; + tmp_stats_combo->relallfrozen += pgclass_stats_combo->relallfrozen; /* * Accumulate the number of QEs, assuming sending only once * per QE for each relid in the VACUUM scenario. @@ -3616,6 +3678,7 @@ vac_update_relstats_from_list(VacuumStatsContext *stats_context) stats->rel_pages = stats->rel_pages / rel->rd_cdbpolicy->numsegments; stats->rel_tuples = stats->rel_tuples / rel->rd_cdbpolicy->numsegments; stats->relallvisible = stats->relallvisible / rel->rd_cdbpolicy->numsegments; + stats->relallfrozen = stats->relallfrozen / rel->rd_cdbpolicy->numsegments; } if (RelationIsAppendOptimized(rel)) @@ -3667,6 +3730,7 @@ vac_update_relstats_from_list(VacuumStatsContext *stats_context) vac_update_relstats(rel, stats->rel_pages, stats->rel_tuples, stats->relallvisible, + stats->relallfrozen, rel->rd_rel->relhasindex, InvalidTransactionId, InvalidMultiXactId, @@ -3718,7 +3782,8 @@ void vac_send_relstats_to_qd(Relation relation, BlockNumber num_pages, double num_tuples, - BlockNumber num_all_visible_pages) + BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages) { StringInfoData buf; @@ -3732,6 +3797,7 @@ vac_send_relstats_to_qd(Relation relation, stats.rel_pages = num_pages; stats.rel_tuples = num_tuples; stats.relallvisible = num_all_visible_pages; + stats.relallfrozen = num_all_frozen_pages; pq_sendbyte(&buf, true); /* Mark the result ready when receive this message */ pq_sendint(&buf, PGExtraTypeVacuumStats, sizeof(PGExtraType)); pq_sendint(&buf, sizeof(VPgClassStats), sizeof(int)); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 075c73fb1c1..6d654dedd21 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -114,6 +114,8 @@ */ #include "postgres.h" +#include + #include "access/table.h" #include "access/aocs_compaction.h" #include "access/aomd.h" @@ -156,13 +158,16 @@ static bool appendonly_tid_reaped(ItemPointer itemptr, void *state); static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static AOVacuumRelStats *init_vacrelstats(void); static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); +static void ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, + double startdelaytime); +static void ao_vacuum_error_callback(void *arg); static void ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) @@ -237,7 +242,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt { BlockNumber relpages; double reltuples; - double deadtuples; + int64 deadtuples; bool relhasindex; /* AO/AOCO total file segment number, use type BlockNumber to * represent same type with num_all_visible_pages in libpq. @@ -294,6 +299,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt total_file_segs, /* AO/AOCO does not currently have an equivalent to Heap's 'all visible pages', use this field to represent AO/AOCO's total segment file count */ + 0, /* relallfrozen: AO has no visibility map */ relhasindex, InvalidTransactionId, InvalidMultiXactId, @@ -302,11 +308,9 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt false, true /* isvacuum */); - /* report results to the stats collector, too */ - pgstat_report_vacuum(RelationGetRelid(onerel), - onerel->rd_rel->relisshared, - reltuples, - deadtuples); + /* Report once all phase timers have been stopped. */ + vacrelstats->live_tuples = reltuples; + vacrelstats->dead_tuples = deadtuples; SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); } @@ -428,11 +432,28 @@ void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { static AOVacuumRelStats *vacrelstats = NULL; + ErrorContextCallback errcallback; + instr_time phase_start; + instr_time phase_end; + double phase_start_delay; Assert(RelationStorageIsAO(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + /* + * The phases of one vacuum share these stats and free them at the end of + * the last phase, so a vacuum that failed in between leaves them behind. + * Drop such leftovers: otherwise the counters of the failed vacuum are + * taken for the next one, whose progress is never reported because the + * progress command is only + * started along with the stats. + */ + if (vacrelstats != NULL && + (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || + vacrelstats->relid != RelationGetRelid(rel))) + cleanup_vacrelstats(&vacrelstats); + if (!vacrelstats) { if (ao_vacuum_phase != VACOPT_AO_PRE_CLEANUP_PHASE && Gp_role == GP_ROLE_EXECUTE) @@ -450,8 +471,18 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); vacrelstats = init_vacrelstats(); + vacrelstats->relid = RelationGetRelid(rel); } + /* Count the vacuums of the relation that an error interrupts */ + errcallback.callback = ao_vacuum_error_callback; + errcallback.arg = rel; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + INSTR_TIME_SET_CURRENT(phase_start); + phase_start_delay = VacuumDelayTime; + /* * Do the actual work --- either FULL or "lazy" vacuum */ @@ -460,14 +491,67 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - { ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); - pgstat_progress_end_command(); - cleanup_vacrelstats(&vacrelstats); - } else /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); + + if (ao_vacuum_phase != 0) + { + INSTR_TIME_SET_CURRENT(phase_end); + INSTR_TIME_SUBTRACT(phase_end, phase_start); + vacrelstats->vacuum_time += INSTR_TIME_GET_MILLISEC(phase_end); + vacrelstats->delay_time += VacuumDelayTime - phase_start_delay; + } + + if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) + { + /* The last phase: report only the phases executed by this worker. */ + pgstat_report_vacuum_elapsed(RelationGetRelid(rel), + rel->rd_rel->relisshared, + vacrelstats->live_tuples, vacrelstats->dead_tuples, + (PgStat_Counter) rint(vacrelstats->vacuum_time), + (PgStat_Counter) rint(vacrelstats->delay_time), + false); /* AO has no failsafe mode. */ + pgstat_progress_end_command(); + cleanup_vacrelstats(&vacrelstats); + } + error_context_stack = errcallback.previous; +} + +/* + * Error context callback of an append-optimized vacuum phase. + * + * Like vacuum_error_callback() of the heap, count a vacuum interrupted by an + * actual ERROR, not by a lower-severity report that merely carries this + * context, in vacuum_interrupt_count of the database. We are inside the + * error handler, so pgstat_count_vacuum_error() only bumps a counter. The + * callback adds no context line: the phases report their own progress. + */ +static void +ao_vacuum_error_callback(void *arg) +{ + Relation rel = (Relation) arg; + + if (geterrlevel() == ERROR) + pgstat_count_vacuum_error(rel->rd_rel->relisshared); +} + +/* + * Accumulate one vacuum pass over an index of an append-optimized table + * into the index's cumulative vacuum and delay times, as the heap does in + * lazy_vacuum_one_index() and lazy_cleanup_one_index(). + */ +static void +ao_report_index_vacuum_time(Relation indrel, TimestampTz starttime, + double startdelaytime) +{ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(starttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + IsAutoVacuumWorkerProcess()); } /* @@ -544,6 +628,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs int i; Relation *Irel; int nindexes; + bool final_cleanup = + (options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE; Assert(RelationStorageIsAO(aoRelation)); @@ -574,29 +660,37 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs { for (i = 0; i < nindexes; i++) { + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; + scan_index(Irel[i], aoRelation, elevel, bstrategy); + ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } else { for (i = 0; i < nindexes; i++) { + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; + vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, bstrategy, vacrelstats); + ao_report_index_vacuum_time(Irel[i], istarttime, startdelaytime); } } } vac_close_indexes(nindexes, Irel, NoLock); pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, - ((options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE) ? + final_cleanup ? PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP : PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP); return nindexes; } @@ -659,6 +753,7 @@ vacuum_appendonly_index(Relation indexRelation, vac_update_relstats(indexRelation, stats->num_pages, stats->num_index_tuples, 0, /* relallvisible */ + 0, /* relallfrozen */ false, InvalidTransactionId, InvalidMultiXactId, @@ -714,7 +809,7 @@ appendonly_tid_reaped(ItemPointer itemptr, void *state) static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) + int64 *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) { FileSegTotals *fstotal; BlockNumber nblocks; @@ -826,6 +921,7 @@ scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac vac_update_relstats(indrel, stats->num_pages, stats->num_index_tuples, 0, /* relallvisible, don't bother for indexes */ + 0, /* relallfrozen */ false, InvalidTransactionId, InvalidMultiXactId, diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index a79067fd461..b95c58aef3f 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -26,6 +26,8 @@ */ #include "postgres.h" +#include + #include "access/amapi.h" #include "access/table.h" #include "access/xact.h" @@ -33,6 +35,7 @@ #include "commands/vacuum.h" #include "optimizer/paths.h" #include "pgstat.h" +#include "postmaster/autovacuum.h" #include "storage/bufmgr.h" #include "tcop/tcopprot.h" #include "utils/lsyscache.h" @@ -64,6 +67,9 @@ typedef struct PVShared Oid relid; int elevel; + /* Leader's autovacuum flag, used for worker timing statistics. */ + bool is_autovacuum; + /* * Fields for both index vacuum and cleanup. * @@ -365,6 +371,7 @@ parallel_vacuum_init(Relation rel, Relation *indrels, int nindexes, shared = (PVShared *) shm_toc_allocate(pcxt->toc, est_shared_len); MemSet(shared, 0, est_shared_len); shared->relid = RelationGetRelid(rel); + shared->is_autovacuum = IsAutoVacuumWorkerProcess(); shared->elevel = elevel; shared->maintenance_work_mem_worker = (nindexes_mwm > 0) ? @@ -836,6 +843,8 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, IndexBulkDeleteResult *istat = NULL; IndexBulkDeleteResult *istat_res; IndexVacuumInfo ivinfo; + TimestampTz istarttime = GetCurrentTimestamp(); + double startdelaytime = VacuumDelayTime; /* * Update the pointer to the corresponding bulk-deletion result if someone @@ -871,6 +880,18 @@ parallel_vacuum_process_one_index(ParallelVacuumState *pvs, Relation indrel, RelationGetRelationName(indrel)); } + /* + * Accumulate this pass into the index's cumulative vacuum times. Use the + * leader's DSM flag to classify autovacuum: a parallel worker of an + * autovacuum leader is a regular background worker. + */ + pgstat_report_index_vacuum_time(indrel, + TimestampDifferenceMilliseconds(istarttime, + GetCurrentTimestamp()), + (PgStat_Counter) rint(VacuumDelayTime - + startdelaytime), + pvs->shared->is_autovacuum); + /* * Copy the index bulk-deletion result returned from ambulkdelete and * amvacuumcleanup to the DSM segment if it's the first cycle because they diff --git a/src/backend/tsearch/ts_typanalyze.c b/src/backend/tsearch/ts_typanalyze.c index c24243b11b4..6aa685ed55f 100644 --- a/src/backend/tsearch/ts_typanalyze.c +++ b/src/backend/tsearch/ts_typanalyze.c @@ -206,7 +206,7 @@ compute_tsvector_stats(VacAttrStats *stats, char *lexemesptr; int j; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, vector_no, &isnull); diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 813aa359fa0..a320a735c22 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -36,6 +36,13 @@ static int pgStatXactCommit = 0; static int pgStatXactRollback = 0; static PgStat_Counter pgLastSessionReportTime = 0; +/* + * Vacuums interrupted by an error, counted from the vacuum error callback + * and folded into the pending database entries at transaction end. + */ +static PgStat_Counter pgStatVacuumErrors = 0; +static PgStat_Counter pgStatSharedVacuumErrors = 0; + /* * Remove entry for the database being dropped. @@ -245,6 +252,20 @@ pgstat_fetch_stat_dbentry(Oid dboid) pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid); } +/* + * Add the vacuum errors counted for a database to its pending entry, and + * reset the counter. + */ +static void +pgstat_fold_vacuum_errors(Oid dboid, PgStat_Counter *count) +{ + if (*count == 0) + return; + + pgstat_prep_database_pending(dboid)->vacuum_interrupt_count += *count; + *count = 0; +} + void AtEOXact_PgStat_Database(bool isCommit, bool parallel) { @@ -260,6 +281,15 @@ AtEOXact_PgStat_Database(bool isCommit, bool parallel) else pgStatXactRollback++; } + + /* + * Fold in the vacuums that pgstat_count_vacuum_error() saw failing. We + * are past LWLockReleaseAll() of the aborting transaction here, so it is + * safe to touch the pending entries, which the next pgstat_report_stat() + * flushes to shared memory as usual. + */ + pgstat_fold_vacuum_errors(MyDatabaseId, &pgStatVacuumErrors); + pgstat_fold_vacuum_errors(InvalidOid, &pgStatSharedVacuumErrors); } /* @@ -326,6 +356,31 @@ pgstat_should_report_connstat(void) return MyBackendType == B_BACKEND; } +/* + * Count a vacuum that was interrupted by an error. + * + * This is called from the vacuum error callback, that is from inside the + * error handler, so it must not do anything that could fail or block: the + * error may well have been raised by the locking code itself, possibly + * while holding an LWLock. Just bump a counter here; the counter reaches + * the pending database entry in AtEOXact_PgStat_Database(), once the + * transaction end has released the locks we might have been holding. + * + * Vacuums of shared relations are counted in the InvalidOid entry, matching + * how their other stats are accounted. + */ +void +pgstat_count_vacuum_error(bool shared) +{ + if (!pgstat_track_counts) + return; + + if (shared) + pgStatSharedVacuumErrors++; + else + pgStatVacuumErrors++; +} + /* * Find or create a local PgStat_StatDBEntry entry for dboid. */ @@ -417,6 +472,12 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(blk_read_time); PGSTAT_ACCUM_DBCOUNT(blk_write_time); + PGSTAT_ACCUM_DBCOUNT(total_vacuum_time); + PGSTAT_ACCUM_DBCOUNT(total_autovacuum_time); + PGSTAT_ACCUM_DBCOUNT(total_vacuum_delay_time); + PGSTAT_ACCUM_DBCOUNT(total_autovacuum_delay_time); + PGSTAT_ACCUM_DBCOUNT(vacuum_failsafe_count); + PGSTAT_ACCUM_DBCOUNT(vacuum_interrupt_count); PGSTAT_ACCUM_DBCOUNT(sessions); PGSTAT_ACCUM_DBCOUNT(session_time); diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 45bbf9abcb7..71212becd50 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -215,7 +215,22 @@ pgstat_drop_relation(Relation rel) */ void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + TimestampTz starttime, PgStat_Counter delaytime, + bool failsafe) +{ + pgstat_report_vacuum_elapsed(tableoid, shared, livetuples, deadtuples, + TimestampDifferenceMilliseconds(starttime, + GetCurrentTimestamp()), + delaytime, failsafe); +} + +/* Report a completed vacuum whose active phases were timed separately (ms). */ +void +pgstat_report_vacuum_elapsed(Oid tableoid, bool shared, + PgStat_Counter livetuples, PgStat_Counter deadtuples, + PgStat_Counter elapsedtime, PgStat_Counter delaytime, + bool failsafe) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; @@ -255,15 +270,46 @@ pgstat_report_vacuum(Oid tableoid, bool shared, { tabentry->last_autovacuum_time = ts; tabentry->autovacuum_count++; + tabentry->total_autovacuum_time += elapsedtime; + tabentry->total_autovacuum_delay_time += delaytime; } else { tabentry->last_vacuum_time = ts; tabentry->vacuum_count++; + tabentry->total_vacuum_time += elapsedtime; + tabentry->total_vacuum_delay_time += delaytime; } + if (failsafe) + tabentry->vacuum_failsafe_count++; + pgstat_unlock_entry(entry_ref); + /* + * Accumulate the same times into the database-wide totals. Index + * processing happens inside the table's run, so per-index times reported + * via pgstat_report_index_vacuum_time() are not added here again. The + * database entry is stored in microseconds, as its other time counters. + */ + { + PgStat_StatDBEntry *dbentry = pgstat_prep_database_pending(dboid); + + if (IsAutoVacuumWorkerProcess()) + { + dbentry->total_autovacuum_time += elapsedtime * 1000; + dbentry->total_autovacuum_delay_time += delaytime * 1000; + } + else + { + dbentry->total_vacuum_time += elapsedtime * 1000; + dbentry->total_vacuum_delay_time += delaytime * 1000; + } + + if (failsafe) + dbentry->vacuum_failsafe_count++; + } + /* * Flush IO statistics now. pgstat_report_stat() will flush IO stats, * however this will not be called until after an entire autovacuum cycle @@ -273,6 +319,49 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } +/* + * Report the time spent vacuuming an index. + * + * Vacuum may process an index several times: a bulkdelete pass per index + * scan cycle plus a final cleanup pass, possibly spread across parallel + * workers. Each pass adds its elapsed and delay time here, accumulating + * into the index's total_vacuum_time or total_autovacuum_time, mirroring + * the table-level counters. The caller says whether this is autovacuum: + * parallel workers of an autovacuum leader are regular background workers, + * so IsAutoVacuumWorkerProcess() cannot be relied upon here. + */ +void +pgstat_report_index_vacuum_time(Relation rel, PgStat_Counter elapsedtime, + PgStat_Counter delaytime, bool is_autovacuum) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Relation *shtabentry; + PgStat_StatTabEntry *tabentry; + Oid dboid = (rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId); + + if (!pgstat_track_counts) + return; + + /* block acquiring lock for the same reason as pgstat_report_autovac() */ + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, + RelationGetRelid(rel), false); + shtabentry = (PgStatShared_Relation *) entry_ref->shared_stats; + tabentry = &shtabentry->stats; + + if (is_autovacuum) + { + tabentry->total_autovacuum_time += elapsedtime; + tabentry->total_autovacuum_delay_time += delaytime; + } + else + { + tabentry->total_vacuum_time += elapsedtime; + tabentry->total_vacuum_delay_time += delaytime; + } + + pgstat_unlock_entry(entry_ref); +} + /* * Report that the table was just analyzed and flush IO statistics. * @@ -282,12 +371,14 @@ pgstat_report_vacuum(Oid tableoid, bool shared, void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, - bool resetcounter) + bool resetcounter, TimestampTz starttime) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = (rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId); + TimestampTz ts; + PgStat_Counter elapsedtime; if (!pgstat_track_counts) return; @@ -321,6 +412,10 @@ pgstat_report_analyze(Relation rel, deadtuples = Max(deadtuples, 0); } + /* Store the data in the table's hash table entry. */ + ts = GetCurrentTimestamp(); + elapsedtime = TimestampDifferenceMilliseconds(starttime, ts); + /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, RelationGetRelid(rel), @@ -344,13 +439,15 @@ pgstat_report_analyze(Relation rel, if (IsAutoVacuumWorkerProcess()) { - tabentry->last_autoanalyze_time = GetCurrentTimestamp(); + tabentry->last_autoanalyze_time = ts; tabentry->autoanalyze_count++; + tabentry->total_autoanalyze_time += elapsedtime; } else { - tabentry->last_analyze_time = GetCurrentTimestamp(); + tabentry->last_analyze_time = ts; tabentry->analyze_count++; + tabentry->total_analyze_time += elapsedtime; } pgstat_unlock_entry(entry_ref); @@ -834,6 +931,8 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) tabentry->ins_since_vacuum += lstats->counts.tuples_inserted; tabentry->blocks_fetched += lstats->counts.blocks_fetched; tabentry->blocks_hit += lstats->counts.blocks_hit; + tabentry->visible_page_marks_cleared += lstats->counts.visible_page_marks_cleared; + tabentry->frozen_page_marks_cleared += lstats->counts.frozen_page_marks_cleared; /* Clamp live_tuples in case of negative delta_live_tuples */ tabentry->live_tuples = Max(tabentry->live_tuples, 0); diff --git a/src/backend/utils/adt/array_typanalyze.c b/src/backend/utils/adt/array_typanalyze.c index 52e160d6bbb..2db192d7544 100644 --- a/src/backend/utils/adt/array_typanalyze.c +++ b/src/backend/utils/adt/array_typanalyze.c @@ -314,7 +314,7 @@ compute_array_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int distinct_count; bool count_item_found; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, array_no, &isnull); if (isnull) diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 61c69b3ca4e..9f494e9fec0 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -120,6 +120,49 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +/* pg_stat_get_vacuum_failsafe_count */ +PG_STAT_GET_RELENTRY_INT64(vacuum_failsafe_count) + +/* pg_stat_get_visible_page_marks_cleared */ +PG_STAT_GET_RELENTRY_INT64(visible_page_marks_cleared) + +/* pg_stat_get_frozen_page_marks_cleared */ +PG_STAT_GET_RELENTRY_INT64(frozen_page_marks_cleared) + +#define PG_STAT_GET_RELENTRY_FLOAT8(stat) \ +Datum \ +CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ +{ \ + Oid relid = PG_GETARG_OID(0); \ + double result; \ + PgStat_StatTabEntry *tabentry; \ + \ + if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL) \ + result = 0; \ + else \ + result = (double) (tabentry->stat); \ + \ + PG_RETURN_FLOAT8(result); \ +} + +/* pg_stat_get_total_vacuum_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_vacuum_time) + +/* pg_stat_get_total_autovacuum_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autovacuum_time) + +/* pg_stat_get_total_analyze_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_analyze_time) + +/* pg_stat_get_total_autoanalyze_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autoanalyze_time) + +/* pg_stat_get_total_vacuum_delay_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_vacuum_delay_time) + +/* pg_stat_get_total_autovacuum_delay_time */ +PG_STAT_GET_RELENTRY_FLOAT8(total_autovacuum_delay_time) + #define PG_STAT_GET_RELENTRY_TIMESTAMPTZ(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -1099,6 +1142,9 @@ PG_STAT_GET_DBENTRY_INT64(conflict_tablespace) /* pg_stat_get_db_deadlocks */ PG_STAT_GET_DBENTRY_INT64(deadlocks) +/* pg_stat_get_db_vacuum_interrupt_count */ +PG_STAT_GET_DBENTRY_INT64(vacuum_interrupt_count) + /* pg_stat_get_db_sessions */ PG_STAT_GET_DBENTRY_INT64(sessions) @@ -1248,6 +1294,21 @@ PG_STAT_GET_DBENTRY_FLOAT8_MS(blk_write_time) /* pg_stat_get_db_idle_in_transaction_time */ PG_STAT_GET_DBENTRY_FLOAT8_MS(idle_in_transaction_time) +/* pg_stat_get_db_total_vacuum_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_vacuum_time) + +/* pg_stat_get_db_total_autovacuum_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_autovacuum_time) + +/* pg_stat_get_db_total_vacuum_delay_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_vacuum_delay_time) + +/* pg_stat_get_db_total_autovacuum_delay_time */ +PG_STAT_GET_DBENTRY_FLOAT8_MS(total_autovacuum_delay_time) + +/* pg_stat_get_db_vacuum_failsafe_count */ +PG_STAT_GET_DBENTRY_INT64(vacuum_failsafe_count) + /* pg_stat_get_db_session_time */ PG_STAT_GET_DBENTRY_FLOAT8_MS(session_time) diff --git a/src/backend/utils/adt/rangetypes_typanalyze.c b/src/backend/utils/adt/rangetypes_typanalyze.c index ff8700e8e43..a3e98fcf766 100644 --- a/src/backend/utils/adt/rangetypes_typanalyze.c +++ b/src/backend/utils/adt/rangetypes_typanalyze.c @@ -169,7 +169,7 @@ compute_range_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, upper; float8 length; - vacuum_delay_point(); + vacuum_delay_point(true); value = fetchfunc(stats, range_no, &isnull); if (isnull) diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 812782aabd1..0466a2d561f 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1994,6 +1994,7 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_rel->relpages = 0; relation->rd_rel->reltuples = -1; relation->rd_rel->relallvisible = 0; + relation->rd_rel->relallfrozen = 0; relation->rd_rel->relkind = RELKIND_RELATION; relation->rd_rel->relnatts = (int16) natts; relation->rd_rel->relam = HEAP_TABLE_AM_OID; @@ -4040,6 +4041,7 @@ RelationSetNewRelfilenumber(Relation relation, char persistence) classform->relpages = 0; /* it's empty until further notice */ classform->reltuples = -1; classform->relallvisible = 0; + classform->relallfrozen = 0; } classform->relfrozenxid = freezeXid; classform->relminmxid = minmulti; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1ff92994527..6e9313b393b 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -1480,6 +1480,15 @@ struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"track_cost_delay_timing", PGC_SUSET, STATS_CUMULATIVE, + gettext_noop("Collects timing statistics for cost-based vacuum delay."), + NULL + }, + &track_cost_delay_timing, + false, + NULL, NULL, NULL + }, { {"track_io_timing", PGC_SUSET, STATS_CUMULATIVE, gettext_noop("Collects timing statistics for database I/O activity."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index f1848a3d072..b8f3e6586ab 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -646,6 +646,7 @@ optimizer_analyze_root_partition = on # stats collection on root partitions #track_activities = on #track_activity_query_size = 1024 # (change requires restart) #track_counts = off +#track_cost_delay_timing = off #track_io_timing = off #track_wal_io_timing = off #track_functions = none # none, pl, all diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index 44aa78a39fb..a2039c22145 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -13,6 +13,7 @@ #ifndef APPENDONLY_COMPACTION_H #define APPENDONLY_COMPACTION_H +#include "datatype/timestamp.h" #include "nodes/pg_list.h" #include "access/appendonly_visimap.h" #include "utils/rel.h" @@ -28,9 +29,17 @@ */ typedef struct AOVacuumRelStats { - int nbytes_truncated; /* current # of bytes truncated from segment file */ - int num_dead_tuples; /* current # of dead tuples */ + int64 nbytes_truncated; /* current # of bytes truncated from segment file */ + int64 num_dead_tuples; /* current # of dead tuples */ int num_index_vacuumed; /* current # of indexes been vacuumed */ + /* Active phase durations and delays in milliseconds, excluding phase gaps. */ + double vacuum_time; + double delay_time; + int64 live_tuples; + int64 dead_tuples; + + /* the relation these stats were started for */ + Oid relid; } AOVacuumRelStats; extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h index daaa01a2578..6af0fd5304d 100644 --- a/src/include/access/visibilitymap.h +++ b/src/include/access/visibilitymap.h @@ -26,6 +26,8 @@ #define VM_ALL_FROZEN(r, b, v) \ ((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0) +extern bool visibilitymap_clear_rel(Relation rel, BlockNumber heapBlk, + Buffer vmbuf, uint8 flags); extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags); extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk, diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index c9c2fd83275..21dd3822811 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302608261 +#define CATALOG_VERSION_NO 302610070 #endif diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 0b1964b6288..b4aa1c9db38 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -68,6 +68,9 @@ CATALOG(pg_class,1259,RelationRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83,Relat /* # of all-visible blocks (not always up-to-date) */ int32 relallvisible BKI_DEFAULT(0); + /* # of all-frozen blocks (not always up-to-date) */ + int32 relallfrozen BKI_DEFAULT(0); + /* OID of toast table; 0 if none */ Oid reltoastrelid BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_class); diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 509c2e3be2c..c6a7bca35ce 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5473,6 +5473,22 @@ proname => 'pg_stat_get_autoanalyze_count', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_autoanalyze_count' }, +{ oid => '8406', descr => 'total vacuum time, in milliseconds', + proname => 'pg_stat_get_total_vacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_vacuum_time' }, +{ oid => '8407', descr => 'total autovacuum time, in milliseconds', + proname => 'pg_stat_get_total_autovacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autovacuum_time' }, +{ oid => '8408', descr => 'total analyze time, in milliseconds', + proname => 'pg_stat_get_total_analyze_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_analyze_time' }, +{ oid => '8409', descr => 'total autoanalyze time, in milliseconds', + proname => 'pg_stat_get_total_autoanalyze_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autoanalyze_time' }, { oid => '1936', descr => 'statistics: currently active backend IDs', proname => 'pg_stat_get_backend_idset', prorows => '100', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'int4', @@ -5709,6 +5725,54 @@ proname => 'pg_stat_get_db_blk_write_time', provolatile => 's', proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', prosrc => 'pg_stat_get_db_blk_write_time' }, +{ oid => '4707', + descr => 'statistics: number of times the all-visible marks in the visibility map were cleared for pages of this table', + proname => 'pg_stat_get_visible_page_marks_cleared', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_visible_page_marks_cleared' }, +{ oid => '4708', + descr => 'statistics: number of times the all-frozen marks in the visibility map were cleared for pages of this table', + proname => 'pg_stat_get_frozen_page_marks_cleared', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_frozen_page_marks_cleared' }, +{ oid => '4706', descr => 'statistics: number of vacuums interrupted by errors in database', + proname => 'pg_stat_get_db_vacuum_interrupt_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_vacuum_interrupt_count' }, +{ oid => '4700', descr => 'total vacuum delay time, in milliseconds', + proname => 'pg_stat_get_total_vacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_vacuum_delay_time' }, +{ oid => '4701', descr => 'total autovacuum delay time, in milliseconds', + proname => 'pg_stat_get_total_autovacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_total_autovacuum_delay_time' }, +{ oid => '4702', descr => 'statistics: total vacuum time of tables in database, in milliseconds', + proname => 'pg_stat_get_db_total_vacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_vacuum_time' }, +{ oid => '4703', descr => 'statistics: total autovacuum time of tables in database, in milliseconds', + proname => 'pg_stat_get_db_total_autovacuum_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_autovacuum_time' }, +{ oid => '4704', descr => 'statistics: total vacuum delay time in database, in milliseconds', + proname => 'pg_stat_get_db_total_vacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_vacuum_delay_time' }, +{ oid => '4705', descr => 'statistics: total autovacuum delay time in database, in milliseconds', + proname => 'pg_stat_get_db_total_autovacuum_delay_time', provolatile => 's', + proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_total_autovacuum_delay_time' }, +{ oid => '8691', + descr => 'statistics: number of vacuums that entered the wraparound failsafe mode', + proname => 'pg_stat_get_vacuum_failsafe_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_vacuum_failsafe_count' }, +{ oid => '8692', + descr => 'statistics: number of vacuums in database that entered the wraparound failsafe mode', + proname => 'pg_stat_get_db_vacuum_failsafe_count', provolatile => 's', + proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_db_vacuum_failsafe_count' }, { oid => '6185', descr => 'statistics: session time, in milliseconds', proname => 'pg_stat_get_db_session_time', provolatile => 's', proparallel => 'r', prorettype => 'float8', proargtypes => 'oid', diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index fc9ce7c1a66..01e0b766cd3 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -25,6 +25,7 @@ #define PROGRESS_VACUUM_NUM_INDEX_VACUUMS 4 #define PROGRESS_VACUUM_MAX_DEAD_TUPLES 5 #define PROGRESS_VACUUM_NUM_DEAD_TUPLES 6 +#define PROGRESS_VACUUM_DELAY_TIME 7 /* Phases of vacuum (as advertised via PROGRESS_VACUUM_PHASE) */ #define PROGRESS_VACUUM_PHASE_SCAN_HEAP 1 @@ -47,6 +48,7 @@ #define PROGRESS_ANALYZE_CHILD_TABLES_TOTAL 5 #define PROGRESS_ANALYZE_CHILD_TABLES_DONE 6 #define PROGRESS_ANALYZE_CURRENT_CHILD_TABLE_RELID 7 +#define PROGRESS_ANALYZE_DELAY_TIME 8 /* Phases of analyze (as advertised via PROGRESS_ANALYZE_PHASE) */ #define PROGRESS_ANALYZE_PHASE_ACQUIRE_SAMPLE_ROWS 1 diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 431cd7221a8..ffbb2a62ecf 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -276,6 +276,7 @@ typedef struct VPgClassStats BlockNumber rel_pages; double rel_tuples; BlockNumber relallvisible; + BlockNumber relallfrozen; } VPgClassStats; typedef struct VPgClassStatsCombo @@ -284,6 +285,7 @@ typedef struct VPgClassStatsCombo BlockNumber rel_pages; double rel_tuples; BlockNumber relallvisible; + BlockNumber relallfrozen; int count; /* expect to equal to the number of dispatched segments */ } VPgClassStatsCombo; @@ -434,6 +436,7 @@ extern PGDLLIMPORT int vacuum_multixact_freeze_min_age; extern PGDLLIMPORT int vacuum_multixact_freeze_table_age; extern PGDLLIMPORT int vacuum_failsafe_age; extern PGDLLIMPORT int vacuum_multixact_failsafe_age; +extern PGDLLIMPORT bool track_cost_delay_timing; /* Variables for cost-based parallel vacuum */ extern PGDLLIMPORT pg_atomic_uint32 *VacuumSharedCostBalance; @@ -444,6 +447,9 @@ extern PGDLLIMPORT bool VacuumFailsafeActive; extern PGDLLIMPORT double vacuum_cost_delay; extern PGDLLIMPORT int vacuum_cost_limit; +extern PGDLLIMPORT int64 parallel_vacuum_worker_delay_ns; +extern PGDLLIMPORT double VacuumDelayTime; + /* in commands/vacuum.c */ extern void ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_stats); extern void vacuum(List *relations, VacuumParams *params, @@ -459,11 +465,13 @@ extern double vac_estimate_reltuples(Relation relation, extern void vac_send_relstats_to_qd(Relation relation, BlockNumber num_pages, double num_tuples, - BlockNumber num_all_visible_pages); + BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages); extern void vac_update_relstats(Relation relation, BlockNumber num_pages, double num_tuples, BlockNumber num_all_visible_pages, + BlockNumber num_all_frozen_pages, bool hasindex, TransactionId frozenxid, MultiXactId minmulti, @@ -475,7 +483,7 @@ extern bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs); extern bool vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs); extern void vac_update_datfrozenxid(void); -extern void vacuum_delay_point(void); +extern void vacuum_delay_point(bool is_analyze); extern bool vacuum_is_relation_owner(Oid relid, Form_pg_class reltuple, bits32 options); extern Relation vacuum_open_relation(Oid relid, RangeVar *relation, diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2a85773c30c..cc557cf4848 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -180,6 +180,14 @@ typedef struct PgStat_TableCounts PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + + /* + * Revocations of all-visible and all-frozen marks in the visibility map. + * The bits are cleared by the page modification itself, so these count + * whether or not the transaction commits. + */ + PgStat_Counter visible_page_marks_cleared; + PgStat_Counter frozen_page_marks_cleared; } PgStat_TableCounts; /* ---------- @@ -238,7 +246,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCAD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCB4 typedef struct PgStat_ArchiverStats { @@ -354,6 +362,26 @@ typedef struct PgStat_StatDBEntry PgStat_Counter sessions_fatal; PgStat_Counter sessions_killed; + /* + * Cumulative time vacuums spent processing tables of this database and + * sleeping in cost-based delay points, in microseconds (the delay parts + * advance only when track_cost_delay_timing is enabled). Times of index + * processing are included in the owning table's run. + */ + PgStat_Counter total_vacuum_time; + PgStat_Counter total_autovacuum_time; + PgStat_Counter total_vacuum_delay_time; + PgStat_Counter total_autovacuum_delay_time; + + /* + * Number of vacuums in this database that entered the wraparound + * failsafe mode (see vacuum_failsafe_age). + */ + PgStat_Counter vacuum_failsafe_count; + + /* # of vacuums in this database interrupted by errors */ + PgStat_Counter vacuum_interrupt_count; + TimestampTz stat_reset_timestamp; } PgStat_StatDBEntry; @@ -418,6 +446,8 @@ typedef struct PgStat_StatTabEntry PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + PgStat_Counter visible_page_marks_cleared; + PgStat_Counter frozen_page_marks_cleared; TimestampTz last_vacuum_time; /* user initiated vacuum */ PgStat_Counter vacuum_count; @@ -427,6 +457,25 @@ typedef struct PgStat_StatTabEntry PgStat_Counter analyze_count; TimestampTz last_autoanalyze_time; /* autovacuum initiated */ PgStat_Counter autoanalyze_count; + + PgStat_Counter total_vacuum_time; /* times in milliseconds */ + PgStat_Counter total_autovacuum_time; + PgStat_Counter total_analyze_time; + PgStat_Counter total_autoanalyze_time; + + /* + * Time slept in cost-based vacuum delay points while vacuuming this + * relation, in milliseconds; nonzero only when track_cost_delay_timing + * is enabled. + */ + PgStat_Counter total_vacuum_delay_time; + PgStat_Counter total_autovacuum_delay_time; + + /* + * Number of vacuums of this relation that entered the wraparound + * failsafe mode (see vacuum_failsafe_age). + */ + PgStat_Counter vacuum_failsafe_count; } PgStat_StatTabEntry; typedef struct PgStat_WalStats @@ -650,11 +699,22 @@ extern void pgstat_init_relation(Relation rel); extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); +extern void pgstat_report_vacuum_elapsed(Oid tableoid, bool shared, + PgStat_Counter livetuples, PgStat_Counter deadtuples, + PgStat_Counter elapsedtime, PgStat_Counter delaytime, + bool failsafe); extern void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + TimestampTz starttime, PgStat_Counter delaytime, + bool failsafe); +extern void pgstat_report_index_vacuum_time(Relation rel, + PgStat_Counter elapsedtime, + PgStat_Counter delaytime, + bool is_autovacuum); +extern void pgstat_count_vacuum_error(bool shared); extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, - bool resetcounter); + bool resetcounter, TimestampTz starttime); /* * If stats are enabled, but pending data hasn't been prepared yet, call @@ -703,6 +763,26 @@ extern void pgstat_report_analyze(Relation rel, (rel)->pgstat_info->counts.blocks_hit++; \ } while (0) +/* + * Count revocations of all-visible and all-frozen marks in the visibility + * map. Only tables have a visibility map, so these always work on the + * table's pending counters. + */ +#define pgstat_count_visible_page_marks_cleared(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + { \ + (rel)->pgstat_info->counts.visible_page_marks_cleared++; \ + } \ + } while (0) +#define pgstat_count_frozen_page_marks_cleared(rel) \ + do { \ + if (pgstat_should_count_relation(rel)) \ + { \ + (rel)->pgstat_info->counts.frozen_page_marks_cleared++; \ + } \ + } while (0) + extern void pgstat_count_heap_insert(Relation rel, PgStat_Counter n); extern void pgstat_count_heap_update(Relation rel, bool hot, bool newpage); extern void pgstat_count_heap_delete(Relation rel); diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index c42970d35b2..3f6f3887d46 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -401,7 +401,8 @@ RelationGuessNumberOfBlocksFromSize(uint64 szbytes) RelationGetNumberOfBlocksInFork(reln, MAIN_FORKNUM) extern BlockNumber AcquireNumberOfBlocks(Relation onerel); -extern BlockNumber AcquireNumberOfAllVisibleBlocks(Relation onerel); +extern void AcquireVisibilityMapCounts(Relation onerel, BlockNumber *all_visible, + BlockNumber *all_frozen); extern bool BufferIsPermanent(Buffer buffer); extern XLogRecPtr BufferGetLSNAtomic(Buffer buffer); diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 02b04fdc5c4..235732f7266 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -616,6 +616,7 @@ "track_activities", "track_activity_query_size", "track_commit_timestamp", + "track_cost_delay_timing", "track_counts", "track_functions", "track_io_timing", diff --git a/src/test/isolation/expected/vacuum-extending-freeze.out b/src/test/isolation/expected/vacuum-extending-freeze.out new file mode 100644 index 00000000000..fbd03d8da92 --- /dev/null +++ b/src/test/isolation/expected/vacuum-extending-freeze.out @@ -0,0 +1,197 @@ +Parsed test spec with 2 sessions + +starting permutation: s2_vacuum_freeze s1_get_set_vm_flags_stats s1_update_table s1_get_cleared_vm_flags_stats s2_vacuum_freeze s1_get_set_vm_flags_stats s2_vacuum_freeze s1_select_from_index s2_delete_from_table s1_get_cleared_vm_flags_stats s2_vacuum_freeze s1_get_set_vm_flags_stats s1_commit s1_get_cleared_vm_flags_stats +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +t |t +(1 row) + +step s1_update_table: + UPDATE vestat SET x = x + 1001 where x >= 2500; + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +t |t +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +t |t +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_select_from_index: + BEGIN ISOLATION LEVEL REPEATABLE READ; + SELECT count(x) FROM vestat WHERE x > 2000; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +count +----- + 3000 +(1 row) + +step s2_delete_from_table: + DELETE FROM vestat WHERE x > 4930; + SELECT pg_stat_force_next_flush(); + +pg_stat_force_next_flush +------------------------ + +(1 row) + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +f |f +(1 row) + +step s2_vacuum_freeze: + VACUUM FREEZE vestat; + +step s1_get_set_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +relallfrozen|relallvisible +------------+------------- +f |f +(1 row) + +step s1_commit: + COMMIT; + +step s1_get_cleared_vm_flags_stats: + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; + +pg_stat_force_next_flush +------------------------ + +(1 row) + +visible_page_marks_cleared|frozen_page_marks_cleared +--------------------------+------------------------- +t |t +(1 row) + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 5418fbf8b39..45cd0847b9f 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -173,3 +173,4 @@ test: vacuum-ao-concurrent-drop test: serializable-parallel-3 test: matview-write-skew +test: vacuum-extending-freeze diff --git a/src/test/isolation/specs/vacuum-extending-freeze.spec b/src/test/isolation/specs/vacuum-extending-freeze.spec new file mode 100644 index 00000000000..21173bfaff7 --- /dev/null +++ b/src/test/isolation/specs/vacuum-extending-freeze.spec @@ -0,0 +1,127 @@ +# Check VM counts after freezing and cumulative counts of cleared VM bits. +# A repeatable-read transaction keeps deleted tuples visible to its snapshot. +# It also caches the table statistics before the other session deletes rows: +# the VM bits clear during DELETE, while the cached counters stay unchanged +# until the reader commits. Force the writer's counters to flush so this +# distinction does not depend on the statistics flush interval. + +setup +{ + CREATE TABLE vestat (x int, y int) + WITH (autovacuum_enabled = off, fillfactor = 70); + + INSERT INTO vestat + SELECT i, i FROM generate_series(1, 5000) AS g(i); + + CREATE INDEX vestat_idx ON vestat (x); + + CREATE TABLE stats_state (frozen_flag_count int, all_visibile_flag_count int, + cleared_frozen_flag_count int, cleared_all_visibile_flag_count int); + INSERT INTO stats_state VALUES (0,0,0,0); + ANALYZE vestat; + + -- Ensure stats are flushed before starting the scenario + SELECT pg_stat_force_next_flush(); +} + +teardown +{ + DROP TABLE IF EXISTS vestat; + DROP TABLE IF EXISTS stats_state; + RESET vacuum_freeze_min_age; + RESET vacuum_freeze_table_age; + +} + +session s1 +setup +{ + SET stats_fetch_consistency = 'cache'; +} + +step s1_get_set_vm_flags_stats +{ + SELECT pg_stat_force_next_flush(); + + SELECT c.relallfrozen > frozen_flag_count as relallfrozen, c.relallvisible > all_visibile_flag_count as relallvisible + FROM pg_class c, stats_state + WHERE c.oid = 'vestat'::regclass; + + UPDATE stats_state + SET frozen_flag_count = c.relallfrozen, + all_visibile_flag_count = c.relallvisible + FROM pg_class c + WHERE c.oid = 'vestat'::regclass; +} + +step s1_get_cleared_vm_flags_stats +{ + SELECT pg_stat_force_next_flush(); + + SELECT v.visible_page_marks_cleared > cleared_all_visibile_flag_count as visible_page_marks_cleared, + v.frozen_page_marks_cleared > cleared_frozen_flag_count as frozen_page_marks_cleared + FROM pg_stat_all_tables v, stats_state + WHERE v.relname = 'vestat'; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; +} + +step s1_select_from_index +{ + BEGIN ISOLATION LEVEL REPEATABLE READ; + SELECT count(x) FROM vestat WHERE x > 2000; + + UPDATE stats_state + SET cleared_all_visibile_flag_count = v.visible_page_marks_cleared, + cleared_frozen_flag_count = v.frozen_page_marks_cleared + FROM pg_stat_all_tables v + WHERE v.relname = 'vestat'; +} + +step s1_commit +{ + COMMIT; +} + +session s2 +setup +{ + -- Configure aggressive freezing vacuum behavior + SET vacuum_freeze_min_age = 0; + SET vacuum_freeze_table_age = 0; +} +step s2_delete_from_table +{ + DELETE FROM vestat WHERE x > 4930; + SELECT pg_stat_force_next_flush(); +} +step s2_vacuum_freeze +{ + VACUUM FREEZE vestat; +} + +step s1_update_table +{ + UPDATE vestat SET x = x + 1001 where x >= 2500; + SELECT pg_stat_force_next_flush(); +} + +permutation + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s1_update_table + s1_get_cleared_vm_flags_stats + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s2_vacuum_freeze + s1_select_from_index + s2_delete_from_table + s1_get_cleared_vm_flags_stats + s2_vacuum_freeze + s1_get_set_vm_flags_stats + s1_commit + s1_get_cleared_vm_flags_stats diff --git a/src/test/isolation2/expected/vacuum_ao_stats.out b/src/test/isolation2/expected/vacuum_ao_stats.out new file mode 100644 index 00000000000..d70951bc01e --- /dev/null +++ b/src/test/isolation2/expected/vacuum_ao_stats.out @@ -0,0 +1,140 @@ +-- start_matchsubs +-- +-- # the segment that reports the injected error first is not deterministic +-- +-- m/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ +-- s/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error'/ +-- end_matchsubs +-- +-- The vacuum statistics of the core for append-optimized tables: the vacuum +-- and delay times of the tables and their indexes, and vacuum_interrupt_count. +-- +CREATE TABLE vacstat_ao_row (i int, v text) WITH (appendonly=true) DISTRIBUTED BY (i); +CREATE +CREATE TABLE vacstat_ao_col (i int, v text) WITH (appendonly=true, orientation=column) DISTRIBUTED BY (i); +CREATE +CREATE INDEX vacstat_ao_row_idx ON vacstat_ao_row (i); +CREATE +CREATE INDEX vacstat_ao_col_idx ON vacstat_ao_col (i); +CREATE +INSERT INTO vacstat_ao_row SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT 3000 +INSERT INTO vacstat_ao_col SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT 3000 +DELETE FROM vacstat_ao_row WHERE i % 2 = 0; +DELETE 1500 +DELETE FROM vacstat_ao_col WHERE i % 2 = 0; +DELETE 1500 + +-- A cost-limited vacuum sleeps, in the index passes at least, so both the +-- tables and the indexes get vacuum and delay time on every segment. +1: SET track_cost_delay_timing = on; +SET +1: SET vacuum_cost_delay = '1ms'; +SET +1: SET vacuum_cost_limit = 10; +SET +1: VACUUM vacstat_ao_row, vacstat_ao_col; +VACUUM +1: SELECT relname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed FROM gp_dist_random('pg_stat_all_tables') WHERE relname LIKE 'vacstat_ao_%' GROUP BY relname ORDER BY relname; + relname | segments_timed | segments_delayed +----------------+----------------+------------------ + vacstat_ao_col | 3 | 3 + vacstat_ao_row | 3 | 3 +(2 rows) +1: SELECT indexrelname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed FROM gp_dist_random('pg_stat_all_indexes') WHERE indexrelname LIKE 'vacstat_ao_%' GROUP BY indexrelname ORDER BY indexrelname; + indexrelname | segments_timed | segments_delayed +--------------------+----------------+------------------ + vacstat_ao_col_idx | 3 | 3 + vacstat_ao_row_idx | 3 | 3 +(2 rows) +1: RESET vacuum_cost_limit; +RESET +1: RESET vacuum_cost_delay; +RESET +1: RESET track_cost_delay_timing; +RESET + +-- A vacuum interrupted by an error. Start from zero on every instance. +1: SELECT pg_stat_reset() FROM gp_dist_random('gp_id'); + pg_stat_reset +--------------- + + + +(3 rows) +1: SELECT pg_stat_reset(); + pg_stat_reset +--------------- + +(1 row) +INSERT INTO vacstat_ao_row SELECT g, repeat('y', 100) FROM generate_series(1, 100) g; +INSERT 100 +DELETE FROM vacstat_ao_row WHERE i % 2 = 1; +DELETE 1550 + +-- Make the compact phase fail on the segments. The fault is scoped to this +-- session, so that no other backend can consume it. +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'error', '', '', '', 1, -1, 0, dbid, current_setting('gp_session_id')::int) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1: VACUUM vacstat_ao_row; +ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- The phases of an append-optimized vacuum share one AOVacuumRelStats, which +-- is freed at the end of the last phase. A vacuum that fails in between used +-- to leave it behind, and the next vacuum of the relation then reused its +-- start time, so the time of everything that happened in between was added +-- to the relation's cumulative vacuum time. Let time pass. +1: SELECT pg_sleep(5); + pg_sleep +---------- + +(1 row) + +1: VACUUM vacstat_ao_row; +VACUUM + +-- The second vacuum takes milliseconds; without resetting the stats of the +-- failed one it would be credited with the five seconds slept above. +1: SELECT sum(total_vacuum_time) < 3000 AS vacuum_time_is_sane FROM gp_dist_random('pg_stat_all_tables') WHERE relname = 'vacstat_ao_row'; + vacuum_time_is_sane +--------------------- + t +(1 row) + +-- The segments that raised the error counted it; the coordinator, which only +-- dispatched the failing phase, did not. A segment may also be canceled +-- after its phase is already done, so check for "some", not for "all". +-- +-- A segment adds the count to its pending statistics when its transaction +-- ends, and flushes them when it is idle outside a transaction, which it is +-- not while its part of the failed distributed transaction waits for the +-- coordinator; the second vacuum above has ended that. Give the flush the +-- second it may wait for (PGSTAT_MIN_INTERVAL). +1: SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) +1: SELECT gp_segment_id = -1 AS coordinator, sum(vacuum_interrupt_count) > 0 AS interrupted FROM gp_stat_vacuum WHERE datname = current_database() GROUP BY 1 ORDER BY 1; + coordinator | interrupted +-------------+------------- + f | t + t | f +(2 rows) + +1: DROP TABLE vacstat_ao_row, vacstat_ao_col; +DROP +1q: ... diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 82d23731fb6..3e72b37ce0a 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -152,6 +152,7 @@ test: uao/fast_analyze_row test: uao/create_index_allows_readonly_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop +test: vacuum_ao_stats # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/src/test/isolation2/sql/vacuum_ao_stats.sql b/src/test/isolation2/sql/vacuum_ao_stats.sql new file mode 100644 index 00000000000..4fc15732c6c --- /dev/null +++ b/src/test/isolation2/sql/vacuum_ao_stats.sql @@ -0,0 +1,84 @@ +-- start_matchsubs +-- +-- # the segment that reports the injected error first is not deterministic +-- +-- m/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ +-- s/^ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error' \(seg\d+ .*\)/ERROR: fault triggered, fault name:'vacuum_ao_after_compact' fault type:'error'/ +-- end_matchsubs +-- +-- The vacuum statistics of the core for append-optimized tables: the vacuum +-- and delay times of the tables and their indexes, and vacuum_interrupt_count. +-- +CREATE TABLE vacstat_ao_row (i int, v text) WITH (appendonly=true) DISTRIBUTED BY (i); +CREATE TABLE vacstat_ao_col (i int, v text) WITH (appendonly=true, orientation=column) DISTRIBUTED BY (i); +CREATE INDEX vacstat_ao_row_idx ON vacstat_ao_row (i); +CREATE INDEX vacstat_ao_col_idx ON vacstat_ao_col (i); +INSERT INTO vacstat_ao_row SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +INSERT INTO vacstat_ao_col SELECT g, repeat('x', 100) FROM generate_series(1, 3000) g; +DELETE FROM vacstat_ao_row WHERE i % 2 = 0; +DELETE FROM vacstat_ao_col WHERE i % 2 = 0; + +-- A cost-limited vacuum sleeps, in the index passes at least, so both the +-- tables and the indexes get vacuum and delay time on every segment. +1: SET track_cost_delay_timing = on; +1: SET vacuum_cost_delay = '1ms'; +1: SET vacuum_cost_limit = 10; +1: VACUUM vacstat_ao_row, vacstat_ao_col; +1: SELECT relname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, + count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed + FROM gp_dist_random('pg_stat_all_tables') + WHERE relname LIKE 'vacstat_ao_%' GROUP BY relname ORDER BY relname; +1: SELECT indexrelname, count(*) FILTER (WHERE total_vacuum_time > 0) AS segments_timed, + count(*) FILTER (WHERE total_vacuum_delay_time > 0) AS segments_delayed + FROM gp_dist_random('pg_stat_all_indexes') + WHERE indexrelname LIKE 'vacstat_ao_%' GROUP BY indexrelname ORDER BY indexrelname; +1: RESET vacuum_cost_limit; +1: RESET vacuum_cost_delay; +1: RESET track_cost_delay_timing; + +-- A vacuum interrupted by an error. Start from zero on every instance. +1: SELECT pg_stat_reset() FROM gp_dist_random('gp_id'); +1: SELECT pg_stat_reset(); +INSERT INTO vacstat_ao_row SELECT g, repeat('y', 100) FROM generate_series(1, 100) g; +DELETE FROM vacstat_ao_row WHERE i % 2 = 1; + +-- Make the compact phase fail on the segments. The fault is scoped to this +-- session, so that no other backend can consume it. +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'error', '', '', '', 1, -1, 0, dbid, + current_setting('gp_session_id')::int) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1: VACUUM vacstat_ao_row; +1: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- The phases of an append-optimized vacuum share one AOVacuumRelStats, which +-- is freed at the end of the last phase. A vacuum that fails in between used +-- to leave it behind, and the next vacuum of the relation then reused its +-- start time, so the time of everything that happened in between was added +-- to the relation's cumulative vacuum time. Let time pass. +1: SELECT pg_sleep(5); + +1: VACUUM vacstat_ao_row; + +-- The second vacuum takes milliseconds; without resetting the stats of the +-- failed one it would be credited with the five seconds slept above. +1: SELECT sum(total_vacuum_time) < 3000 AS vacuum_time_is_sane + FROM gp_dist_random('pg_stat_all_tables') WHERE relname = 'vacstat_ao_row'; + +-- The segments that raised the error counted it; the coordinator, which only +-- dispatched the failing phase, did not. A segment may also be canceled +-- after its phase is already done, so check for "some", not for "all". +-- +-- A segment adds the count to its pending statistics when its transaction +-- ends, and flushes them when it is idle outside a transaction, which it is +-- not while its part of the failed distributed transaction waits for the +-- coordinator; the second vacuum above has ended that. Give the flush the +-- second it may wait for (PGSTAT_MIN_INTERVAL). +1: SELECT pg_sleep(2); +1: SELECT gp_segment_id = -1 AS coordinator, + sum(vacuum_interrupt_count) > 0 AS interrupted + FROM gp_stat_vacuum WHERE datname = current_database() + GROUP BY 1 ORDER BY 1; + +1: DROP TABLE vacstat_ao_row, vacstat_ao_col; +1q: diff --git a/src/test/modules/test_misc/meson.build b/src/test/modules/test_misc/meson.build index 911084ac0fe..e64c2f843f5 100644 --- a/src/test/modules/test_misc/meson.build +++ b/src/test/modules/test_misc/meson.build @@ -10,6 +10,10 @@ tests += { 't/002_tablespace.pl', 't/003_check_guc.pl', 't/004_io_direct.pl', + 't/016_index_vacuum_time.pl', + 't/017_vacuum_failsafe.pl', + 't/018_vacuum_interrupts.pl', + 't/020_relallfrozen.pl', ], }, } diff --git a/src/test/modules/test_misc/t/016_index_vacuum_time.pl b/src/test/modules/test_misc/t/016_index_vacuum_time.pl new file mode 100644 index 00000000000..4eb02ab2517 --- /dev/null +++ b/src/test/modules/test_misc/t/016_index_vacuum_time.pl @@ -0,0 +1,80 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test the total_vacuum_time counter of pg_stat_all_indexes: the time vacuum +# spent processing each index, accumulated over bulkdelete and cleanup passes, +# mirroring the table-level counter in pg_stat_all_tables. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +track_cost_delay_timing = on +]); +$node->start; + +# The index has to span enough pages for its vacuum passes to exceed +# vacuum_cost_limit on buffer hits alone (its pages are still dirty from the +# load, so dirtying them costs nothing). Scale the row count with the block +# size, so that builds with larger blocks (32kB in Cloudberry) keep the page +# count of the default 8kB. +my $nrows = 100000 * + ($node->safe_psql('postgres', 'SHOW block_size') / 8192); + +# Enough rows that the bulkdelete pass over the index takes measurable time; +# a small vacuum_cost_delay adds deterministic delay time on fast machines. +$node->safe_psql( + 'postgres', qq[ +CREATE TABLE vactime_t (id int PRIMARY KEY, v text) WITH (autovacuum_enabled = off); +INSERT INTO vactime_t SELECT g, repeat('x', 10) FROM generate_series(1, $nrows) g; +DELETE FROM vactime_t WHERE id % 2 = 0; +]); +$node->safe_psql( + 'postgres', qq[ +SET vacuum_cost_delay = '1ms'; +SET vacuum_cost_limit = 200; +VACUUM vactime_t; +]); + +# The upper bound guards against garbage such as an epoch-based elapsed time +# leaking into the counter. +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_time > 0 AND total_vacuum_time < 600000 + FROM pg_stat_all_indexes WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'total_vacuum_time advanced sanely for the index in pg_stat_all_indexes'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_delay_time > 0 AND total_vacuum_delay_time <= total_vacuum_time + FROM pg_stat_all_indexes WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'total_vacuum_delay_time advanced and not above total_vacuum_time'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_autovacuum_time = 0 FROM pg_stat_all_indexes + WHERE indexrelname = 'vactime_t_pkey']), + 't', + 'manual vacuum did not count into total_autovacuum_time'); + +# The same run must have accumulated into the database-wide totals. +is( $node->safe_psql( + 'postgres', qq[ +SELECT total_vacuum_time > 0 AND total_vacuum_time < 600000 + AND total_vacuum_delay_time > 0 + AND total_vacuum_delay_time <= total_vacuum_time + FROM pg_stat_vacuum WHERE datname = current_database()]), + 't', + 'database-wide vacuum times advanced sanely in pg_stat_vacuum'); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_misc/t/017_vacuum_failsafe.pl b/src/test/modules/test_misc/t/017_vacuum_failsafe.pl new file mode 100644 index 00000000000..9332c250477 --- /dev/null +++ b/src/test/modules/test_misc/t/017_vacuum_failsafe.pl @@ -0,0 +1,71 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that vacuums entering the wraparound failsafe mode are counted in +# pg_stat_all_tables.vacuum_failsafe_count and aggregated per database in +# pg_stat_vacuum.vacuum_failsafe_count. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +# The failsafe cutoff is clamped to 1.05 * autovacuum_freeze_max_age, so use +# the minimum allowed value to keep the number of XIDs to burn small. +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +autovacuum_freeze_max_age = 100000 +]); +$node->start; + +$node->safe_psql( + 'postgres', qq[ + CREATE TABLE tab_failsafe (i int); + INSERT INTO tab_failsafe SELECT generate_series(1, 100); +]); + +# A vacuum without failsafe pressure must not bump the counter. +$node->safe_psql('postgres', 'VACUUM tab_failsafe;'); +my $count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_all_tables WHERE relname = 'tab_failsafe';] +); +is($count, '0', 'normal vacuum does not count as failsafe'); + +# Age the table past 1.05 * autovacuum_freeze_max_age: burn XIDs with +# aborted subtransactions (each aborted subxact consumes an assigned XID). +$node->safe_psql( + 'postgres', qq[ + CREATE TABLE burn_xids (i int); + DO \$\$ + BEGIN + FOR i IN 1..110000 LOOP + BEGIN + INSERT INTO burn_xids VALUES (1); + RAISE EXCEPTION 'burn'; + EXCEPTION WHEN OTHERS THEN + END; + END LOOP; + END \$\$; +]); + +# vacuum_failsafe_age = 0 makes the (clamped) cutoff kick in immediately. +$node->safe_psql( + 'postgres', qq[ + SET vacuum_failsafe_age = 0; + SET vacuum_multixact_failsafe_age = 0; + VACUUM tab_failsafe; +]); + +$count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_all_tables WHERE relname = 'tab_failsafe';] +); +is($count, '1', 'failsafe vacuum counted in pg_stat_all_tables'); + +my $db_count = $node->safe_psql('postgres', + q[SELECT vacuum_failsafe_count FROM pg_stat_vacuum WHERE datname = 'postgres';] +); +is($db_count, '1', 'failsafe vacuum counted in pg_stat_vacuum'); + +done_testing(); diff --git a/src/test/modules/test_misc/t/018_vacuum_interrupts.pl b/src/test/modules/test_misc/t/018_vacuum_interrupts.pl new file mode 100644 index 00000000000..f1cb11462ac --- /dev/null +++ b/src/test/modules/test_misc/t/018_vacuum_interrupts.pl @@ -0,0 +1,71 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test the vacuum_interrupt_count counter of pg_stat_vacuum. +# +# vacuum_interrupt_count records how many times a vacuum in the database was +# interrupted by an error. We provoke that by starting a vacuum that sleeps +# at its cost-based delay points and canceling it, with pg_cancel_backend(), +# while it is still running. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +]); +$node->start; + +# fillfactor = 10 spreads the rows over many pages so the vacuum hits enough +# cost-delay points to stay running until we cancel it. +$node->safe_psql( + 'postgres', qq[ +CREATE TABLE vacstat_int (id int) WITH (autovacuum_enabled = off, fillfactor = 10); +INSERT INTO vacstat_int SELECT generate_series(1, 1000); +DELETE FROM vacstat_int; +]); + +# Start a vacuum that sleeps at every cost-delay point, in the background. The +# \echo lets query_until() return as soon as the VACUUM has been launched. +my $appname = 'vacuum_interrupt_test'; +my $vac = $node->background_psql('postgres', on_error_stop => 0); +$vac->query_until( + qr/start/, qq[ +SET application_name = '$appname'; +SET vacuum_cost_delay = '100ms'; +SET vacuum_cost_limit = 1; +\\echo start +VACUUM vacstat_int; +]); + +# Wait until the vacuum is actually running, then cancel it. +$node->poll_query_until( + 'postgres', qq[ +SELECT count(*) = 1 FROM pg_stat_activity + WHERE application_name = '$appname' AND query LIKE 'VACUUM%' AND state = 'active']) + or die "timed out waiting for the vacuum to start"; + +my $cancelled = $node->safe_psql( + 'postgres', qq[ +SELECT pg_cancel_backend(pid) FROM pg_stat_activity + WHERE application_name = '$appname' AND query LIKE 'VACUUM%']); +is($cancelled, 't', 'canceled the running vacuum'); + +$vac->quit; +like($vac->{stderr}, qr/canceling statement due to user request/, + 'vacuum canceled by user request'); + +is( $node->safe_psql( + 'postgres', qq[ +SELECT vacuum_interrupt_count > 0 FROM pg_stat_vacuum WHERE datname = current_database()]), + 't', + 'vacuum_interrupt_count advanced in pg_stat_vacuum'); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_misc/t/020_relallfrozen.pl b/src/test/modules/test_misc/t/020_relallfrozen.pl new file mode 100644 index 00000000000..5bc2f0b085b --- /dev/null +++ b/src/test/modules/test_misc/t/020_relallfrozen.pl @@ -0,0 +1,67 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Exercise the catalog update paths independently of vacuum reporting. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('relallfrozen'); +$node->init; +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->start; +$node->safe_psql('postgres', q{ +CREATE EXTENSION pg_visibility; +CREATE TABLE frozen_heap (id int, padding text); +INSERT INTO frozen_heap SELECT i, repeat('x', 100) FROM generate_series(1, 2000) i; +VACUUM (FREEZE) frozen_heap; +}); + +sub check_map +{ + my ($label) = @_; + is($node->safe_psql('postgres', q{ +SELECT relallfrozen = all_frozen AND relallvisible = all_visible + AND relallfrozen <= relallvisible AND relallvisible <= relpages +FROM pg_class, pg_visibility_map_summary('frozen_heap') +WHERE oid = 'frozen_heap'::regclass; +}), 't', $label); +} + +check_map('VACUUM FREEZE records both visibility-map counts'); +is($node->safe_psql('postgres', q{ +SELECT relallfrozen > 0 FROM pg_class WHERE oid = 'frozen_heap'::regclass +}), 't', 'frozen heap has all-frozen pages'); +$node->safe_psql('postgres', q{ +UPDATE frozen_heap SET padding = 'updated' WHERE id = 1; +ANALYZE frozen_heap; +}); +check_map('ANALYZE refreshes both counts after a VM bit is cleared'); +$node->safe_psql('postgres', 'CREATE INDEX frozen_heap_idx ON frozen_heap(id)'); +check_map('CREATE INDEX preserves heap visibility-map counts'); +is($node->safe_psql('postgres', q{ +SELECT relallfrozen = 0 AND relallvisible = 0 +FROM pg_class WHERE oid = 'frozen_heap_idx'::regclass +}), 't', 'index has zero visibility-map counts'); +$node->safe_psql('postgres', 'VACUUM (FULL) frozen_heap'); +check_map('VACUUM FULL swaps visibility-map statistics with the new heap'); +$node->safe_psql('postgres', 'TRUNCATE frozen_heap'); +check_map('TRUNCATE resets both visibility-map counts'); +for my $orientation ('row', 'column') +{ + $node->safe_psql('postgres', qq{ +CREATE TABLE frozen_ao_$orientation (id int) +WITH (appendoptimized=true, orientation=$orientation); +INSERT INTO frozen_ao_$orientation SELECT generate_series(1, 100); +CREATE INDEX frozen_ao_${orientation}_idx ON frozen_ao_$orientation(id); +VACUUM (ANALYZE) frozen_ao_$orientation; +}); + is($node->safe_psql('postgres', qq{ +SELECT bool_and(relallfrozen = 0 AND relallvisible = 0) +FROM pg_class WHERE oid IN ('frozen_ao_$orientation'::regclass, + 'frozen_ao_${orientation}_idx'::regclass) +}), 't', "AO $orientation table and index have no visibility map"); +} +$node->stop; +done_testing(); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 614ffea1784..94c1f53b4c9 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1798,7 +1798,16 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_vacuum_count(c.oid) AS vacuum_count, pg_stat_get_autovacuum_count(c.oid) AS autovacuum_count, pg_stat_get_analyze_count(c.oid) AS analyze_count, - pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count + pg_stat_get_autoanalyze_count(c.oid) AS autoanalyze_count, + pg_stat_get_total_vacuum_time(c.oid) AS total_vacuum_time, + pg_stat_get_total_autovacuum_time(c.oid) AS total_autovacuum_time, + pg_stat_get_total_analyze_time(c.oid) AS total_analyze_time, + pg_stat_get_total_autoanalyze_time(c.oid) AS total_autoanalyze_time, + pg_stat_get_total_vacuum_delay_time(c.oid) AS total_vacuum_delay_time, + pg_stat_get_total_autovacuum_delay_time(c.oid) AS total_autovacuum_delay_time, + pg_stat_get_vacuum_failsafe_count(c.oid) AS vacuum_failsafe_count, + pg_stat_get_visible_page_marks_cleared(c.oid) AS visible_page_marks_cleared, + pg_stat_get_frozen_page_marks_cleared(c.oid) AS frozen_page_marks_cleared FROM ((pg_class c LEFT JOIN pg_index i ON ((c.oid = i.indrelid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) @@ -1916,7 +1925,8 @@ pg_stat_progress_analyze| SELECT d.gp_segment_id, s.param5 AS ext_stats_computed, s.param6 AS child_tables_total, s.param7 AS child_tables_done, - (s.param8)::oid AS current_child_table_relid + (s.param8)::oid AS current_child_table_relid, + ((s.param9)::double precision / (1000000)::double precision) AS delay_time FROM (pg_stat_get_progress_info('ANALYZE'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_basebackup| SELECT gp_execution_segment() AS gp_segment_id, @@ -2043,7 +2053,8 @@ pg_stat_progress_vacuum| SELECT d.gp_segment_id, s.param4 AS heap_blks_vacuumed, s.param5 AS index_vacuum_count, s.param6 AS max_dead_tuples, - s.param7 AS num_dead_tuples + s.param7 AS num_dead_tuples, + ((s.param8)::double precision / (1000000)::double precision) AS delay_time FROM (pg_stat_get_progress_info('VACUUM'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_recovery_prefetch| SELECT stats_reset, @@ -2168,7 +2179,16 @@ pg_stat_sys_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + total_vacuum_time, + total_autovacuum_time, + total_analyze_time, + total_autoanalyze_time, + total_vacuum_delay_time, + total_autovacuum_delay_time, + vacuum_failsafe_count, + visible_page_marks_cleared, + frozen_page_marks_cleared FROM pg_stat_all_tables WHERE ((schemaname = ANY (ARRAY['pg_catalog'::name, 'information_schema'::name])) OR (schemaname ~ '^pg_toast'::text)); pg_stat_user_functions| SELECT p.oid AS funcid, @@ -2216,9 +2236,33 @@ pg_stat_user_tables| SELECT relid, vacuum_count, autovacuum_count, analyze_count, - autoanalyze_count + autoanalyze_count, + total_vacuum_time, + total_autovacuum_time, + total_analyze_time, + total_autoanalyze_time, + total_vacuum_delay_time, + total_autovacuum_delay_time, + vacuum_failsafe_count, + visible_page_marks_cleared, + frozen_page_marks_cleared FROM pg_stat_all_tables WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum| SELECT d.oid AS datid, + d.datname, + pg_stat_get_db_total_vacuum_time(d.oid) AS total_vacuum_time, + pg_stat_get_db_total_autovacuum_time(d.oid) AS total_autovacuum_time, + pg_stat_get_db_total_vacuum_delay_time(d.oid) AS total_vacuum_delay_time, + pg_stat_get_db_total_autovacuum_delay_time(d.oid) AS total_autovacuum_delay_time, + pg_stat_get_db_vacuum_failsafe_count(d.oid) AS vacuum_failsafe_count, + pg_stat_get_db_vacuum_interrupt_count(d.oid) AS vacuum_interrupt_count, + pg_stat_get_db_stat_reset_time(d.oid) AS stats_reset + FROM ( SELECT 0 AS oid, + NULL::name AS datname + UNION ALL + SELECT pg_database.oid, + pg_database.datname + FROM pg_database) d; pg_stat_wal| SELECT wal_records, wal_fpi, wal_bytes, diff --git a/src/test/regress/expected/vacuum_stats.out b/src/test/regress/expected/vacuum_stats.out index 573f7527fde..8b302003e8a 100644 --- a/src/test/regress/expected/vacuum_stats.out +++ b/src/test/regress/expected/vacuum_stats.out @@ -28,4 +28,170 @@ AND relallvisible = t (1 row) -DROP TABLE vacstat_test +DROP TABLE vacstat_test; +-- relallfrozen follows segment catalogs for dist tables. +CREATE TABLE frozen_stats_dist (a int) DISTRIBUTED BY (a); +INSERT INTO frozen_stats_dist SELECT generate_series(1, 2000); +ANALYZE frozen_stats_dist; +VACUUM (FREEZE) frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +UPDATE frozen_stats_dist SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +VACUUM (FULL) frozen_stats_dist; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; + rewrite_clears_map +-------------------- + t +(1 row) + +DROP TABLE frozen_stats_dist; +-- relallfrozen follows segment catalogs for repl tables. +CREATE TABLE frozen_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO frozen_stats_repl SELECT generate_series(1, 2000); +ANALYZE frozen_stats_repl; +VACUUM (FREEZE) frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +UPDATE frozen_stats_repl SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + frozen_stats_match +-------------------- + t +(1 row) + +VACUUM (FULL) frozen_stats_repl; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; + rewrite_clears_map +-------------------- + t +(1 row) + +DROP TABLE frozen_stats_repl; +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; + segment_timing_local +---------------------- + t +(1 row) + +RESET track_cost_delay_timing; +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +CREATE TABLE core_timing_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO core_timing_repl SELECT generate_series(1, 1000); +DELETE FROM core_timing_repl; +VACUUM core_summary_test, core_timing_repl; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; + summary_indexes | distinct_indexes +-----------------+------------------ + 2 | 2 +(1 row) + +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; + index_totals_match +-------------------- + t +(1 row) + +-- Native timing summaries work without a statistics extension. +SELECT count(*) = 2 AND bool_and(total_vacuum_time > 0) AS tables_vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname IN ('core_summary_test', 'core_timing_repl'); + tables_vacuum_timed +--------------------- + t +(1 row) + +-- Flush database timing before checking its cluster aggregate. +SELECT gp_stat_force_next_flush(); + gp_stat_force_next_flush +-------------------------- + +(1 row) + +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); + database_vacuum_timed +----------------------- + t +(1 row) + +DROP TABLE core_summary_test, core_timing_repl; diff --git a/src/test/regress/sql/vacuum_stats.sql b/src/test/regress/sql/vacuum_stats.sql index 45a9b6466c4..83620a49294 100644 --- a/src/test/regress/sql/vacuum_stats.sql +++ b/src/test/regress/sql/vacuum_stats.sql @@ -19,4 +19,104 @@ AND relallvisible = (SELECT SUM(relallvisible) FROM gp_dist_random('pg_class') WHERE oid='vacstat_test'::regclass); -DROP TABLE vacstat_test +DROP TABLE vacstat_test; + +-- relallfrozen follows segment catalogs for dist tables. +CREATE TABLE frozen_stats_dist (a int) DISTRIBUTED BY (a); +INSERT INTO frozen_stats_dist SELECT generate_series(1, 2000); +ANALYZE frozen_stats_dist; +VACUUM (FREEZE) frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +UPDATE frozen_stats_dist SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_dist; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_dist'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +VACUUM (FULL) frozen_stats_dist; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_dist'::regclass; +DROP TABLE frozen_stats_dist; + +-- relallfrozen follows segment catalogs for repl tables. +CREATE TABLE frozen_stats_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO frozen_stats_repl SELECT generate_series(1, 2000); +ANALYZE frozen_stats_repl; +VACUUM (FREEZE) frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +UPDATE frozen_stats_repl SET a = a + 10000 WHERE a = 1; +ANALYZE frozen_stats_repl; +SELECT relallfrozen > 0 AND relallfrozen <= relallvisible + AND relallfrozen = (SELECT sum(relallfrozen) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AND relallvisible = (SELECT sum(relallvisible) FROM gp_dist_random('pg_class') + WHERE oid = 'frozen_stats_repl'::regclass) / (SELECT numsegments FROM gp_distribution_policy WHERE localoid = 'frozen_stats_repl'::regclass) + AS frozen_stats_match + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +VACUUM (FULL) frozen_stats_repl; +SELECT relallfrozen = 0 AND relallvisible = 0 AS rewrite_clears_map + FROM pg_class WHERE oid = 'frozen_stats_repl'::regclass; +DROP TABLE frozen_stats_repl; + +-- Core GUC regression: reconnect with timing enabled at session startup. +-- Segment processes must keep the default off across coordinator startup +-- options, SET, and ROLLBACK. All three checks expect t. +\connect -reuse-previous=on "options='-c track_cost_delay_timing=on'" +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +BEGIN; +SET track_cost_delay_timing = off; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +ROLLBACK; +SELECT bool_and(setting = 'off') AS segment_timing_local + FROM gp_dist_random('pg_settings') WHERE name = 'track_cost_delay_timing'; +RESET track_cost_delay_timing; + +-- Core index-summary regression: two indexes on the same table must give +-- exactly two summary rows. Joining by table OID alone used to multiply +-- rows and mix index timings; compare each index with its own segment sum. +CREATE TABLE core_summary_test (a int, b int) DISTRIBUTED BY (a); +CREATE INDEX core_summary_a ON core_summary_test(a); +CREATE INDEX core_summary_b ON core_summary_test(b); +INSERT INTO core_summary_test SELECT g, g FROM generate_series(1, 10000) g; +DELETE FROM core_summary_test WHERE a % 2 = 0; +CREATE TABLE core_timing_repl (a int) DISTRIBUTED REPLICATED; +INSERT INTO core_timing_repl SELECT generate_series(1, 1000); +DELETE FROM core_timing_repl; +VACUUM core_summary_test, core_timing_repl; +SELECT count(*) AS summary_indexes, + count(DISTINCT indexrelid) AS distinct_indexes + FROM gp_stat_all_indexes_summary WHERE relname = 'core_summary_test'; +SELECT bool_and(s.total_vacuum_time = d.total_vacuum_time) AS index_totals_match + FROM gp_stat_all_indexes_summary s + JOIN (SELECT indexrelid, sum(total_vacuum_time) AS total_vacuum_time + FROM gp_dist_random('pg_stat_all_indexes') + WHERE relname = 'core_summary_test' GROUP BY indexrelid) d + USING (indexrelid) + WHERE s.relname = 'core_summary_test'; +-- Native timing summaries work without a statistics extension. +SELECT count(*) = 2 AND bool_and(total_vacuum_time > 0) AS tables_vacuum_timed + FROM gp_stat_all_tables_summary + WHERE relname IN ('core_summary_test', 'core_timing_repl'); +-- Flush database timing before checking its cluster aggregate. +SELECT gp_stat_force_next_flush(); +SELECT total_vacuum_time > 0 AS database_vacuum_timed + FROM gp_stat_vacuum_summary + WHERE datname = current_database(); +DROP TABLE core_summary_test, core_timing_repl;