Skip to content

Commit ab0dfc9

Browse files
alvherrealvherre
committed
Report progress of CREATE INDEX operations
This uses the progress reporting infrastructure added by c16dc1a, adding support for CREATE INDEX and CREATE INDEX CONCURRENTLY. There are two pieces to this: one is index-AM-agnostic, and the other is AM-specific. The latter is fairly elaborate for btrees, including reportage for parallel index builds and the separate phases that btree index creation uses; other index AMs, which are much simpler in their building procedures, have simplistic reporting only, but that seems sufficient, at least for non-concurrent builds. The index-AM-agnostic part is fairly complete, providing insight into the CONCURRENTLY wait phases as well as block-based progress during the index validation table scan. (The index validation index scan requires patching each AM, which has not been included here.) Reviewers: Rahila Syed, Pavan Deolasee, Tatsuro Yamada Discussion: https://postgr.es/m/[email protected]
1 parent 4d0e994 commit ab0dfc9

37 files changed

+768
-46
lines changed

‎contrib/amcheck/verify_nbtree.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,7 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace,
566566
RelationGetRelationName(state->rel),
567567
RelationGetRelationName(state->heaprel));
568568

569-
table_index_build_scan(state->heaprel, state->rel, indexinfo, true,
569+
table_index_build_scan(state->heaprel, state->rel, indexinfo, true, false,
570570
bt_tuple_present_callback, (void *) state, scan);
571571

572572
ereport(DEBUG1,

‎contrib/bloom/blinsert.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ blbuild(Relation heap, Relation index, IndexInfo *indexInfo)
142142
initCachedPage(&buildstate);
143143

144144
/* Do the heap scan */
145-
reltuples = table_index_build_scan(heap, index, indexInfo, true,
145+
reltuples = table_index_build_scan(heap, index, indexInfo, true, true,
146146
bloomBuildCallback, (void *) &buildstate,
147147
NULL);
148148

‎contrib/bloom/blutils.c

+1
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ blhandler(PG_FUNCTION_ARGS)
132132
amroutine->amcostestimate = blcostestimate;
133133
amroutine->amoptions = bloptions;
134134
amroutine->amproperty = NULL;
135+
amroutine->ambuildphasename = NULL;
135136
amroutine->amvalidate = blvalidate;
136137
amroutine->ambeginscan = blbeginscan;
137138
amroutine->amrescan = blrescan;

‎doc/src/sgml/indexam.sgml

+13
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ typedef struct IndexAmRoutine
127127
amcostestimate_function amcostestimate;
128128
amoptions_function amoptions;
129129
amproperty_function amproperty; /* can be NULL */
130+
ambuildphasename_function ambuildphasename; /* can be NULL */
130131
amvalidate_function amvalidate;
131132
ambeginscan_function ambeginscan;
132133
amrescan_function amrescan;
@@ -468,6 +469,18 @@ amproperty (Oid index_oid, int attno,
468469

469470
<para>
470471
<programlisting>
472+
char *
473+
ambuildphasename (int64 phasenum);
474+
</programlisting>
475+
Return the textual name of the given build phase number.
476+
The phase numbers are those reported during an index build via the
477+
<function>pgstat_progress_update_param</function> interface.
478+
The phase names are then exposed in the
479+
<structname>pg_stat_progress_create_index</structname> view.
480+
</para>
481+
482+
<para>
483+
<programlisting>
471484
bool
472485
amvalidate (Oid opclassoid);
473486
</programlisting>

‎doc/src/sgml/monitoring.sgml

+223-1
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
336336
</entry>
337337
</row>
338338

339+
<row>
340+
<entry><structname>pg_stat_progress_create_index</structname><indexterm><primary>pg_stat_progress_create_index</primary></indexterm></entry>
341+
<entry>One row for each backend running <command>CREATE INDEX</command>, showing
342+
current progress.
343+
See <xref linkend='create-index-progress-reporting'/>.
344+
</entry>
345+
</row>
346+
339347
<row>
340348
<entry><structname>pg_stat_progress_vacuum</structname><indexterm><primary>pg_stat_progress_vacuum</primary></indexterm></entry>
341349
<entry>One row for each backend (including autovacuum worker processes) running
@@ -3403,10 +3411,224 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
34033411
<para>
34043412
<productname>PostgreSQL</productname> has the ability to report the progress of
34053413
certain commands during command execution. Currently, the only commands
3406-
which support progress reporting are <command>VACUUM</command> and
3414+
which support progress reporting are <command>CREATE INDEX</command>,
3415+
<command>VACUUM</command> and
34073416
<command>CLUSTER</command>. This may be expanded in the future.
34083417
</para>
34093418

3419+
<sect2 id="create-index-progress-reporting">
3420+
<title>CREATE INDEX Progress Reporting</title>
3421+
3422+
<para>
3423+
Whenever <command>CREATE INDEX</command> is running, the
3424+
<structname>pg_stat_progress_create_index</structname> view will contain
3425+
one row for each backend that is currently creating indexes. The tables
3426+
below describe the information that will be reported and provide information
3427+
about how to interpret it.
3428+
</para>
3429+
3430+
<table id="pg-stat-progress-create-index-view" xreflabel="pg_stat_progress_create_index">
3431+
<title><structname>pg_stat_progress_create_index</structname> View</title>
3432+
<tgroup cols="3">
3433+
<thead>
3434+
<row>
3435+
<entry>Column</entry>
3436+
<entry>Type</entry>
3437+
<entry>Description</entry>
3438+
</row>
3439+
</thead>
3440+
3441+
<tbody>
3442+
<row>
3443+
<entry><structfield>pid</structfield></entry>
3444+
<entry><type>integer</type></entry>
3445+
<entry>Process ID of backend.</entry>
3446+
</row>
3447+
<row>
3448+
<entry><structfield>datid</structfield></entry>
3449+
<entry><type>oid</type></entry>
3450+
<entry>OID of the database to which this backend is connected.</entry>
3451+
</row>
3452+
<row>
3453+
<entry><structfield>datname</structfield></entry>
3454+
<entry><type>name</type></entry>
3455+
<entry>Name of the database to which this backend is connected.</entry>
3456+
</row>
3457+
<row>
3458+
<entry><structfield>relid</structfield></entry>
3459+
<entry><type>oid</type></entry>
3460+
<entry>OID of the table on which the index is being created.</entry>
3461+
</row>
3462+
<row>
3463+
<entry><structfield>phase</structfield></entry>
3464+
<entry><type>text</type></entry>
3465+
<entry>
3466+
Current processing phase of index creation. See <xref linkend='create-index-phases'/>.
3467+
</entry>
3468+
</row>
3469+
<row>
3470+
<entry><structfield>lockers_total</structfield></entry>
3471+
<entry><type>bigint</type></entry>
3472+
<entry>
3473+
Total number of lockers to wait for, when applicable.
3474+
</entry>
3475+
</row>
3476+
<row>
3477+
<entry><structfield>lockers_done</structfield></entry>
3478+
<entry><type>bigint</type></entry>
3479+
<entry>
3480+
Number of lockers already waited for.
3481+
</entry>
3482+
</row>
3483+
<row>
3484+
<entry><structfield>current_locked_pid</structfield></entry>
3485+
<entry><type>bigint</type></entry>
3486+
<entry>
3487+
Process ID of the locker currently being waited for.
3488+
</entry>
3489+
</row>
3490+
<row>
3491+
<entry><structfield>blocks_total</structfield></entry>
3492+
<entry><type>bigint</type></entry>
3493+
<entry>
3494+
Total number of blocks to be processed in the current phase.
3495+
</entry>
3496+
</row>
3497+
<row>
3498+
<entry><structfield>blocks_done</structfield></entry>
3499+
<entry><type>bigint</type></entry>
3500+
<entry>
3501+
Number of blocks already processed in the current phase.
3502+
</entry>
3503+
</row>
3504+
<row>
3505+
<entry><structfield>tuples_total</structfield></entry>
3506+
<entry><type>bigint</type></entry>
3507+
<entry>
3508+
Total number of tuples to be processed in the current phase.
3509+
</entry>
3510+
</row>
3511+
<row>
3512+
<entry><structfield>tuples_done</structfield></entry>
3513+
<entry><type>bigint</type></entry>
3514+
<entry>
3515+
Number of tuples already processed in the current phase.
3516+
</entry>
3517+
</row>
3518+
<row>
3519+
<entry><structfield>partitions_total</structfield></entry>
3520+
<entry><type>bigint</type></entry>
3521+
<entry>
3522+
When creating an index on a partitioned table, this column is set to
3523+
the total number of partitions on which the index is to be created.
3524+
</entry>
3525+
</row>
3526+
<row>
3527+
<entry><structfield>partitions_done</structfield></entry>
3528+
<entry><type>bigint</type></entry>
3529+
<entry>
3530+
When creating an index on a partitioned table, this column is set to
3531+
the number of partitions on which the index has been completed.
3532+
</entry>
3533+
</row>
3534+
</tbody>
3535+
</tgroup>
3536+
</table>
3537+
3538+
<table id="create-index-phases">
3539+
<title>CREATE INDEX phases</title>
3540+
<tgroup cols="2">
3541+
<thead>
3542+
<row>
3543+
<entry>Phase</entry>
3544+
<entry>Description</entry>
3545+
</row>
3546+
</thead>
3547+
<tbody>
3548+
<row>
3549+
<entry><literal>initializing</literal></entry>
3550+
<entry>
3551+
<command>CREATE INDEX</command> is preparing to create the index. This
3552+
phase is expected to be very brief.
3553+
</entry>
3554+
</row>
3555+
<row>
3556+
<entry><literal>waiting for old snapshots</literal></entry>
3557+
<entry>
3558+
<command>CREATE INDEX CONCURRENTLY</command> is waiting for transactions
3559+
that can potentially see the table to release their snapshots.
3560+
This phase is skipped when not in concurrent mode.
3561+
Columns <structname>lockers_total</structname>, <structname>lockers_done</structname>
3562+
and <structname>current_locker_pid</structname> contain the progress
3563+
information for this phase.
3564+
</entry>
3565+
</row>
3566+
<row>
3567+
<entry><literal>building index</literal></entry>
3568+
<entry>
3569+
The index is being built by the access method-specific code. In this phase,
3570+
access methods that support progress reporting fill in their own progress data,
3571+
and the subphase is indicated in this column. Typically,
3572+
<structname>blocks_total</structname> and <structname>blocks_done</structname>
3573+
will contain progress data, as well as potentially
3574+
<structname>tuples_total</structname> and <structname>tuples_done</structname>.
3575+
</entry>
3576+
</row>
3577+
<row>
3578+
<entry><literal>waiting for writer snapshots</literal></entry>
3579+
<entry>
3580+
<command>CREATE INDEX CONCURRENTLY</command> is waiting for transactions
3581+
that can potentially write into the table to release their snapshots.
3582+
This phase is skipped when not in concurrent mode.
3583+
Columns <structname>lockers_total</structname>, <structname>lockers_done</structname>
3584+
and <structname>current_locker_pid</structname> contain the progress
3585+
information for this phase.
3586+
</entry>
3587+
</row>
3588+
<row>
3589+
<entry><literal>index validation: scanning index</literal></entry>
3590+
<entry>
3591+
<command>CREATE INDEX CONCURRENTLY</command> is scanning the index searching
3592+
for tuples that need to be validated.
3593+
This phase is skipped when not in concurrent mode.
3594+
Columns <structname>blocks_total</structname> (set to the total size of the index)
3595+
and <structname>blocks_done</structname> contain the progress information for this phase.
3596+
</entry>
3597+
</row>
3598+
<row>
3599+
<entry><literal>index validation: sorting tuples</literal></entry>
3600+
<entry>
3601+
<command>CREATE INDEX CONCURRENTLY</command> is sorting the output of the
3602+
index scanning phase.
3603+
</entry>
3604+
</row>
3605+
<row>
3606+
<entry><literal>index validation: scanning table</literal></entry>
3607+
<entry>
3608+
<command>CREATE INDEX CONCURRENTLY</command> is scanning the table
3609+
to validate the index tuples collected in the previous two phases.
3610+
This phase is skipped when not in concurrent mode.
3611+
Columns <structname>blocks_total</structname> (set to the total size of the table)
3612+
and <structname>blocks_done</structname> contain the progress information for this phase.
3613+
</entry>
3614+
</row>
3615+
<row>
3616+
<entry><literal>waiting for reader snapshots</literal></entry>
3617+
<entry>
3618+
<command>CREATE INDEX CONCURRENTLY</command> is waiting for transactions
3619+
that can potentially see the table to release their snapshots. This
3620+
phase is skipped when not in concurrent mode.
3621+
Columns <structname>lockers_total</structname>, <structname>lockers_done</structname>
3622+
and <structname>current_locker_pid</structname> contain the progress
3623+
information for this phase.
3624+
</entry>
3625+
</row>
3626+
</tbody>
3627+
</tgroup>
3628+
</table>
3629+
3630+
</sect2>
3631+
34103632
<sect2 id="vacuum-progress-reporting">
34113633
<title>VACUUM Progress Reporting</title>
34123634

‎src/backend/access/brin/brin.c

+3-2
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ brinhandler(PG_FUNCTION_ARGS)
112112
amroutine->amcostestimate = brincostestimate;
113113
amroutine->amoptions = brinoptions;
114114
amroutine->amproperty = NULL;
115+
amroutine->ambuildphasename = NULL;
115116
amroutine->amvalidate = brinvalidate;
116117
amroutine->ambeginscan = brinbeginscan;
117118
amroutine->amrescan = brinrescan;
@@ -719,7 +720,7 @@ brinbuild(Relation heap, Relation index, IndexInfo *indexInfo)
719720
* Now scan the relation. No syncscan allowed here because we want the
720721
* heap blocks in physical order.
721722
*/
722-
reltuples = table_index_build_scan(heap, index, indexInfo, false,
723+
reltuples = table_index_build_scan(heap, index, indexInfo, false, true,
723724
brinbuildCallback, (void *) state, NULL);
724725

725726
/* process the final batch */
@@ -1236,7 +1237,7 @@ summarize_range(IndexInfo *indexInfo, BrinBuildState *state, Relation heapRel,
12361237
* cases.
12371238
*/
12381239
state->bs_currRangeStart = heapBlk;
1239-
table_index_build_range_scan(heapRel, state->bs_irel, indexInfo, false, true,
1240+
table_index_build_range_scan(heapRel, state->bs_irel, indexInfo, false, true, false,
12401241
heapBlk, scanNumBlks,
12411242
brinbuildCallback, (void *) state, NULL);
12421243

‎src/backend/access/gin/gininsert.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo)
395395
* Do the heap scan. We disallow sync scan here because dataPlaceToPage
396396
* prefers to receive tuples in TID order.
397397
*/
398-
reltuples = table_index_build_scan(heap, index, indexInfo, false,
398+
reltuples = table_index_build_scan(heap, index, indexInfo, false, true,
399399
ginBuildCallback, (void *) &buildstate,
400400
NULL);
401401

‎src/backend/access/gin/ginutil.c

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ ginhandler(PG_FUNCTION_ARGS)
6464
amroutine->amcostestimate = gincostestimate;
6565
amroutine->amoptions = ginoptions;
6666
amroutine->amproperty = NULL;
67+
amroutine->ambuildphasename = NULL;
6768
amroutine->amvalidate = ginvalidate;
6869
amroutine->ambeginscan = ginbeginscan;
6970
amroutine->amrescan = ginrescan;

‎src/backend/access/gist/gist.c

+1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ gisthandler(PG_FUNCTION_ARGS)
8686
amroutine->amcostestimate = gistcostestimate;
8787
amroutine->amoptions = gistoptions;
8888
amroutine->amproperty = gistproperty;
89+
amroutine->ambuildphasename = NULL;
8990
amroutine->amvalidate = gistvalidate;
9091
amroutine->ambeginscan = gistbeginscan;
9192
amroutine->amrescan = gistrescan;

‎src/backend/access/gist/gistbuild.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo)
205205
/*
206206
* Do the heap scan.
207207
*/
208-
reltuples = table_index_build_scan(heap, index, indexInfo, true,
208+
reltuples = table_index_build_scan(heap, index, indexInfo, true, true,
209209
gistBuildCallback,
210210
(void *) &buildstate, NULL);
211211

‎src/backend/access/hash/hash.c

+6-1
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@
2323
#include "access/relscan.h"
2424
#include "access/tableam.h"
2525
#include "catalog/index.h"
26+
#include "commands/progress.h"
2627
#include "commands/vacuum.h"
2728
#include "miscadmin.h"
2829
#include "optimizer/plancat.h"
30+
#include "pgstat.h"
2931
#include "utils/builtins.h"
3032
#include "utils/index_selfuncs.h"
3133
#include "utils/rel.h"
@@ -83,6 +85,7 @@ hashhandler(PG_FUNCTION_ARGS)
8385
amroutine->amcostestimate = hashcostestimate;
8486
amroutine->amoptions = hashoptions;
8587
amroutine->amproperty = NULL;
88+
amroutine->ambuildphasename = NULL;
8689
amroutine->amvalidate = hashvalidate;
8790
amroutine->ambeginscan = hashbeginscan;
8891
amroutine->amrescan = hashrescan;
@@ -160,9 +163,11 @@ hashbuild(Relation heap, Relation index, IndexInfo *indexInfo)
160163
buildstate.heapRel = heap;
161164

162165
/* do the heap scan */
163-
reltuples = table_index_build_scan(heap, index, indexInfo, true,
166+
reltuples = table_index_build_scan(heap, index, indexInfo, true, true,
164167
hashbuildCallback,
165168
(void *) &buildstate, NULL);
169+
pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_TOTAL,
170+
buildstate.indtuples);
166171

167172
if (buildstate.spool)
168173
{

0 commit comments

Comments
 (0)