Invent SERIALIZE option for EXPLAIN.

EXPLAIN (ANALYZE, SERIALIZE) allows collection of statistics about
the volume of data emitted by a query, as well as the time taken
to convert the data to the on-the-wire format.  Previously there
was no way to investigate this without actually sending the data
to the client, in which case network transmission costs might
swamp what you wanted to see.  In particular this feature allows
investigating the costs of de-TOASTing compressed or out-of-line
data during formatting.

Stepan Rutz and Matthias van de Meent,
reviewed by Tomas Vondra and myself

Discussion: https://postgr.es/m/ca0adb0e-fa4e-c37e-1cd7-91170b18cae1@gmx.de

Author: tglsfdc

doc/src/sgml/perform.sgml

@@ -144,11 +144,11 @@ EXPLAIN SELECT * FROM tenk1;
     It's important to understand that the cost of an upper-level node includes
     the cost of all its child nodes.  It's also important to realize that
     the cost only reflects things that the planner cares about.
-    In particular, the cost does not consider the time spent transmitting
-    result rows to the client, which could be an important
-    factor in the real elapsed time; but the planner ignores it because
-    it cannot change it by altering the plan.  (Every correct plan will
-    output the same row set, we trust.)
+    In particular, the cost does not consider the time spent to convert
+    output values to text form or to transmit them to the client, which
+    could be important factors in the real elapsed time; but the planner
+    ignores those costs because it cannot change them by altering the
+    plan.  (Every correct plan will output the same row set, we trust.)
    </para>
 
    <para>
@@ -956,6 +956,17 @@ EXPLAIN UPDATE parent SET f2 = f2 + 1 WHERE f1 = 101;
     <command>EXPLAIN ANALYZE</command>.
    </para>
 
+   <para>
+    The time shown for the top-level node does not include any time needed
+    to convert the query's output data into displayable form or to send it
+    to the client.  While <command>EXPLAIN ANALYZE</command> will never
+    send the data to the client, it can be told to convert the query's
+    output data to displayable form and measure the time needed for that,
+    by specifying the <literal>SERIALIZE</literal> option.  That time will
+    be shown separately, and it's also included in the
+    total <literal>Execution time</literal>.
+   </para>
+
   </sect2>
 
   <sect2 id="using-explain-caveats">
@@ -965,7 +976,8 @@ EXPLAIN UPDATE parent SET f2 = f2 + 1 WHERE f1 = 101;
     There are two significant ways in which run times measured by
     <command>EXPLAIN ANALYZE</command> can deviate from normal execution of
     the same query.  First, since no output rows are delivered to the client,
-    network transmission costs and I/O conversion costs are not included.
+    network transmission costs are not included.  I/O conversion costs are
+    not included either unless <literal>SERIALIZE</literal> is specified.
     Second, the measurement overhead added by <command>EXPLAIN
     ANALYZE</command> can be significant, especially on machines with slow
     <function>gettimeofday()</function> operating-system calls. You can use the

doc/src/sgml/ref/explain.sgml

@@ -41,6 +41,7 @@ EXPLAIN [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] <rep
     SETTINGS [ <replaceable class="parameter">boolean</replaceable> ]
     GENERIC_PLAN [ <replaceable class="parameter">boolean</replaceable> ]
     BUFFERS [ <replaceable class="parameter">boolean</replaceable> ]
+    SERIALIZE [ { NONE | TEXT | BINARY } ]
     WAL [ <replaceable class="parameter">boolean</replaceable> ]
     TIMING [ <replaceable class="parameter">boolean</replaceable> ]
     SUMMARY [ <replaceable class="parameter">boolean</replaceable> ]
@@ -206,6 +207,34 @@ ROLLBACK;
     </listitem>
    </varlistentry>
 
+   <varlistentry>
+    <term><literal>SERIALIZE</literal></term>
+    <listitem>
+     <para>
+      Include information on the cost
+      of <firstterm>serializing</firstterm> the query's output data, that
+      is converting it to text or binary format to send to the client.
+      This can be a significant part of the time required for regular
+      execution of the query, if the datatype output functions are
+      expensive or if <acronym>TOAST</acronym>ed values must be fetched
+      from out-of-line storage.  <command>EXPLAIN</command>'s default
+      behavior, <literal>SERIALIZE NONE</literal>, does not perform these
+      conversions.  If <literal>SERIALIZE TEXT</literal>
+      or <literal>SERIALIZE BINARY</literal> is specified, the appropriate
+      conversions are performed, and the time spent doing so is measured
+      (unless <literal>TIMING OFF</literal> is specified).  If
+      the <literal>BUFFERS</literal> option is also specified, then any
+      buffer accesses involved in the conversions are counted too.
+      In no case, however, will <command>EXPLAIN</command> actually send
+      the resulting data to the client; hence network transmission costs
+      cannot be investigated this way.
+      Serialization may only be enabled when <literal>ANALYZE</literal> is
+      also enabled.  If <literal>SERIALIZE</literal> is written without an
+      argument, <literal>TEXT</literal> is assumed.
+     </para>
+    </listitem>
+   </varlistentry>
+
    <varlistentry>
     <term><literal>WAL</literal></term>
     <listitem>

src/backend/access/common/printtup.c

@@ -294,6 +294,9 @@ printtup_prepare_info(DR_printtup *myState, TupleDesc typeinfo, int numAttrs)
 
 /* ----------------
  *		printtup --- send a tuple to the client
+ *
+ * Note: if you change this function, see also serializeAnalyzeReceive
+ * in explain.c, which is meant to replicate the computations done here.
  * ----------------
  */
 static bool
@@ -317,7 +320,7 @@ printtup(TupleTableSlot *slot, DestReceiver *self)
 	oldcontext = MemoryContextSwitchTo(myState->tmpcontext);
 
 	/*
-	 * Prepare a DataRow message (note buffer is in per-row context)
+	 * Prepare a DataRow message (note buffer is in per-query context)
 	 */
 	pq_beginmessage_reuse(buf, 'D');
 

src/backend/commands/explain.c

@@ -33,6 +33,7 @@
 #include "access/xact.h"
 #include "commands/copy.h"
 #include "commands/createas.h"
+#include "commands/explain.h"
 #include "commands/matview.h"
 #include "executor/functions.h"
 #include "executor/tqueue.h"
@@ -151,6 +152,9 @@ CreateDestReceiver(CommandDest dest)
 
 		case DestTupleQueue:
 			return CreateTupleQueueDestReceiver(NULL);
+
+		case DestExplainSerialize:
+			return CreateExplainSerializeDestReceiver(NULL);
 	}
 
 	/* should never get here */
@@ -186,6 +190,7 @@ EndCommand(const QueryCompletion *qc, CommandDest dest, bool force_undecorated_o
 		case DestSQLFunction:
 		case DestTransientRel:
 		case DestTupleQueue:
+		case DestExplainSerialize:
 			break;
 	}
 }
@@ -231,6 +236,7 @@ NullCommand(CommandDest dest)
 		case DestSQLFunction:
 		case DestTransientRel:
 		case DestTupleQueue:
+		case DestExplainSerialize:
 			break;
 	}
 }
@@ -274,6 +280,7 @@ ReadyForQuery(CommandDest dest)
 		case DestSQLFunction:
 		case DestTransientRel:
 		case DestTupleQueue:
+		case DestExplainSerialize:
 			break;
 	}
 }

src/backend/tcop/dest.c

@@ -17,6 +17,13 @@
 #include "lib/stringinfo.h"
 #include "parser/parse_node.h"
 
+typedef enum ExplainSerializeOption
+{
+	EXPLAIN_SERIALIZE_NONE,
+	EXPLAIN_SERIALIZE_TEXT,
+	EXPLAIN_SERIALIZE_BINARY,
+} ExplainSerializeOption;
+
 typedef enum ExplainFormat
 {
 	EXPLAIN_FORMAT_TEXT,
@@ -48,6 +55,7 @@ typedef struct ExplainState
 	bool		memory;			/* print planner's memory usage information */
 	bool		settings;		/* print modified settings */
 	bool		generic;		/* generate a generic plan */
+	ExplainSerializeOption serialize;	/* serialize the query's output? */
 	ExplainFormat format;		/* output format */
 	/* state for output formatting --- not reset for each new plan tree */
 	int			indent;			/* current indentation level */
@@ -132,4 +140,6 @@ extern void ExplainOpenGroup(const char *objtype, const char *labelname,
 extern void ExplainCloseGroup(const char *objtype, const char *labelname,
 							  bool labeled, ExplainState *es);
 
+extern DestReceiver *CreateExplainSerializeDestReceiver(ExplainState *es);
+
 #endif							/* EXPLAIN_H */

src/include/commands/explain.h

@@ -96,6 +96,7 @@ typedef enum
 	DestSQLFunction,			/* results sent to SQL-language func mgr */
 	DestTransientRel,			/* results sent to transient relation */
 	DestTupleQueue,				/* results sent to tuple queue */
+	DestExplainSerialize,		/* results are serialized and discarded */
 } CommandDest;
 
 /* ----------------

src/include/tcop/dest.h

@@ -135,7 +135,7 @@ select explain_filter('explain (analyze, buffers, format xml) select * from int8
  </explain>
 (1 row)
 
-select explain_filter('explain (analyze, buffers, format yaml) select * from int8_tbl i8');
+select explain_filter('explain (analyze, serialize, buffers, format yaml) select * from int8_tbl i8');
         explain_filter         
 -------------------------------
  - Plan:                      +
@@ -175,6 +175,20 @@ select explain_filter('explain (analyze, buffers, format yaml) select * from int
      Temp Written Blocks: N   +
    Planning Time: N.N         +
    Triggers:                  +
+   Serialization:             +
+     Time: N.N                +
+     Output Volume: N         +
+     Format: "text"           +
+     Shared Hit Blocks: N     +
+     Shared Read Blocks: N    +
+     Shared Dirtied Blocks: N +
+     Shared Written Blocks: N +
+     Local Hit Blocks: N      +
+     Local Read Blocks: N     +
+     Local Dirtied Blocks: N  +
+     Local Written Blocks: N  +
+     Temp Read Blocks: N      +
+     Temp Written Blocks: N   +
    Execution Time: N.N
 (1 row)
 
@@ -639,3 +653,41 @@ select explain_filter('explain (verbose) select * from int8_tbl i8');
  Query Identifier: N
 (3 rows)
 
+-- Test SERIALIZE option
+select explain_filter('explain (analyze,serialize) select * from int8_tbl i8');
+                                        explain_filter                                         
+-----------------------------------------------------------------------------------------------
+ Seq Scan on int8_tbl i8  (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N)
+ Planning Time: N.N ms
+ Serialization: time=N.N ms  output=NkB  format=text
+ Execution Time: N.N ms
+(4 rows)
+
+select explain_filter('explain (analyze,serialize text,buffers,timing off) select * from int8_tbl i8');
+                                 explain_filter                                  
+---------------------------------------------------------------------------------
+ Seq Scan on int8_tbl i8  (cost=N.N..N.N rows=N width=N) (actual rows=N loops=N)
+ Planning Time: N.N ms
+ Serialization: output=NkB  format=text
+ Execution Time: N.N ms
+(4 rows)
+
+select explain_filter('explain (analyze,serialize binary,buffers,timing) select * from int8_tbl i8');
+                                        explain_filter                                         
+-----------------------------------------------------------------------------------------------
+ Seq Scan on int8_tbl i8  (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N)
+ Planning Time: N.N ms
+ Serialization: time=N.N ms  output=NkB  format=binary
+ Execution Time: N.N ms
+(4 rows)
+
+-- this tests an edge case where we have no data to return
+select explain_filter('explain (analyze,serialize) create temp table explain_temp as select * from int8_tbl i8');
+                                        explain_filter                                         
+-----------------------------------------------------------------------------------------------
+ Seq Scan on int8_tbl i8  (cost=N.N..N.N rows=N width=N) (actual time=N.N..N.N rows=N loops=N)
+ Planning Time: N.N ms
+ Serialization: time=N.N ms  output=NkB  format=text
+ Execution Time: N.N ms
+(4 rows)
+

src/test/regress/expected/explain.out

@@ -66,7 +66,7 @@ select explain_filter('explain (analyze) select * from int8_tbl i8');
 select explain_filter('explain (analyze, verbose) select * from int8_tbl i8');
 select explain_filter('explain (analyze, buffers, format text) select * from int8_tbl i8');
 select explain_filter('explain (analyze, buffers, format xml) select * from int8_tbl i8');
-select explain_filter('explain (analyze, buffers, format yaml) select * from int8_tbl i8');
+select explain_filter('explain (analyze, serialize, buffers, format yaml) select * from int8_tbl i8');
 select explain_filter('explain (buffers, format text) select * from int8_tbl i8');
 select explain_filter('explain (buffers, format json) select * from int8_tbl i8');
 
@@ -162,3 +162,10 @@ select explain_filter('explain (verbose) select * from t1 where pg_temp.mysin(f1
 -- Test compute_query_id
 set compute_query_id = on;
 select explain_filter('explain (verbose) select * from int8_tbl i8');
+
+-- Test SERIALIZE option
+select explain_filter('explain (analyze,serialize) select * from int8_tbl i8');
+select explain_filter('explain (analyze,serialize text,buffers,timing off) select * from int8_tbl i8');
+select explain_filter('explain (analyze,serialize binary,buffers,timing) select * from int8_tbl i8');
+-- this tests an edge case where we have no data to return
+select explain_filter('explain (analyze,serialize) create temp table explain_temp as select * from int8_tbl i8');

src/test/regress/sql/explain.sql

@@ -713,6 +713,7 @@ ExplainForeignModify_function
 ExplainForeignScan_function
 ExplainFormat
 ExplainOneQuery_hook_type
+ExplainSerializeOption
 ExplainState
 ExplainStmt
 ExplainWorkersState
@@ -2536,6 +2537,8 @@ SerCommitSeqNo
 SerialControl
 SerialIOData
 SerializableXactHandle
+SerializeDestReceiver
+SerializeMetrics
 SerializedActiveRelMaps
 SerializedClientConnectionInfo
 SerializedRanges