Support base backup targets.

pg_basebackup now has a --target=TARGET[:DETAIL] option. If specfied,
it is sent to the server as the value of the TARGET option to the
BASE_BACKUP command. If DETAIL is included, it is sent as the value of
the new TARGET_DETAIL option to the BASE_BACKUP command.  If the
target is anything other than 'client', pg_basebackup assumes that it
will now be the server's job to write the backup in a location somehow
defined by the target, and that it therefore needs to write nothing
locally. However, the server will still send messages to the client
for progress reporting purposes.

On the server side, we now support two additional types of backup
targets.  There is a 'blackhole' target, which just throws away the
backup data without doing anything at all with it. Naturally, this
should only be used for testing and debugging purposes, since you will
not actually have a backup when it finishes running. More usefully,
there is also a 'server' target, so you can now use something like
'pg_basebackup -Xnone -t server:/SOME/PATH' to write a backup to some
location on the server. We can extend this to more types of targets
in the future, and might even want to create an extensibility
mechanism for adding new target types.

Since WAL fetching is handled with separate client-side logic, it's
not part of this mechanism; thus, backups with non-default targets
must use -Xnone or -Xfetch.

Patch by me, with a bug fix by Jeevan Ladhe.  The patch set of which
this is a part has also had review and/or testing from Tushar Ahuja,
Suraj Kharage, Dipesh Pandit, and Mark Dilger.

Discussion: http://postgr.es/m/CA+TgmoaYZbz0=Yk797aOJwkGJC-LK3iXn+wzzMx7KdwNpZhS5g@mail.gmail.com

Author: robertmhaas

doc/src/sgml/protocol.sgml

@@ -2640,8 +2640,27 @@ The commands accepted in replication mode are:
          </para>
 
          <para>
-          At present, the only supported value for this parameter is
-          <literal>client</literal>.
+          If the target is <literal>client</literal>, the backup data is
+          sent to the client. If it is <literal>server</literal>, the backup
+          data is written to the server at the pathname specified by the
+          <literal>TARGET_DETAIL</literal> option. If it is
+          <literal>blackhole</literal>, the backup data is not sent
+          anywhere; it is simply discarded.
+         </para>
+        </listitem>
+       </varlistentry>
+
+       <varlistentry>
+        <term><literal>TARGET_DETAIL</literal> <replaceable>'detail'</replaceable></term>
+        <listitem>
+         <para>
+          Provides additional information about the backup target.
+         </para>
+
+         <para>
+          Currently, this option can only be used when the backup target is
+          <literal>server</literal>. It specifies the server directory
+          to which the backup should be written.
          </para>
         </listitem>
        </varlistentry>

doc/src/sgml/ref/pg_basebackup.sgml

@@ -224,6 +224,36 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-t <replaceable class="parameter">target</replaceable></option></term>
+      <term><option>--target=<replaceable class="parameter">target</replaceable></option></term>
+      <listitem>
+
+       <para>
+        Instructs the server where to place the base backup. The default target
+        is <literal>client</literal>, which specifies that the backup should
+        be sent to the machine where <application>pg_basebackup</application>
+        is running. If the target is instead set to
+        <literal>server:/some/path</literal>, the backup will be stored on
+        the machine where the server is running in the
+        <literal>/some/path</literal> directory. Storing a backup on the
+        server requires superuser privileges. If the target is set to
+        <literal>blackhole</literal>, the contents are discarded and not
+        stored anywhere. This should only be used for testing purposes, as you
+        will not end up with an actual backup.
+       </para>
+
+       <para>
+        Since WAL streaming is implemented by
+        <application>pg_basebackup</application> rather than by the server,
+        this option cannot be used together with <literal>-Xstream</literal>.
+        Since that is the default, when this option is specified, you must also
+        specify either <literal>-Xfetch</literal> or <literal>-Xnone</literal>.
+       </para>
+
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-T <replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>
       <term><option>--tablespace-mapping=<replaceable class="parameter">olddir</replaceable>=<replaceable class="parameter">newdir</replaceable></option></term>

src/backend/replication/Makefile

@@ -19,6 +19,7 @@ OBJS = \
 	basebackup.o \
 	basebackup_copy.o \
 	basebackup_progress.o \
+	basebackup_server.o \
 	basebackup_sink.o \
 	basebackup_throttle.o \
 	repl_gram.o \

src/backend/replication/basebackup.c

@@ -55,8 +55,10 @@
 
 typedef enum
 {
+	BACKUP_TARGET_BLACKHOLE,
 	BACKUP_TARGET_COMPAT,
-	BACKUP_TARGET_CLIENT
+	BACKUP_TARGET_CLIENT,
+	BACKUP_TARGET_SERVER
 } backup_target_type;
 
 typedef struct
@@ -69,6 +71,7 @@ typedef struct
 	uint32		maxrate;
 	bool		sendtblspcmapfile;
 	backup_target_type target;
+	char	   *target_detail;
 	backup_manifest_option manifest;
 	pg_checksum_type manifest_checksum_type;
 } basebackup_options;
@@ -702,6 +705,8 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 	bool		o_manifest = false;
 	bool		o_manifest_checksums = false;
 	bool		o_target = false;
+	bool		o_target_detail = false;
+	char	   *target_str = "compat";	/* placate compiler */
 
 	MemSet(opt, 0, sizeof(*opt));
 	opt->target = BACKUP_TARGET_COMPAT;
@@ -847,25 +852,35 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 		}
 		else if (strcmp(defel->defname, "target") == 0)
 		{
-			char	   *optval = defGetString(defel);
+			target_str = defGetString(defel);
 
 			if (o_target)
 				ereport(ERROR,
 						(errcode(ERRCODE_SYNTAX_ERROR),
 						 errmsg("duplicate option \"%s\"", defel->defname)));
-			if (strcmp(optval, "client") == 0)
+			if (strcmp(target_str, "blackhole") == 0)
+				opt->target = BACKUP_TARGET_BLACKHOLE;
+			else if (strcmp(target_str, "client") == 0)
 				opt->target = BACKUP_TARGET_CLIENT;
+			else if (strcmp(target_str, "server") == 0)
+				opt->target = BACKUP_TARGET_SERVER;
 			else
 				ereport(ERROR,
 						(errcode(ERRCODE_SYNTAX_ERROR),
-						 errmsg("unrecognized target: \"%s\"", optval)));
+						 errmsg("unrecognized target: \"%s\"", target_str)));
 			o_target = true;
 		}
-		else
-			ereport(ERROR,
-					errcode(ERRCODE_SYNTAX_ERROR),
-					errmsg("option \"%s\" not recognized",
-						   defel->defname));
+		else if (strcmp(defel->defname, "target_detail") == 0)
+		{
+			char	   *optval = defGetString(defel);
+
+			if (o_target_detail)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("duplicate option \"%s\"", defel->defname)));
+			opt->target_detail = optval;
+			o_target_detail = true;
+		}
 	}
 	if (opt->label == NULL)
 		opt->label = "base backup";
@@ -877,6 +892,22 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 					 errmsg("manifest checksums require a backup manifest")));
 		opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
 	}
+	if (opt->target == BACKUP_TARGET_SERVER)
+	{
+		if (opt->target_detail == NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					 errmsg("target '%s' requires a target detail",
+							target_str)));
+	}
+	else
+	{
+		if (opt->target_detail != NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					 errmsg("target '%s' does not accept a target detail",
+							target_str)));
+	}
 }
 
 
@@ -908,14 +939,38 @@ SendBaseBackup(BaseBackupCmd *cmd)
 
 	/*
 	 * If the TARGET option was specified, then we can use the new copy-stream
-	 * protocol. If not, we must fall back to the old and less capable
-	 * copy-tablespace protocol.
+	 * protocol. If the target is specifically 'client' then set up to stream
+	 * the backup to the client; otherwise, it's being sent someplace else and
+	 * should not be sent to the client.
+	 *
+	 * If the TARGET option was not specified, we must fall back to the older
+	 * and less capable copy-tablespace protocol.
 	 */
-	if (opt.target != BACKUP_TARGET_COMPAT)
-		sink = bbsink_copystream_new();
+	if (opt.target == BACKUP_TARGET_CLIENT)
+		sink = bbsink_copystream_new(true);
+	else if (opt.target != BACKUP_TARGET_COMPAT)
+		sink = bbsink_copystream_new(false);
 	else
 		sink = bbsink_copytblspc_new();
 
+	/*
+	 * If a non-default backup target is in use, arrange to send the data
+	 * wherever it needs to go.
+	 */
+	switch (opt.target)
+	{
+		case BACKUP_TARGET_BLACKHOLE:
+			/* Nothing to do, just discard data. */
+			break;
+		case BACKUP_TARGET_COMPAT:
+		case BACKUP_TARGET_CLIENT:
+			/* Nothing to do, handling above is sufficient. */
+			break;
+		case BACKUP_TARGET_SERVER:
+			sink = bbsink_server_new(sink, opt.target_detail);
+			break;
+	}
+
 	/* Set up network throttling, if client requested it */
 	if (opt.maxrate > 0)
 		sink = bbsink_throttle_new(sink, opt.maxrate);

src/backend/replication/basebackup_copy.c

@@ -44,6 +44,9 @@ typedef struct bbsink_copystream
 	/* Common information for all types of sink. */
 	bbsink		base;
 
+	/* Are we sending the archives to the client, or somewhere else? */
+	bool		send_to_client;
+
 	/*
 	 * Protocol message buffer. We assemble CopyData protocol messages by
 	 * setting the first character of this buffer to 'd' (archive or manifest
@@ -131,11 +134,12 @@ const bbsink_ops bbsink_copytblspc_ops = {
  * Create a new 'copystream' bbsink.
  */
 bbsink *
-bbsink_copystream_new(void)
+bbsink_copystream_new(bool send_to_client)
 {
 	bbsink_copystream *sink = palloc0(sizeof(bbsink_copystream));
 
 	*((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_copystream_ops;
+	sink->send_to_client = send_to_client;
 
 	/* Set up for periodic progress reporting. */
 	sink->last_progress_report_time = GetCurrentTimestamp();
@@ -212,8 +216,12 @@ bbsink_copystream_archive_contents(bbsink *sink, size_t len)
 	StringInfoData buf;
 	uint64		targetbytes;
 
-	/* Send the archive content to the client (with leading type byte). */
-	pq_putmessage('d', mysink->msgbuffer, len + 1);
+	/* Send the archive content to the client, if appropriate. */
+	if (mysink->send_to_client)
+	{
+		/* Add one because we're also sending a leading type byte. */
+		pq_putmessage('d', mysink->msgbuffer, len + 1);
+	}
 
 	/* Consider whether to send a progress report to the client. */
 	targetbytes = mysink->bytes_done_at_last_time_check
@@ -294,8 +302,11 @@ bbsink_copystream_manifest_contents(bbsink *sink, size_t len)
 {
 	bbsink_copystream *mysink = (bbsink_copystream *) sink;
 
-	/* Send the manifest content to the client (with leading type byte). */
-	pq_putmessage('d', mysink->msgbuffer, len + 1);
+	if (mysink->send_to_client)
+	{
+		/* Add one because we're also sending a leading type byte. */
+		pq_putmessage('d', mysink->msgbuffer, len + 1);
+	}
 }
 
 /*