pg_upgrade: dramatically reduce memory consumption

Report from Jeff Janes

Author: bmomjian

contrib/pg_upgrade/check.c

@@ -707,28 +707,28 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name)
 			fprintf(script, "\n");
 			/* remove PG_VERSION? */
 			if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
-				fprintf(script, RM_CMD " %s%s%cPG_VERSION\n",
+				fprintf(script, RM_CMD " %s%cPG_VERSION\n",
 						fix_path_separator(os_info.old_tablespaces[tblnum]),
-						fix_path_separator(old_cluster.tablespace_suffix),
 						PATH_SEPARATOR);
 
 			for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
-			{
-				fprintf(script, RMDIR_CMD " %s%s%c%d\n",
+				fprintf(script, RMDIR_CMD " %s%c%d\n",
 						fix_path_separator(os_info.old_tablespaces[tblnum]),
-						fix_path_separator(old_cluster.tablespace_suffix),
 						PATH_SEPARATOR, old_cluster.dbarr.dbs[dbnum].db_oid);
-			}
 		}
 		else
+		{
+			char	*suffix_path = pg_strdup(old_cluster.tablespace_suffix);
 
 			/*
 			 * Simply delete the tablespace directory, which might be ".old"
 			 * or a version-specific subdirectory.
 			 */
 			fprintf(script, RMDIR_CMD " %s%s\n",
 					fix_path_separator(os_info.old_tablespaces[tblnum]),
-					fix_path_separator(old_cluster.tablespace_suffix));
+					fix_path_separator(suffix_path));
+			pfree(suffix_path);
+		}
 	}
 
 	fclose(script);

contrib/pg_upgrade/info.c

@@ -108,20 +108,18 @@ create_rel_filename_map(const char *old_data, const char *new_data,
 		 * relation belongs to the default tablespace, hence relfiles should
 		 * exist in the data directories.
 		 */
-		strlcpy(map->old_tablespace, old_data, sizeof(map->old_tablespace));
-		strlcpy(map->new_tablespace, new_data, sizeof(map->new_tablespace));
-		strlcpy(map->old_tablespace_suffix, "/base", sizeof(map->old_tablespace_suffix));
-		strlcpy(map->new_tablespace_suffix, "/base", sizeof(map->new_tablespace_suffix));
+		map->old_tablespace = old_data;
+		map->new_tablespace = new_data;
+		map->old_tablespace_suffix = "/base";
+		map->new_tablespace_suffix = "/base";
 	}
 	else
 	{
 		/* relation belongs to a tablespace, so use the tablespace location */
-		strlcpy(map->old_tablespace, old_rel->tablespace, sizeof(map->old_tablespace));
-		strlcpy(map->new_tablespace, new_rel->tablespace, sizeof(map->new_tablespace));
-		strlcpy(map->old_tablespace_suffix, old_cluster.tablespace_suffix,
-				sizeof(map->old_tablespace_suffix));
-		strlcpy(map->new_tablespace_suffix, new_cluster.tablespace_suffix,
-				sizeof(map->new_tablespace_suffix));
+		map->old_tablespace = old_rel->tablespace;
+		map->new_tablespace = new_rel->tablespace;
+		map->old_tablespace_suffix = old_cluster.tablespace_suffix;
+		map->new_tablespace_suffix = new_cluster.tablespace_suffix;
 	}
 
 	map->old_db_oid = old_db->db_oid;
@@ -231,7 +229,7 @@ get_db_infos(ClusterInfo *cluster)
 	{
 		dbinfos[tupnum].db_oid = atooid(PQgetvalue(res, tupnum, i_oid));
 		dbinfos[tupnum].db_name = pg_strdup(PQgetvalue(res, tupnum, i_datname));
-		snprintf(dbinfos[tupnum].db_tblspace, sizeof(dbinfos[tupnum].db_tblspace), "%s",
+		snprintf(dbinfos[tupnum].db_tablespace, sizeof(dbinfos[tupnum].db_tablespace), "%s",
 				 PQgetvalue(res, tupnum, i_spclocation));
 	}
 	PQclear(res);
@@ -264,13 +262,15 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
 	int			num_rels = 0;
 	char	   *nspname = NULL;
 	char	   *relname = NULL;
+	char	   *tablespace = NULL;
 	int			i_spclocation,
 				i_nspname,
 				i_relname,
 				i_oid,
 				i_relfilenode,
 				i_reltablespace;
 	char		query[QUERY_ALLOC];
+	char	   *last_namespace = NULL, *last_tablespace = NULL;
 
 	/*
 	 * pg_largeobject contains user data that does not appear in pg_dumpall
@@ -366,26 +366,53 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
 	for (relnum = 0; relnum < ntups; relnum++)
 	{
 		RelInfo    *curr = &relinfos[num_rels++];
-		const char *tblspace;
 
 		curr->reloid = atooid(PQgetvalue(res, relnum, i_oid));
 
 		nspname = PQgetvalue(res, relnum, i_nspname);
-		curr->nspname = pg_strdup(nspname);
+		curr->nsp_alloc = false;
+
+		/*
+		 * Many of the namespace and tablespace strings are identical,
+		 * so we try to reuse the allocated string pointers where possible
+		 * to reduce memory consumption.
+		 */
+		/* Can we reuse the previous string allocation? */
+		if (last_namespace && strcmp(nspname, last_namespace) == 0)
+			curr->nspname = last_namespace;
+		else
+		{
+			last_namespace = curr->nspname = pg_strdup(nspname);
+			curr->nsp_alloc = true;
+		}
 
 		relname = PQgetvalue(res, relnum, i_relname);
 		curr->relname = pg_strdup(relname);
 
 		curr->relfilenode = atooid(PQgetvalue(res, relnum, i_relfilenode));
+		curr->tblsp_alloc = false;
 
+		/* Is the tablespace oid non-zero? */
 		if (atooid(PQgetvalue(res, relnum, i_reltablespace)) != 0)
-			/* Might be "", meaning the cluster default location. */
-			tblspace = PQgetvalue(res, relnum, i_spclocation);
+		{
+			/*
+			 * The tablespace location might be "", meaning the cluster
+			 * default location, i.e. pg_default or pg_global.
+			 */
+			tablespace = PQgetvalue(res, relnum, i_spclocation);
+
+			/* Can we reuse the previous string allocation? */
+			if (last_tablespace && strcmp(tablespace, last_tablespace) == 0)
+				curr->tablespace = last_tablespace;
+			else
+			{
+				last_tablespace = curr->tablespace = pg_strdup(tablespace);
+				curr->tblsp_alloc = true;
+			}
+		}
 		else
-			/* A zero reltablespace indicates the database tablespace. */
-			tblspace = dbinfo->db_tblspace;
-
-		strlcpy(curr->tablespace, tblspace, sizeof(curr->tablespace));
+			/* A zero reltablespace oid indicates the database tablespace. */
+			curr->tablespace = dbinfo->db_tablespace;
 	}
 	PQclear(res);
 
@@ -419,8 +446,11 @@ free_rel_infos(RelInfoArr *rel_arr)
 
 	for (relnum = 0; relnum < rel_arr->nrels; relnum++)
 	{
-		pg_free(rel_arr->rels[relnum].nspname);
+		if (rel_arr->rels[relnum].nsp_alloc)
+			pg_free(rel_arr->rels[relnum].nspname);
 		pg_free(rel_arr->rels[relnum].relname);
+		if (rel_arr->rels[relnum].tblsp_alloc)
+			pg_free(rel_arr->rels[relnum].tablespace);
 	}
 	pg_free(rel_arr->rels);
 	rel_arr->nrels = 0;

contrib/pg_upgrade/pg_upgrade.h

@@ -127,7 +127,9 @@ typedef struct
 	Oid			reloid;			/* relation oid */
 	Oid			relfilenode;	/* relation relfile node */
 	/* relation tablespace path, or "" for the cluster default */
-	char		tablespace[MAXPGPATH];
+	char	   *tablespace;
+	bool		nsp_alloc;
+	bool		tblsp_alloc;
 } RelInfo;
 
 typedef struct
@@ -141,10 +143,10 @@ typedef struct
  */
 typedef struct
 {
-	char		old_tablespace[MAXPGPATH];
-	char		new_tablespace[MAXPGPATH];
-	char		old_tablespace_suffix[MAXPGPATH];
-	char		new_tablespace_suffix[MAXPGPATH];
+	const char		*old_tablespace;
+	const char		*new_tablespace;
+	const char		*old_tablespace_suffix;
+	const char		*new_tablespace_suffix;
 	Oid			old_db_oid;
 	Oid			new_db_oid;
 
@@ -166,7 +168,7 @@ typedef struct
 {
 	Oid			db_oid;			/* oid of the database */
 	char	   *db_name;		/* database name */
-	char		db_tblspace[MAXPGPATH]; /* database default tablespace path */
+	char		db_tablespace[MAXPGPATH]; /* database default tablespace path */
 	RelInfoArr	rel_arr;		/* array of all user relinfos */
 } DbInfo;
 
@@ -256,7 +258,7 @@ typedef struct
 	Oid			pg_database_oid;	/* OID of pg_database relation */
 	Oid			install_role_oid;		/* OID of connected role */
 	Oid			role_count;		/* number of roles defined in the cluster */
-	char	   *tablespace_suffix;		/* directory specification */
+	const char *tablespace_suffix;		/* directory specification */
 } ClusterInfo;