retention-size

.travis.yml

@@ -41,7 +41,13 @@ env:
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica
-#    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention
+    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE MODE=retention
+    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE MODE=retention
+    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention
+    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE MODE=retention
+    - PG_VERSION=10 PG_BRANCH=REL_10_STABLE MODE=retention
+    - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE MODE=retention
+    - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE MODE=retention
 #    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore
     - PG_VERSION=15 PG_BRANCH=master
 

src/configure.c

@@ -31,6 +31,7 @@ static void show_configure_json(ConfigOption *opt);
 
 #define RETENTION_REDUNDANCY_DEFAULT	0
 #define RETENTION_WINDOW_DEFAULT		0
+#define RETENTION_SIZE_DEFAULT			0
 
 #define OPTION_INSTANCE_GROUP	"Backup instance information"
 #define OPTION_CONN_GROUP		"Connection parameters"
@@ -189,6 +190,11 @@ ConfigOption instance_options[] =
 		&instance_config.retention_window, SOURCE_CMD, 0,
 		OPTION_RETENTION_GROUP, 0, option_get_value
 	},
+	{
+		'U', 231, "retention-size",
+		&instance_config.retention_size, SOURCE_CMD, 0,
+		OPTION_RETENTION_GROUP, OPTION_UNIT_KB, option_get_value
+	},
 	{
 		'u', 221, "wal-depth",
 		&instance_config.wal_depth, SOURCE_CMD, 0,
@@ -375,6 +381,7 @@ init_config(InstanceConfig *config, const char *instance_name)
 
 	config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT;
 	config->retention_window = RETENTION_WINDOW_DEFAULT;
+	config->retention_size = RETENTION_SIZE_DEFAULT;
 	config->wal_depth = 0;
 
 	config->compress_alg = COMPRESS_ALG_DEFAULT;
@@ -549,6 +556,11 @@ readInstanceConfigFile(const char *instance_name)
 			&instance->retention_window, SOURCE_CMD, 0,
 			OPTION_RETENTION_GROUP, 0, option_get_value
 		},
+		{
+			'u', 231, "retention-size",
+			&instance->retention_size, SOURCE_CMD, 0,
+			OPTION_RETENTION_GROUP, 0, option_get_value
+		},
 		{
 			'u', 221, "wal-depth",
 			&instance->wal_depth, SOURCE_CMD, 0,

src/delete.c

@@ -150,9 +150,12 @@ void do_retention(bool no_validate, bool no_sync)
 			elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy);
 		if (instance_config.retention_window > 0)
 			elog(LOG, "WINDOW=%u", instance_config.retention_window);
+		if (instance_config.retention_size > 0)
+					elog(LOG, "SIZE=%lu", instance_config.retention_size);
 
 		if (instance_config.retention_redundancy == 0 &&
-			instance_config.retention_window == 0)
+			instance_config.retention_window == 0 &&
+			instance_config.retention_size == 0)
 		{
 			/* Retention is disabled but we still can cleanup wal */
 			elog(WARNING, "Retention policy is not set");
@@ -211,46 +214,67 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	int			i;
 
 	parray *redundancy_full_backup_list = NULL;
+	parray *size_backup_list = NULL;
 
 	/* For retention calculation */
+	uint64		total_backups_size = 0;
 	uint32		n_full_backups = 0;
 	int			cur_full_backup_num = 0;
 	time_t		days_threshold = 0;
 
 	/* For fancy reporting */
 	uint32		actual_window = 0;
+	uint64		actual_space = 0;
 
-	/* Calculate n_full_backups and days_threshold */
-	if (instance_config.retention_redundancy > 0)
+	/* Calculate n_full_backups, total_backups_size and days_threshold */
+	if (instance_config.retention_redundancy > 0 ||
+			instance_config.retention_size > 0)
 	{
 		for (i = 0; i < parray_num(backup_list); i++)
 		{
 			pgBackup   *backup = (pgBackup *) parray_get(backup_list, i);
 
+			total_backups_size += backup->data_bytes;
+
 			if (backup->backup_mode == BACKUP_MODE_FULL)
+					n_full_backups++;
+
+			/* Consider only valid FULL backups for Redundancy */
+			if (((instance_config.retention_redundancy > 0 ) &&
+				backup->backup_mode == BACKUP_MODE_FULL &&
+				(backup->status == BACKUP_STATUS_OK ||
+					backup->status == BACKUP_STATUS_DONE)))
 			{
 				/* Add every FULL backup that satisfy Redundancy policy to separate list */
-				if (n_full_backups < instance_config.retention_redundancy)
+				if (((n_full_backups <= instance_config.retention_redundancy) &&
+						(instance_config.retention_redundancy > 0)))
 				{
 					if (!redundancy_full_backup_list)
 						redundancy_full_backup_list = parray_new();
 
 					parray_append(redundancy_full_backup_list, backup);
 				}
+			}
 
-				/* Consider only valid FULL backups for Redundancy fulfillment */
-				if (backup->status == BACKUP_STATUS_OK ||
-					backup->status == BACKUP_STATUS_DONE)
-				{
-					n_full_backups++;
-				}
+			if ((instance_config.retention_size > 0) &&
+				(total_backups_size <= instance_config.retention_size * 1024 * 1024))
+			{
+				if (!size_backup_list)
+					size_backup_list = parray_new();
+
+				parray_append(size_backup_list, backup);
 			}
 		}
+
 		/* Sort list of full backups to keep */
 		if (redundancy_full_backup_list)
 			parray_qsort(redundancy_full_backup_list, pgBackupCompareIdDesc);
+		/* Sort list */
+		if (size_backup_list)
+			parray_qsort(size_backup_list, pgBackupCompareIdDesc);
 	}
 
+
 	if (instance_config.retention_window > 0)
 	{
 		days_threshold = current_time -
@@ -262,6 +286,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	{
 
 		bool redundancy_keep = false;
+		bool size_keep = false;
 		time_t backup_time = 0;
 		pgBackup   *backup = (pgBackup *) parray_get(backup_list, (size_t) i);
 
@@ -276,6 +301,12 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 				redundancy_keep = true;
 		}
 
+		if (size_backup_list)
+		{
+			if (parray_bsearch(size_backup_list, backup, pgBackupCompareIdDesc))
+				size_keep = true;
+		}
+
 		/* Remember the serial number of latest valid FULL backup */
 		if (backup->backup_mode == BACKUP_MODE_FULL &&
 			(backup->status == BACKUP_STATUS_OK ||
@@ -292,10 +323,19 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 		else
 			backup_time = backup->start_time;
 
+		/* Fancy message about actual state of backups */
+		if (delete_expired)
+		{
+			actual_space += backup->data_bytes;
+			actual_window = ((float)current_time - (float)backup->recovery_time)/(60 * 60 * 24);
+		}
+
 		/* Check if backup in needed by retention policy */
-		if ((days_threshold == 0 || (days_threshold > backup_time)) &&
-			(instance_config.retention_redundancy == 0 || !redundancy_keep))
+		if (((days_threshold == 0) || (days_threshold > backup_time)) &&
+			(((instance_config.retention_redundancy == 0) || !redundancy_keep) &&
+			((instance_config.retention_size == 0) || !size_keep)))
 		{
+			elog(INFO, "Check if backup %s in needed by retention", base36enc(backup->start_time));
 			/* This backup is not guarded by retention
 			 *
 			 * Redundancy = 1
@@ -327,8 +367,27 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 			/* Add backup to purge_list */
 			elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time));
 			parray_append(to_purge_list, backup);
+
+			/* Calculate residuary space */
+			total_backups_size -= backup->data_bytes;
 			continue;
 		}
+		else
+		{
+			 /*If the incremental backup that allowed by retention has merge flag
+			 * we need to merge it to it's parent FULL backup */
+			if (((backup->backup_mode == BACKUP_MODE_DIFF_PAGE)||
+				(backup->backup_mode == BACKUP_MODE_DIFF_DELTA))
+					&& merge_expired)
+			{
+				pgBackup *full_backup = find_parent_full_backup(backup);
+				if (!parray_bsearch(to_purge_list, full_backup, pgBackupCompareIdDesc))
+					parray_append(to_purge_list, full_backup);
+				if (!parray_bsearch(to_keep_list, backup, pgBackupCompareIdDesc))
+					parray_append(to_keep_list, backup);
+				continue;
+			}
+		}
 	}
 
 	/* sort keep_list and purge list */
@@ -384,13 +443,16 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	 */
 
 	cur_full_backup_num = 1;
+	actual_space = 0;
 	for (i = 0; i < parray_num(backup_list); i++)
 	{
 		char		*action = "Active";
 		uint32		pinning_window = 0;
 
 		pgBackup	*backup = (pgBackup *) parray_get(backup_list, i);
 
+		actual_space += backup->data_bytes;
+
 		if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
 			action = "Expired";
 
@@ -404,14 +466,16 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 			pinning_window = (backup->expire_time - backup->recovery_time)/(3600 * 24);
 
 		/* TODO: add ancestor(chain full backup) ID */
-		elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s",
+		elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud, Size: %.2fGB/%.2fGB. %s",
 				base36enc(backup->start_time),
 				pgBackupGetBackupMode(backup),
 				status2str(backup->status),
 				cur_full_backup_num,
 				instance_config.retention_redundancy,
 				actual_window,
 				pinning_window ? pinning_window : instance_config.retention_window,
+				(float)total_backups_size/(1024L * 1024L * 1024L),
+				(float)instance_config.retention_size, // /(1024L * 1024L),
 				action);
 
 		/* Only valid full backups are count to something */
@@ -576,7 +640,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list)
 	int j;
 
 	/* Remove backups by retention policy. Retention policy is configured by
-	 * retention_redundancy and retention_window
+	 * retention_redundancy, retention_window and retention_size
 	 * Remove only backups, that do not have children guarded by retention
 	 *
 	 * TODO: We do not consider the situation if child is marked for purge

src/help.c

@@ -93,6 +93,7 @@ help_pg_probackup(void)
 	printf(_("                 [--log-rotation-age=log-rotation-age]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [--compress-algorithm=compress-algorithm]\n"));
 	printf(_("                 [--compress-level=compress-level]\n"));
@@ -131,6 +132,7 @@ help_pg_probackup(void)
 	printf(_("                 [--delete-expired] [--delete-wal] [--merge-expired]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [--compress]\n"));
 	printf(_("                 [--compress-algorithm=compress-algorithm]\n"));
@@ -194,6 +196,7 @@ help_pg_probackup(void)
 	printf(_("                 [-j num-threads] [--progress]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n"));
 	printf(_("                 [--delete-wal]\n"));
@@ -278,6 +281,7 @@ help_backup(void)
 	printf(_("                 [--delete-expired] [--delete-wal] [--merge-expired]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [--compress]\n"));
 	printf(_("                 [--compress-algorithm=compress-algorithm]\n"));
@@ -341,6 +345,9 @@ help_backup(void)
 	printf(_("                                   number of full backups to keep; 0 disables; (default: 0)\n"));
 	printf(_("      --retention-window=retention-window\n"));
 	printf(_("                                   number of days of recoverability; 0 disables; (default: 0)\n"));
+	printf(_("      --retention-size=retention-size\n"));
+	printf(_("                                   max allowed data size of all backups; 0 disables; (default: 0)\n"));
+	printf(_("                                   available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n"));
 	printf(_("      --wal-depth=wal-depth        number of latest valid backups per timeline that must\n"));
 	printf(_("                                   retain the ability to perform PITR; 0 disables; (default: 0)\n"));
 	printf(_("      --dry-run                    perform a trial run without any changes\n"));
@@ -632,6 +639,7 @@ help_delete(void)
 	printf(_("                 [-j num-threads] [--progress]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [--no-validate] [--no-sync]\n\n"));
 
@@ -653,6 +661,8 @@ help_delete(void)
 	printf(_("                                   number of full backups to keep; 0 disables; (default: 0)\n"));
 	printf(_("      --retention-window=retention-window\n"));
 	printf(_("                                   number of days of recoverability; 0 disables; (default: 0)\n"));
+	printf(_("      --retention-size=retention-size\n"));
+	printf(_("                                   max allowed data size of all backups to keep; 0 disables; (default: 0)\n"));
 	printf(_("      --wal-depth=wal-depth        number of latest valid backups per timeline that must\n"));
 	printf(_("                                   retain the ability to perform PITR; 0 disables; (default: 0)\n"));
 	printf(_("      --dry-run                    perform a trial run without any changes\n"));
@@ -757,6 +767,7 @@ help_set_config(void)
 	printf(_("                 [--log-rotation-age=log-rotation-age]\n"));
 	printf(_("                 [--retention-redundancy=retention-redundancy]\n"));
 	printf(_("                 [--retention-window=retention-window]\n"));
+	printf(_("                 [--retention-size=retention-size]\n"));
 	printf(_("                 [--wal-depth=wal-depth]\n"));
 	printf(_("                 [--compress-algorithm=compress-algorithm]\n"));
 	printf(_("                 [--compress-level=compress-level]\n"));
@@ -800,6 +811,9 @@ help_set_config(void)
 	printf(_("                                   number of full backups to keep; 0 disables; (default: 0)\n"));
 	printf(_("      --retention-window=retention-window\n"));
 	printf(_("                                   number of days of recoverability; 0 disables; (default: 0)\n"));
+	printf(_("      --retention-size=retention-size\n"));
+	printf(_("                                   max allowed data size of all backups; 0 disables; (default: 0)\n"));
+	printf(_("                                   available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n"));
 	printf(_("      --wal-depth=wal-depth        number of latest valid backups with ability to perform\n"));
 	printf(_("                                   the point in time recovery;  disables; (default: 0)\n"));
 

src/pg_probackup.h

@@ -375,6 +375,7 @@ typedef struct InstanceConfig
 	/* Retention options. 0 disables the option. */
 	uint32		retention_redundancy;
 	uint32		retention_window;
+	uint64		retention_size;
 	uint32		wal_depth;
 
 	CompressAlg	compress_alg;