diff --git a/src/archive.c b/src/archive.c index bcc0bccbc..1c1b5e419 100644 --- a/src/archive.c +++ b/src/archive.c @@ -388,9 +388,9 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, char to_fullpath[MAXPGPATH]; char to_fullpath_part[MAXPGPATH]; /* partial handling */ - struct stat st; + pio_stat_t st; int partial_try_count = 0; - ssize_t partial_file_size = 0; + int64_t partial_file_size = 0; bool partial_is_stale = true; size_t len; err_i err = $noerr(); @@ -466,11 +466,11 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", archive_timeout, to_fullpath_part); - partial_file_size = st.st_size; + partial_file_size = st.pst_size; } /* file size is changing */ - if (st.st_size != partial_file_size) + if (st.pst_size != partial_file_size) partial_is_stale = false; sleep(1); diff --git a/src/backup.c b/src/backup.c index c285218fb..6b1ee6bd3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -392,7 +392,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pgFile *file = (pgFile *) parray_get(backup_files_list, i); /* if the entry was a directory, create it in the backup */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { char dirpath[MAXPGPATH]; @@ -569,7 +569,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, pgFile *file = (pgFile *) parray_get(backup_files_list, i); /* TODO: sync directory ? */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (file->write_size <= 0) @@ -1837,7 +1837,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c file = pgFileNew(full_filename, filename, true, 0, FIO_BACKUP_HOST); - if (S_ISREG(file->mode)) + if (file->kind == PIO_KIND_REGULAR) { file->crc = pgFileGetCRC32C(full_filename, false); @@ -1991,7 +1991,7 @@ backup_files(void *arg) pgFile *prev_file = NULL; /* We have already copied all directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (arguments->thread_num == 1) @@ -2046,9 +2046,9 @@ backup_files(void *arg) } /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + if (file->kind != PIO_KIND_REGULAR) + elog(WARNING, "Unexpected type %s of file \"%s\", skipping", + pio_file_kind2str(file->kind, from_fullpath), from_fullpath); /* Check that file exist in previous backup */ if (current.backup_mode != BACKUP_MODE_FULL) @@ -2121,7 +2121,7 @@ parse_filelist_filenames(parray *files, const char *root) pgFile *file = (pgFile *) parray_get(files, i); int sscanf_result; - if (S_ISREG(file->mode) && + if (file->kind == PIO_KIND_REGULAR && path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) { /* @@ -2148,7 +2148,7 @@ parse_filelist_filenames(parray *files, const char *root) } } - if (S_ISREG(file->mode) && file->tblspcOid != 0 && + if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 && file->name && file->name[0]) { if (file->forkName == init) @@ -2218,7 +2218,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) { - if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + if (prev_file->kind == PIO_KIND_REGULAR && prev_file->is_datafile) { elog(LOG, "Setting 'is_cfs' on file %s, name %s", prev_file->rel_path, prev_file->name); @@ -2375,7 +2375,7 @@ calculate_datasize_of_filelist(parray *filelist) if (file->external_dir_num != 0 || file->excluded) continue; - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { // TODO is a dir always 4K? bytes += 4096; diff --git a/src/catalog.c b/src/catalog.c index a41e861cf..21a25f3d9 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -847,12 +847,22 @@ pgBackupGetBackupMode(pgBackup *backup, bool show_color) static bool IsDir(const char *dirpath, const char *entry, fio_location location) { + FOBJ_FUNC_ARP(); char path[MAXPGPATH]; - struct stat st; + pio_stat_t st; + err_i err; join_path_components(path, dirpath, entry); - return fio_stat(location, path, &st, false) == 0 && S_ISDIR(st.st_mode); + st = $i(pioStat, pioDriveForLocation(location), + .path = path, .follow_symlink = false, .err = &err); + if ($haserr(err)) + { + ft_logerr(FT_WARNING, $errmsg(err), "IsDir"); + return false; + } + + return st.pst_kind == PIO_KIND_DIRECTORY; } /* @@ -1074,6 +1084,7 @@ get_backup_filelist(pgBackup *backup, bool strict) char path[MAXPGPATH]; char linked[MAXPGPATH]; char compress_alg_string[MAXPGPATH]; + char kind[16]; int64 write_size, uncompressed_size, mode, /* bit length of mode_t depends on platforms */ @@ -1115,6 +1126,11 @@ get_backup_filelist(pgBackup *backup, bool strict) /* * Optional fields */ + if (get_control_value_str(buf, "kind", kind, sizeof(kind), false)) + file->kind = pio_str2file_kind(kind, path); + else /* fallback to mode for old backups */ + file->kind = pio_statmode2file_kind(file->mode, path); + if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0]) { file->linked = pgut_strdup(linked); @@ -1146,7 +1162,7 @@ get_backup_filelist(pgBackup *backup, bool strict) if (!file->is_datafile || file->is_cfs) file->size = file->uncompressed_size; - if (file->external_dir_num == 0 && S_ISREG(file->mode)) + if (file->external_dir_num == 0 && file->kind == PIO_KIND_REGULAR) { bool is_datafile = file->is_datafile; set_forkname(file); @@ -2564,14 +2580,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, if (file->write_size == FILE_NOT_FOUND) continue; - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) { backup_size_on_disk += 4096; uncompressed_size_on_disk += 4096; } /* Count the amount of the data actually copied */ - if (S_ISREG(file->mode) && file->write_size > 0) + if (file->kind == PIO_KIND_REGULAR && file->write_size > 0) { /* * Size of WAL files in 'pg_wal' is counted separately @@ -2587,11 +2603,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, } len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " - "\"mode\":\"%u\", \"is_datafile\":\"%u\", " + "\"kind\":\"%s\", \"mode\":\"%u\", \"is_datafile\":\"%u\", " "\"is_cfs\":\"%u\", \"crc\":\"%u\", " "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " "\"dbOid\":\"%u\"", - file->rel_path, file->write_size, file->mode, + file->rel_path, file->write_size, + pio_file_kind2str(file->kind, file->rel_path), + file->mode, file->is_datafile ? 1 : 0, file->is_cfs ? 1 : 0, file->crc, diff --git a/src/catchup.c b/src/catchup.c index 7cdcc82cd..038767470 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -378,7 +378,7 @@ catchup_thread_runner(void *arg) pgFile *dest_file = NULL; /* We have already copied all directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (file->excluded) @@ -400,9 +400,9 @@ catchup_thread_runner(void *arg) join_path_components(to_fullpath, arguments->to_root, file->rel_path); /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + if (file->kind != PIO_KIND_REGULAR) + elog(WARNING, "Unexpected kind %s of file \"%s\", skipping", + pio_file_kind2str(file->kind, from_fullpath), from_fullpath); /* Check that file exist in dest pgdata */ if (arguments->backup_mode != BACKUP_MODE_FULL) @@ -439,7 +439,7 @@ catchup_thread_runner(void *arg) if (file->write_size == BYTES_INVALID) { - elog(LOG, "Skipping the unchanged file: \"%s\", read %zu bytes", from_fullpath, file->read_size); + elog(LOG, "Skipping the unchanged file: \"%s\", read %lld bytes", from_fullpath, (long long)file->read_size); continue; } @@ -546,7 +546,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p * - but PG itself is not relying on fs, its durable_sync * includes directory sync */ - if (S_ISDIR(file->mode) || file->excluded) + if (file->kind == PIO_KIND_DIRECTORY || file->excluded) continue; Assert(file->external_dir_num == 0); @@ -630,8 +630,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* for fancy reporting */ time_t start_time, end_time; - ssize_t transfered_datafiles_bytes = 0; - ssize_t transfered_walfiles_bytes = 0; + int64_t transfered_datafiles_bytes = 0; + int64_t transfered_walfiles_bytes = 0; char pretty_source_bytes[20]; err_i err = $noerr(); @@ -807,7 +807,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, pgFile *file = (pgFile *) parray_get(source_filelist, i); char parent_dir[MAXPGPATH]; - if (!S_ISDIR(file->mode) || file->excluded) + if (file->kind != PIO_KIND_DIRECTORY || file->excluded) continue; /* diff --git a/src/checkdb.c b/src/checkdb.c index 2943665a2..bc3c60fc5 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -147,7 +147,7 @@ check_files(void *arg) elog(ERROR, "interrupted during checkdb"); /* No need to check directories */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) continue; if (!pg_atomic_test_set_flag(&file->lock)) @@ -161,7 +161,7 @@ check_files(void *arg) elog(INFO, "Progress: (%d/%d). Process file \"%s\"", i + 1, n_files_list, from_fullpath); - if (S_ISREG(file->mode)) + if (file->kind == PIO_KIND_REGULAR) { /* check only uncompressed by cfs datafiles */ if (file->is_datafile && !file->is_cfs) diff --git a/src/data.c b/src/data.c index d9eaf0807..bc1f18097 100644 --- a/src/data.c +++ b/src/data.c @@ -507,7 +507,8 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat * NOTE This is a normal situation, if the file size has changed * since the moment we computed it. */ - file->n_blocks = file->size/BLCKSZ; + file->n_blocks = (typeof(file->n_blocks))(file->size/BLCKSZ); + Assert((int64_t)file->n_blocks * BLCKSZ == file->size); /* * Skip unchanged file only if it exists in previous backup. @@ -611,12 +612,15 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } - file->read_size = rc * BLCKSZ; + file->read_size = (int64_t)rc * BLCKSZ; /* refresh n_blocks for FULL and DELTA */ if (backup_mode == BACKUP_MODE_FULL || backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; + { + file->n_blocks = (typeof(file->n_blocks))(file->read_size / BLCKSZ); + Assert((int64_t)file->n_blocks * BLCKSZ == file->read_size); + } /* Determine that file didn`t changed in case of incremental backup */ if (backup_mode != BACKUP_MODE_FULL && @@ -650,7 +654,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size) + uint32 checksum_version, int64_t prev_size) { int rc; bool use_pagemap; @@ -760,7 +764,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } - file->read_size = rc * BLCKSZ; + file->read_size = (int64_t)rc * BLCKSZ; /* Determine that file didn`t changed in case of incremental catchup */ if (backup_mode != BACKUP_MODE_FULL && @@ -1595,7 +1599,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, * NOTE This is a normal situation, if the file size has changed * since the moment we computed it. */ - nblocks = file->size/BLCKSZ; + nblocks = (typeof(nblocks))(file->size/BLCKSZ); + Assert((int64_t)nblocks * BLCKSZ == file->size); for (blknum = 0; blknum < nblocks; blknum++) { @@ -2275,7 +2280,7 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", to_fullpath, strerror(errno)); { - long pos = ftell(out); + int64_t pos = ftell(out); if (pos < 0) elog(ERROR, "Cannot get position in destination file \"%s\": %s", diff --git a/src/delete.c b/src/delete.c index b6ed23966..9873cb163 100644 --- a/src/delete.c +++ b/src/delete.c @@ -794,8 +794,8 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli char first_to_del_str[MAXFNAMELEN]; char oldest_to_keep_str[MAXFNAMELEN]; int i; - size_t wal_size_logical = 0; - size_t wal_size_actual = 0; + int64_t wal_size_logical = 0; + int64_t wal_size_actual = 0; char wal_pretty_size[20]; bool purge_all = false; @@ -837,7 +837,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli /* sanity */ if (OldestToKeepSegNo > FirstToDeleteSegNo) { - wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; + wal_size_logical = (int64_t)(OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; /* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */ if (purge_all) diff --git a/src/dir.c b/src/dir.c index f3dad2917..f245bcd37 100644 --- a/src/dir.c +++ b/src/dir.c @@ -145,23 +145,26 @@ pgFile * pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location) { - struct stat st; + FOBJ_FUNC_ARP(); + pio_stat_t st; pgFile *file; + err_i err; /* stat the file */ - if (fio_stat(location, path, &st, follow_symlink) < 0) - { + st = $i(pioStat, pioDriveForLocation(location), .path = path, + .follow_symlink = follow_symlink, .err = &err); + if ($haserr(err)) { /* file not found is not an error case */ - if (errno == ENOENT) + if (getErrno(err) == ENOENT) return NULL; - elog(ERROR, "cannot stat file \"%s\": %s", path, - strerror(errno)); + ft_logerr(FT_FATAL, $errmsg(err), "pgFileNew"); } file = pgFileInit(rel_path); - file->size = st.st_size; - file->mode = st.st_mode; - file->mtime = st.st_mtime; + file->size = st.pst_size; + file->kind = st.pst_kind; + file->mode = st.pst_mode; + file->mtime = st.pst_mtime; file->external_dir_num = external_dir_num; return file; @@ -393,7 +396,7 @@ dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool fol return; } - if (!S_ISDIR(file->mode)) + if (file->kind != PIO_KIND_DIRECTORY) { if (external_dir_num > 0) elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", @@ -435,7 +438,7 @@ dir_check_file(pgFile *file, bool backup_logs) in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path); /* Check if we need to exclude file by name */ - if (S_ISREG(file->mode)) + if (file->kind == PIO_KIND_REGULAR) { for (i = 0; pgdata_exclude_files[i]; i++) if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) @@ -449,7 +452,7 @@ dir_check_file(pgFile *file, bool backup_logs) * If the directory name is in the exclude list, do not list the * contents. */ - else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0) + else if (file->kind == PIO_KIND_DIRECTORY && !in_tablespace && file->external_dir_num == 0) { /* * If the item in the exclude list starts with '/', compare to @@ -478,14 +481,14 @@ dir_check_file(pgFile *file, bool backup_logs) } /* Do not backup ptrack_init files */ - if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) + if (file->kind == PIO_KIND_REGULAR && strcmp(file->name, "ptrack_init") == 0) return CHECK_FALSE; /* * Check files located inside database directories including directory * 'global' */ - if (S_ISREG(file->mode) && file->tblspcOid != 0 && + if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 && file->name && file->name[0]) { if (strcmp(file->name, "pg_internal.init") == 0) @@ -607,7 +610,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, struct dirent *dent; bool in_tablespace = false; - if (!S_ISDIR(parent->mode)) + if (parent->kind != PIO_KIND_DIRECTORY) elog(ERROR, "\"%s\" is not a directory", parent_dir); in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path); @@ -641,7 +644,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, continue; /* Skip entries point current dir or parent dir */ - if (S_ISDIR(file->mode) && + if (file->kind == PIO_KIND_DIRECTORY && (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) { pgFileFree(file); @@ -659,7 +662,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, * Add only files, directories and links. Skip sockets and other * unexpected file formats. */ - if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) + if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR) { elog(WARNING, "Skip '%s': unexpected file format", child); pgFileFree(file); @@ -671,7 +674,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, * Do not copy tablespaces twice. It may happen if the tablespace is located * inside the PGDATA. */ - if (S_ISDIR(file->mode) && + if (file->kind == PIO_KIND_DIRECTORY && strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) { Oid tblspcOid; @@ -718,7 +721,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, * If the entry is a directory call dir_list_file_internal() * recursively. */ - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink, backup_logs, skip_hidden, external_dir_num, location); } @@ -886,7 +889,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba { pgFile *file = (pgFile *) parray_get(dest_files, i); - if (!S_ISDIR(file->mode)) + if (file->kind != PIO_KIND_DIRECTORY) continue; /* skip external directory content */ @@ -919,7 +922,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba char parent_dir[MAXPGPATH]; pgFile *dir = (pgFile *) parray_get(dest_files, i); - if (!S_ISDIR(dir->mode)) + if (dir->kind != PIO_KIND_DIRECTORY) continue; /* skip external directory content */ @@ -1523,14 +1526,13 @@ dir_is_empty(const char *path, fio_location location) bool fileExists(const char *path, fio_location location) { - struct stat buf; + FOBJ_FUNC_ARP(); + err_i err; + bool exists; - if (fio_stat(location, path, &buf, true) == -1 && errno == ENOENT) - return false; - else if (!S_ISREG(buf.st_mode)) - return false; - else - return true; + exists = $i(pioExists, pioDriveForLocation(location), path, &err); + + return exists; } /* diff --git a/src/fetch.c b/src/fetch.c index 980bf531b..d283af129 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -28,9 +28,10 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t * { int fd; char *buffer; - struct stat statbuf; + pio_stat_t statbuf; char fullpath[MAXPGPATH]; - int len; + size_t len; + err_i err; join_path_components(fullpath, datadir, path); @@ -43,16 +44,21 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t * fullpath, strerror(errno)); } - if (fio_stat(location, fullpath, &statbuf, true) < 0) + statbuf = $i(pioStat, pioDriveForLocation(location), .path = fullpath, + .follow_symlink = true, .err = &err); + if ($haserr(err)) { if (safe) return NULL; else - elog(ERROR, "Could not stat file \"%s\": %s", - fullpath, strerror(errno)); + ft_logerr(FT_FATAL, $errmsg(err), "slurpFile"); } - len = statbuf.st_size; + if (statbuf.pst_size > SSIZE_MAX) + ft_log(FT_FATAL, "file \"%s\" is too large: %lld", + fullpath, (long long)statbuf.pst_size); + + len = statbuf.pst_size; buffer = pg_malloc(len + 1); if (fio_read(fd, buffer, len) != len) diff --git a/src/merge.c b/src/merge.c index a03cd2209..5d787e567 100644 --- a/src/merge.c +++ b/src/merge.c @@ -639,7 +639,7 @@ merge_chain(InstanceState *instanceState, pgFile *file = (pgFile *) parray_get(dest_backup->files, i); /* if the entry was an external directory, create it in the backup */ - if (file->external_dir_num && S_ISDIR(file->mode)) + if (file->external_dir_num && file->kind == PIO_KIND_DIRECTORY) { char dirpath[MAXPGPATH]; char new_container[MAXPGPATH]; @@ -955,6 +955,7 @@ merge_files(void *arg) continue; tmp_file = pgFileInit(dest_file->rel_path); + tmp_file->kind = dest_file->kind; tmp_file->mode = dest_file->mode; tmp_file->is_datafile = dest_file->is_datafile; tmp_file->is_cfs = dest_file->is_cfs; @@ -962,7 +963,7 @@ merge_files(void *arg) tmp_file->dbOid = dest_file->dbOid; /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) + if (dest_file->kind == PIO_KIND_DIRECTORY) goto done; elog(progress ? INFO : LOG, "Progress: (%d/%zu). Merging file \"%s\"", diff --git a/src/pg_probackup.c b/src/pg_probackup.c index eaeabd2bc..32d28c004 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -291,6 +291,7 @@ main(int argc, char *argv[]) { char *command = NULL; ProbackupSubcmd backup_subcmd = NO_CMD; + err_i err; ft_init_log(elog_ft_log); fobj_init(); @@ -536,13 +537,17 @@ main(int argc, char *argv[]) if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != ARCHIVE_GET_CMD) { - struct stat st; + pio_stat_t st; - if (fio_stat(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path, - &st, true) != 0) + st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST), + .path = instanceState->instance_backup_subdir_path, + .follow_symlink = true, + .err = &err); + + if ($haserr(err)) { - elog(WARNING, "Failed to access directory \"%s\": %s", - instanceState->instance_backup_subdir_path, strerror(errno)); + ft_logerr(FT_WARNING, $errmsg(err), "Failed to access directory \"%s\"", + instanceState->instance_backup_subdir_path); // TODO: redundant message, should we get rid of it? elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -551,7 +556,7 @@ main(int argc, char *argv[]) else { /* Ensure that backup_path is a path to a directory */ - if (!S_ISDIR(st.st_mode)) + if (st.pst_kind != PIO_KIND_DIRECTORY) elog(ERROR, "-B, --backup-path must be a path to directory"); } } @@ -896,7 +901,8 @@ main(int argc, char *argv[]) */ char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); - if (fio_is_same_file(FIO_DB_HOST, stripped_wal_file_path, archive_push_xlog_dir, true)) + if ($i(pioFilesAreSame, pioDriveForLocation(FIO_DB_HOST), + .file1 = stripped_wal_file_path, .file2 = archive_push_xlog_dir)) { /* 2nd case */ system_id = get_system_identifier(FIO_DB_HOST, instance_config.pgdata, false); diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 41f888d18..f4571707d 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -211,18 +211,21 @@ typedef enum ForkName typedef struct pgFile { char *name; /* file or directory name */ - mode_t mode; /* protection (file type and permission) */ - size_t size; /* size of the file */ - time_t mtime; /* file st_mtime attribute, can be used only - during backup */ - size_t read_size; /* size of the portion read (if only some pages are + + pio_file_kind_e kind; /* kind of file */ + uint32_t mode; /* protection (permission) */ + int64_t size; /* size of the file */ + + int64_t read_size; /* size of the portion read (if only some pages are backed up, it's different from size) */ - int64 write_size; /* size of the backed-up file. BYTES_INVALID means + int64_t write_size; /* size of the backed-up file. BYTES_INVALID means that the file existed but was not backed up because not modified since last backup. */ - size_t uncompressed_size; /* size of the backed-up file before compression + int64_t uncompressed_size; /* size of the backed-up file before compression * and adding block headers. */ + time_t mtime; /* file st_mtime attribute, can be used only + during backup */ /* we need int64 here to store '-1' value */ pg_crc32 crc; /* CRC value of the file, regular file only */ char *rel_path; /* relative path of the file */ @@ -590,7 +593,7 @@ struct timelineInfo { XLogSegNo end_segno; /* last present segment in this timeline */ size_t n_xlog_files; /* number of segments (only really existing) * does not include lost segments */ - size_t size; /* space on disk taken by regular WAL files */ + int64_t size; /* space on disk taken by regular WAL files */ parray *backups; /* array of pgBackup sturctures with info * about backups belonging to this timeline */ parray *xlog_filelist; /* array of ordinary WAL segments, '.partial' @@ -1022,7 +1025,6 @@ extern bool backup_contains_external(const char *dir, parray *dirs_list); extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); -extern size_t pgFileSize(const char *path); extern pgFile *pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, @@ -1053,7 +1055,7 @@ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size); + uint32 checksum_version, int64_t prev_size); extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, diff --git a/src/restore.c b/src/restore.c index 6e8b4e10e..428a0e1cd 100644 --- a/src/restore.c +++ b/src/restore.c @@ -837,11 +837,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgFile *file = (pgFile *) parray_get(dest_files, i); - if (S_ISDIR(file->mode)) + if (file->kind == PIO_KIND_DIRECTORY) total_bytes += 4096; if (!params->skip_external_dirs && - file->external_dir_num && S_ISDIR(file->mode)) + file->external_dir_num && file->kind == PIO_KIND_DIRECTORY) { char *external_path; char dirpath[MAXPGPATH]; @@ -926,7 +926,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, redundant = true; /* do not delete the useful internal directories */ - if (S_ISDIR(file->mode) && !redundant) + if (file->kind == PIO_KIND_DIRECTORY && !redundant) continue; /* if file does not exists in destination list, then we can safely unlink it */ @@ -1056,7 +1056,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, char to_fullpath[MAXPGPATH]; pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - if (S_ISDIR(dest_file->mode)) + if (dest_file->kind == PIO_KIND_DIRECTORY) continue; /* skip external files if ordered to do so */ @@ -1137,7 +1137,7 @@ restore_files(void *arg) pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) + if (dest_file->kind == PIO_KIND_DIRECTORY) continue; if (!pg_atomic_test_set_flag(&dest_file->lock)) @@ -1543,13 +1543,14 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, char path[MAXPGPATH]; FILE *fp = NULL; FILE *fp_tmp = NULL; - struct stat st; + pio_stat_t st; char current_time_str[100]; /* postgresql.auto.conf parsing */ char line[16384] = "\0"; char *buf = NULL; int buf_len = 0; int buf_len_max = 16384; + err_i err; elog(LOG, "update recovery settings in postgresql.auto.conf"); @@ -1557,17 +1558,16 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); - if (fio_stat(FIO_DB_HOST, postgres_auto_path, &st, false) < 0) + st = $i(pioStat, pioDriveForLocation(FIO_DB_HOST), + .path = postgres_auto_path, .follow_symlink = false, .err = &err); + /* file not found is not an error case */ + if ($haserr(err) && getErrno(err) != ENOENT) { - /* file not found is not an error case */ - if (errno != ENOENT) - elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path, - strerror(errno)); - st.st_size = 0; + ft_logerr(FT_FATAL, $errmsg(err), ""); } /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ - if (st.st_size > 0) + if (st.pst_size > 0) { fp = fio_open_stream(FIO_DB_HOST, postgres_auto_path); if (fp == NULL) diff --git a/src/show.c b/src/show.c index 5440e28a2..c7d714062 100644 --- a/src/show.c +++ b/src/show.c @@ -1089,7 +1089,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, appendPQExpBuffer(buf, "%zu", tlinfo->n_xlog_files); json_add_key(buf, "size", json_level); - appendPQExpBuffer(buf, "%zu", tlinfo->size); + appendPQExpBuffer(buf, "%lld", (long long)tlinfo->size); json_add_key(buf, "zratio", json_level); diff --git a/src/util.c b/src/util.c index 5b341fc22..a54e34e08 100644 --- a/src/util.c +++ b/src/util.c @@ -403,9 +403,9 @@ copy_pgcontrol_file(fio_location from_location, const char *from_fullpath, digestControlFile(&ControlFile, buffer, size); file->crc = ControlFile.crc; - file->read_size = size; - file->write_size = size; - file->uncompressed_size = size; + file->read_size = (int64_t)size; + file->write_size = (int64_t)size; + file->uncompressed_size = (int64_t)size; writeControlFile(to_location, to_fullpath, &ControlFile); diff --git a/src/utils/file.c b/src/utils/file.c index df305c73a..2b37ad611 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -51,8 +51,9 @@ typedef struct { typedef struct { + pio_file_kind_e kind; mode_t mode; - size_t size; + int64_t size; time_t mtime; bool is_datafile; Oid tblspcOid; @@ -258,6 +259,110 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); } +pio_file_kind_e +pio_statmode2file_kind(mode_t mode, const char* path) +{ + pio_file_kind_e kind; + if (S_ISREG(mode)) + kind = PIO_KIND_REGULAR; + else if (S_ISDIR(mode)) + kind = PIO_KIND_DIRECTORY; +#ifdef S_ISLNK + else if (S_ISLNK(mode)) + kind = PIO_KIND_SYMLINK; +#endif +#ifdef S_ISFIFO + else if (S_ISFIFO(mode)) + kind = PIO_KIND_FIFO; +#endif +#ifdef S_ISSOCK + else if (S_ISFIFO(mode)) + kind = PIO_KIND_SOCK; +#endif +#ifdef S_ISCHR + else if (S_ISCHR(mode)) + kind = PIO_KIND_CHARDEV; +#endif +#ifdef S_ISBLK + else if (S_ISBLK(mode)) + kind = PIO_KIND_BLOCKDEV; +#endif + else + elog(ERROR, "Unsupported file mode kind \"%x\" for file '%s'", + mode, path); + return kind; +} + +pio_file_kind_e +pio_str2file_kind(const char* str, const char* path) +{ + pio_file_kind_e kind; + if (strncmp(str, "reg", 3) == 0) + kind = PIO_KIND_REGULAR; + else if (strncmp(str, "dir", 3) == 0) + kind = PIO_KIND_DIRECTORY; + else if (strncmp(str, "sym", 3) == 0) + kind = PIO_KIND_SYMLINK; + else if (strncmp(str, "fifo", 4) == 0) + kind = PIO_KIND_FIFO; + else if (strncmp(str, "sock", 4) == 0) + kind = PIO_KIND_SOCK; + else if (strncmp(str, "chdev", 5) == 0) + kind = PIO_KIND_CHARDEV; + else if (strncmp(str, "bldev", 5) == 0) + kind = PIO_KIND_BLOCKDEV; + else + elog(ERROR, "Unknown file kind \"%s\" for file '%s'", + str, path); + return kind; +} + +const char* +pio_file_kind2str(pio_file_kind_e kind, const char *path) +{ + switch (kind) + { + case PIO_KIND_REGULAR: + return "reg"; + case PIO_KIND_DIRECTORY: + return "dir"; + case PIO_KIND_SYMLINK: + return "sym"; + case PIO_KIND_FIFO: + return "fifo"; + case PIO_KIND_SOCK: + return "sock"; + case PIO_KIND_CHARDEV: + return "chdev"; + case PIO_KIND_BLOCKDEV: + return "bldev"; + default: + elog(ERROR, "Unknown file kind \"%d\" for file '%s'", + kind, path); + } + return NULL; +} + +#ifndef S_ISGID +#define S_ISGID 0 +#endif +#ifndef S_ISUID +#define S_ISUID 0 +#endif +#ifndef S_ISVTX +#define S_ISVTX 0 +#endif + +mode_t +pio_limit_mode(mode_t mode) +{ + if (S_ISDIR(mode)) + mode &= 0x1ff | S_ISGID | S_ISUID | S_ISVTX; + else + mode &= 0x1ff; + return mode; +} + /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ FILE* fio_open_stream(fio_location location, const char* path) @@ -1094,66 +1199,6 @@ fio_read(int fd, void* buf, size_t size) } } -/* Get information about file */ -int -fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlink) -{ - if (fio_is_remote(location)) - { - fio_header hdr = { - .cop = FIO_STAT, - .handle = -1, - .size = strlen(path) + 1, - .arg = follow_symlink, - }; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_STAT); - IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - return 0; - } - else - { - return follow_symlink ? stat(path, st) : lstat(path, st); - } -} - -/* - * Compare, that filename1 and filename2 is the same file - * in windows compare only filenames - */ -bool -fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink) -{ -#ifndef WIN32 - struct stat stat1, stat2; - - if (fio_stat(location, filename1, &stat1, follow_symlink) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); - - if (fio_stat(location, filename2, &stat2, follow_symlink) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); - - return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; -#else - char *abs_name1 = make_absolute_path(filename1); - char *abs_name2 = make_absolute_path(filename2); - bool result = strcmp(abs_name1, abs_name2) == 0; - free(abs_name2); - free(abs_name1); - return result; -#endif -} - /* * Read value of a symbolic link * this is a wrapper about readlink() syscall @@ -1869,6 +1914,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, } req.arg.nblocks = file->size/BLCKSZ; + Assert((int64_t)req.arg.nblocks * BLCKSZ == file->size); req.arg.segmentno = file->segno * RELSEG_SIZE; req.arg.horizonLsn = horizonLsn; req.arg.checksumVersion = checksum_version; @@ -3263,6 +3309,7 @@ fio_list_dir_impl(int out, char* buf) fio_pgFile fio_file; pgFile *file = (pgFile *) parray_get(file_files, i); + fio_file.kind = file->kind; fio_file.mode = file->mode; fio_file.size = file->size; fio_file.mtime = file->mtime; @@ -3552,10 +3599,16 @@ fio_communicate(int in, int out) size_t buf_size = 128*1024; char* buf = (char*)pgut_malloc(buf_size); fio_header hdr; - struct stat st; + pioDrive_i drive; + pio_stat_t st; int rc; int tmp_fd; pg_crc32 crc; + err_i err = $noerr(); + + FOBJ_FUNC_ARP(); + + drive = pioDriveForLocation(FIO_LOCAL_HOST); #ifdef WIN32 SYS_CHECK(setmode(in, _O_BINARY)); @@ -3564,6 +3617,7 @@ fio_communicate(int in, int out) /* Main loop until end of processing all master commands */ while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { + FOBJ_LOOP_ARP(); if (hdr.size != 0) { if (hdr.size > buf_size) { /* Extend buffer on demand */ @@ -3656,11 +3710,17 @@ fio_communicate(int in, int out) } case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); - rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); - hdr.arg = rc < 0 ? errno : 0; + st = $i(pioStat, drive, buf, .follow_symlink = hdr.arg != 0, + .err = &err); + hdr.arg = $haserr(err) ? getErrno(err) : 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); break; + case FIO_FILES_ARE_SAME: + hdr.arg = (int)$i(pioFilesAreSame, drive, buf, buf+strlen(buf)+1); + hdr.size = 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + break; case FIO_ACCESS: /* Check presence of file with specified name */ hdr.size = 0; hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0; @@ -3903,7 +3963,7 @@ pioFile_fobjDispose(VSelf) static bool common_pioExists(fobj_t self, path_t path, err_i *err) { - struct stat buf; + pio_stat_t buf; fobj_reset_err(err); /* follow symlink ? */ @@ -3913,7 +3973,7 @@ common_pioExists(fobj_t self, path_t path, err_i *err) *err = $noerr(); return false; } - if ($noerr(*err) && !S_ISREG(buf.st_mode)) + if ($noerr(*err) && buf.pst_kind != PIO_KIND_REGULAR) *err = $err(SysErr, "File {path:q} is not regular", path(path)); if ($haserr(*err)) { *err = $syserr(getErrno(*err), "Could not check file existance: {cause:$M}", @@ -3947,17 +4007,52 @@ pioLocalDrive_pioOpen(VSelf, path_t path, int flags, return bind_pioFile(file); } -static struct stat +static pio_stat_t pioLocalDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err) { struct stat st = {0}; + pio_stat_t pst = {0}; int r; fobj_reset_err(err); r = follow_symlink ? stat(path, &st) : lstat(path, &st); if (r < 0) *err = $syserr(errno, "Cannot stat file {path:q}", path(path)); - return st; + else + { + pst.pst_kind = pio_statmode2file_kind(st.st_mode, path); + pst.pst_mode = pio_limit_mode(st.st_mode); + pst.pst_size = st.st_size; + pst.pst_mtime = st.st_mtime; + } + return pst; +} + +/* + * Compare, that filename1 and filename2 is the same file + * in windows compare only filenames + */ +static bool +pioLocalDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2) +{ +#ifndef WIN32 + struct stat stat1, stat2; + + if (stat(file1, &stat1) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno)); + + if (stat(file2, &stat2) < 0) + elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno)); + + return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; +#else + char *abs_name1 = make_absolute_path(file1); + char *abs_name2 = make_absolute_path(file2); + bool result = strcmp(abs_name1, abs_name2) == 0; + free(abs_name2); + free(abs_name1); + return result; +#endif } #define pioLocalDrive_pioExists common_pioExists @@ -4213,10 +4308,10 @@ pioRemoteDrive_pioOpen(VSelf, path_t path, return bind_pioFile(file); } -static struct stat +static pio_stat_t pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err) { - struct stat st = {0}; + pio_stat_t st = {0}; fio_header hdr = { .cop = FIO_STAT, .handle = -1, @@ -4240,6 +4335,32 @@ pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err) return st; } +static bool +pioRemoteDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2) +{ + fio_header hdr = { + .cop = FIO_FILES_ARE_SAME, + .handle = -1, + .arg = 0, + }; + char _buf[512]; + ft_strbuf_t buf = ft_strbuf_init_stack(_buf, sizeof(_buf)); + ft_strbuf_catc(&buf, file1); + ft_strbuf_cat1(&buf, '\x00'); + ft_strbuf_catc(&buf, file2); + hdr.size = buf.len + 1; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len+1), buf.len+1); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + ft_dbg_assert(hdr.cop == FIO_FILES_ARE_SAME); + + ft_strbuf_free(&buf); + + return hdr.arg == 1; +} + #define pioRemoteDrive_pioExists common_pioExists static err_i @@ -4388,6 +4509,7 @@ pioRemoteDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_ta /* receive metainformation */ IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file)); + file->kind = fio_file.kind; file->mode = fio_file.mode; file->size = fio_file.size; file->mtime = fio_file.mtime; diff --git a/src/utils/file.h b/src/utils/file.h index 5aa26c21e..4dd95c871 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -67,7 +67,8 @@ typedef enum FIO_READLINK, FIO_SYNC_FILE, FIO_SEND_FILE_CONTENT, - FIO_PAGE_ZERO + FIO_PAGE_ZERO, + FIO_FILES_ARE_SAME, } fio_operations; typedef struct @@ -92,6 +93,24 @@ typedef enum FIO_REMOTE_HOST /* date is located at remote host */ } fio_location; +typedef enum pio_file_kind { + PIO_KIND_UNKNOWN = 0, + PIO_KIND_REGULAR = 1, + PIO_KIND_DIRECTORY = 2, + PIO_KIND_SYMLINK = 3, + PIO_KIND_FIFO = 4, + PIO_KIND_SOCK = 5, + PIO_KIND_CHARDEV = 6, + PIO_KIND_BLOCKDEV = 7, +} pio_file_kind_e; + +typedef struct pio_stat { + int64_t pst_size; + int64_t pst_mtime; + uint32_t pst_mode; + pio_file_kind_e pst_kind; +} pio_stat_t; + extern fio_location MyLocation; extern void setMyLocation(ProbackupSubcmd const subcmd); @@ -125,7 +144,6 @@ extern int fio_check_error_fd_gz(gzFile f, char **errmsg); extern ssize_t fio_read(int fd, void* buf, size_t size); extern int fio_flush(int fd); extern int fio_seek(int fd, off_t offs); -extern int fio_fstat(int fd, struct stat* st); extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); @@ -142,7 +160,6 @@ extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); extern int fio_ftruncate(FILE* f, off_t size); extern int fio_fclose(FILE* f); -extern int fio_ffstat(FILE* f, struct stat* st); extern FILE* fio_open_stream(fio_location location, const char* name); extern int fio_close_stream(FILE* f); @@ -175,8 +192,6 @@ extern int fio_symlink(fio_location location, const char* target, const char extern int fio_remove(fio_location location, const char* path, bool missing_ok); extern int fio_chmod(fio_location location, const char* path, int mode); extern int fio_access(fio_location location, const char* path, int mode); -extern int fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlinks); -extern bool fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink); extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz); extern pid_t fio_check_postmaster(fio_location location, const char *pgdata); @@ -198,6 +213,10 @@ extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok); #endif extern pg_crc32 pgFileGetCRC32Cgz(const char *file_path, bool missing_ok); +extern pio_file_kind_e pio_statmode2file_kind(mode_t mode, const char* path); +extern pio_file_kind_e pio_str2file_kind(const char* str, const char* path); +extern const char* pio_file_kind2str(pio_file_kind_e kind, const char* path); +extern mode_t pio_limit_mode(mode_t mode); // OBJECTS @@ -237,19 +256,19 @@ fobj_iface(pioWriteFlush); fobj_iface(pioWriteCloser); fobj_iface(pioReadCloser); -typedef struct stat stat_t; - // Drive #define mth__pioOpen pioFile_i, (path_t, path), (int, flags), \ (int, permissions), (err_i *, err) #define mth__pioOpen__optional() (permissions, FILE_PERMISSION) -#define mth__pioStat stat_t, (path_t, path), (bool, follow_symlink), \ +#define mth__pioStat pio_stat_t, (path_t, path), (bool, follow_symlink), \ (err_i *, err) #define mth__pioRemove err_i, (path_t, path), (bool, missing_ok) #define mth__pioRename err_i, (path_t, old_path), (path_t, new_path) #define mth__pioExists bool, (path_t, path), (err_i *, err) #define mth__pioGetCRC32 pg_crc32, (path_t, path), (bool, compressed), \ (err_i *, err) +/* Compare, that filename1 and filename2 is the same file */ +#define mth__pioFilesAreSame bool, (path_t, file1), (path_t, file2) #define mth__pioIsRemote bool #define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict) #define mth__pioListDir void, (parray *, files), (const char *, root), \ @@ -265,12 +284,14 @@ fobj_method(pioExists); fobj_method(pioIsRemote); fobj_method(pioGetCRC32); fobj_method(pioMakeDir); +fobj_method(pioFilesAreSame); fobj_method(pioListDir); fobj_method(pioRemoveDir); #define iface__pioDrive mth(pioOpen, pioStat, pioRemove, pioRename), \ mth(pioExists, pioGetCRC32, pioIsRemote), \ - mth(pioMakeDir, pioListDir, pioRemoveDir) + mth(pioMakeDir, pioListDir, pioRemoveDir), \ + mth(pioFilesAreSame) fobj_iface(pioDrive); extern pioDrive_i pioDriveForLocation(fio_location location); diff --git a/src/validate.c b/src/validate.c index 84a27adcb..8adeb52a9 100644 --- a/src/validate.c +++ b/src/validate.c @@ -55,6 +55,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) pthread_t *threads; validate_files_arg *threads_args; int i; + err_i err; // parray *dbOid_exclude_list = NULL; /* Check backup program version */ @@ -199,14 +200,16 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) (parse_program_version(backup->program_version) == 20201))) { char path[MAXPGPATH]; - struct stat st; + pio_stat_t st; join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - if (fio_stat(FIO_BACKUP_HOST, path, &st, true) < 0) - elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST), + .path = path, .follow_symlink = true, .err = &err); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), ""); - if (st.st_size >= (BLCKSZ*500)) + if (st.pst_size >= (BLCKSZ*500)) { elog(WARNING, "Backup %s is a victim of metadata corruption. " "Additional information can be found here: " @@ -242,7 +245,7 @@ pgBackupValidateFiles(void *arg) elog(ERROR, "Interrupted during validate"); /* Validate only regular files */ - if (!S_ISREG(file->mode)) + if (file->kind != PIO_KIND_REGULAR) continue; /*