From e7834a1a251d4a28245377f383ff20a657ba8262 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 3 Oct 2024 15:55:37 +0900 Subject: [PATCH] Add log_verbosity = 'silent' support to COPY command. Previously, when the on_error option was set to ignore, the COPY command would always log NOTICE messages for input rows discarded due to data type incompatibility. Users had no way to suppress these messages. This commit introduces a new log_verbosity setting, 'silent', which prevents the COPY command from emitting NOTICE messages when on_error = 'ignore' is used, even if rows are discarded. This feature is particularly useful when processing malformed files frequently, where a flood of NOTICE messages can be undesirable. For example, when frequently loading malformed files via the COPY command or querying foreign tables using file_fdw (with an upcoming patch to add on_error support for file_fdw), users may prefer to suppress these messages to reduce log noise and improve clarity. Author: Atsushi Torikoshi Reviewed-by: Masahiko Sawada, Fujii Masao Discussion: https://postgr.es/m/ab59dad10490ea3734cf022b16c24cfd@oss.nttdata.com --- doc/src/sgml/ref/copy.sgml | 10 +++++++--- src/backend/commands/copy.c | 4 +++- src/backend/commands/copyfrom.c | 3 ++- src/bin/psql/tab-complete.c | 2 +- src/include/commands/copy.h | 4 +++- src/test/regress/expected/copy2.out | 4 +++- src/test/regress/sql/copy2.sql | 4 ++++ 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 33633447f16..b9413d48925 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -407,6 +407,8 @@ COPY { table_name [ ( verbose, a NOTICE message containing the line of the input file and the column name whose input conversion has failed is emitted for each discarded row. + When it is set to silent, no message is emitted + regarding ignored rows. @@ -428,9 +430,11 @@ COPY { table_name [ ( Specifies the amount of messages emitted by a COPY - command: default or verbose. If - verbose is specified, additional messages are emitted - during processing. + command: default, verbose, or + silent. + If verbose is specified, additional messages are + emitted during processing. + silent suppresses both verbose and default messages. This is currently used in COPY FROM command when diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3bb579a3a44..03eb7a4ebac 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -427,9 +427,11 @@ defGetCopyLogVerbosityChoice(DefElem *def, ParseState *pstate) char *sval; /* - * Allow "default", or "verbose" values. + * Allow "silent", "default", or "verbose" values. */ sval = defGetString(def); + if (pg_strcasecmp(sval, "silent") == 0) + return COPY_LOG_VERBOSITY_SILENT; if (pg_strcasecmp(sval, "default") == 0) return COPY_LOG_VERBOSITY_DEFAULT; if (pg_strcasecmp(sval, "verbose") == 0) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 2d3462913e1..47879994f78 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1320,7 +1320,8 @@ CopyFrom(CopyFromState cstate) error_context_stack = errcallback.previous; if (cstate->opts.on_error != COPY_ON_ERROR_STOP && - cstate->num_errors > 0) + cstate->num_errors > 0 && + cstate->opts.log_verbosity >= COPY_LOG_VERBOSITY_DEFAULT) ereport(NOTICE, errmsg_plural("%llu row was skipped due to data type incompatibility", "%llu rows were skipped due to data type incompatibility", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index a7ccde6d7df..6530b0f1ce3 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2916,7 +2916,7 @@ psql_completion(const char *text, int start, int end) /* Complete COPY FROM filename WITH (LOG_VERBOSITY */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "LOG_VERBOSITY")) - COMPLETE_WITH("default", "verbose"); + COMPLETE_WITH("silent", "default", "verbose"); /* Complete COPY FROM WITH () */ else if (Matches("COPY|\\copy", MatchAny, "FROM", MatchAny, "WITH", MatchAny)) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 141fd48dc10..6f64d97fdd9 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -45,7 +45,9 @@ typedef enum CopyOnErrorChoice */ typedef enum CopyLogVerbosityChoice { - COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages, default */ + COPY_LOG_VERBOSITY_SILENT = -1, /* logs none */ + COPY_LOG_VERBOSITY_DEFAULT = 0, /* logs no additional messages. As this is + * the default, assign 0 */ COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */ } CopyLogVerbosityChoice; diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 61a19cdc4ca..4e752977b53 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -760,6 +760,7 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose); NOTICE: skipping row due to data type incompatibility at line 2 for column "l": null input CONTEXT: COPY check_ign_err2 NOTICE: 1 row was skipped due to data type incompatibility +COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent); -- reset context choice \set SHOW_CONTEXT errors SELECT * FROM check_ign_err; @@ -774,7 +775,8 @@ SELECT * FROM check_ign_err2; n | m | k | l ---+-----+---+------- 1 | {1} | 1 | 'foo' -(1 row) + 3 | {3} | 3 | 'bar' +(2 rows) -- test datatype error that can't be handled as soft: should fail CREATE TABLE hard_err(foo widget); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 8b149621947..fa6aa17344a 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -533,6 +533,10 @@ COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity verbose); 1 {1} 1 'foo' 2 {2} 2 \N \. +COPY check_ign_err2 FROM STDIN WITH (on_error ignore, log_verbosity silent); +3 {3} 3 'bar' +4 {4} 4 \N +\. -- reset context choice \set SHOW_CONTEXT errors -- 2.30.2