copy-errors.diff

From 18f328c8a20dfdbd805921c6dcb6d665067507a5 Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Fri, 9 Jun 2017 23:41:51 +0300
Subject: [PATCH 1/8] Allow ignoring some errors during COPY FROM

---
 contrib/file_fdw/file_fdw.c |   4 +-
 src/backend/commands/copy.c | 352 +++++++++++++++++++++++++-------------------
 src/include/commands/copy.h |   2 +-
 3 files changed, 207 insertions(+), 151 deletions(-)

diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 2396bd442f..c83887944a 100644
--- a/contrib/file_fdw/file_fdw.c
+++ b/contrib/file_fdw/file_fdw.c
@@ -689,7 +689,7 @@ fileIterateForeignScan(ForeignScanState *node)
 {
 	FileFdwExecutionState *festate = (FileFdwExecutionState *) node->fdw_state;
 	TupleTableSlot *slot = node->ss.ss_ScanTupleSlot;
-	bool		found;
+	int		found;
 	ErrorContextCallback errcallback;
 
 	/* Set up callback to identify error line number. */
@@ -1080,7 +1080,7 @@ file_acquire_sample_rows(Relation onerel, int elevel,
 	TupleDesc	tupDesc;
 	Datum	   *values;
 	bool	   *nulls;
-	bool		found;
+	int		    found;
 	char	   *filename;
 	bool		is_program;
 	List	   *options;
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index fc5f4f66ea..935ef960b0 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -54,6 +54,16 @@
 #define OCTVALUE(c) ((c) - '0')
 
 /*
+    NextCopyFrom states:
+    0 – Error or stop
+    1 – Successfully read data
+    2 – Data with errors, skip
+*/
+#define NCF_STOP         0
+#define NCF_SUCCESS      1
+#define NCF_SKIP         2
+
+/*
  * Represents the different source/dest cases we need to worry about at
  * the bottom level
  */
@@ -139,6 +149,7 @@ typedef struct CopyStateData
 	int			cur_lineno;		/* line number for error messages */
 	const char *cur_attname;	/* current att for error messages */
 	const char *cur_attval;		/* current att value for error messages */
+    bool        ignore_errors;  /* ignore errors during COPY FROM */
 
 	/*
 	 * Working state for COPY TO/FROM
@@ -2340,6 +2351,7 @@ CopyFrom(CopyState cstate)
 	bool		useHeapMultiInsert;
 	int			nBufferedTuples = 0;
 	int			prev_leaf_part_index = -1;
+    int         next_cf_state; /* NextCopyFrom return state */
 
 #define MAX_BUFFERED_TUPLES 1000
 	HeapTuple  *bufferedTuples = NULL;	/* initialize to silence warning */
@@ -2549,145 +2561,154 @@ CopyFrom(CopyState cstate)
 		/* Switch into its memory context */
 		MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
 
-		if (!NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid))
-			break;
-
-		/* And now we can form the input tuple. */
-		tuple = heap_form_tuple(tupDesc, values, nulls);
-
-		if (loaded_oid != InvalidOid)
-			HeapTupleSetOid(tuple, loaded_oid);
-
-		/*
-		 * Constraints might reference the tableoid column, so initialize
-		 * t_tableOid before evaluating them.
-		 */
-		tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
-
-		/* Triggers and stuff need to be invoked in query context. */
-		MemoryContextSwitchTo(oldcontext);
-
-		/* Place tuple in tuple slot --- but slot shouldn't free it */
-		slot = myslot;
-		ExecStoreTuple(tuple, slot, InvalidBuffer, false);
-
-		/* Determine the partition to heap_insert the tuple into */
-		if (cstate->partition_dispatch_info)
-		{
-			int			leaf_part_index;
-			TupleConversionMap *map;
-
-			/*
-			 * Away we go ... If we end up not finding a partition after all,
-			 * ExecFindPartition() does not return and errors out instead.
-			 * Otherwise, the returned value is to be used as an index into
-			 * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
-			 * will get us the ResultRelInfo and TupleConversionMap for the
-			 * partition, respectively.
-			 */
-			leaf_part_index = ExecFindPartition(resultRelInfo,
-												cstate->partition_dispatch_info,
-												slot,
-												estate);
-			Assert(leaf_part_index >= 0 &&
-				   leaf_part_index < cstate->num_partitions);
-
-			/*
-			 * If this tuple is mapped to a partition that is not same as the
-			 * previous one, we'd better make the bulk insert mechanism gets a
-			 * new buffer.
-			 */
-			if (prev_leaf_part_index != leaf_part_index)
-			{
-				ReleaseBulkInsertStatePin(bistate);
-				prev_leaf_part_index = leaf_part_index;
-			}
-
-			/*
-			 * Save the old ResultRelInfo and switch to the one corresponding
-			 * to the selected partition.
-			 */
-			saved_resultRelInfo = resultRelInfo;
-			resultRelInfo = cstate->partitions + leaf_part_index;
-
-			/* We do not yet have a way to insert into a foreign partition */
-			if (resultRelInfo->ri_FdwRoutine)
-				ereport(ERROR,
-						(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-						 errmsg("cannot route inserted tuples to a foreign table")));
-
-			/*
-			 * For ExecInsertIndexTuples() to work on the partition's indexes
-			 */
-			estate->es_result_relation_info = resultRelInfo;
-
-			/*
-			 * If we're capturing transition tuples, we might need to convert
-			 * from the partition rowtype to parent rowtype.
-			 */
-			if (cstate->transition_capture != NULL)
-			{
-				if (resultRelInfo->ri_TrigDesc &&
-					(resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
-					 resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
-				{
-					/*
-					 * If there are any BEFORE or INSTEAD triggers on the
-					 * partition, we'll have to be ready to convert their
-					 * result back to tuplestore format.
-					 */
-					cstate->transition_capture->tcs_original_insert_tuple = NULL;
-					cstate->transition_capture->tcs_map =
-						cstate->transition_tupconv_maps[leaf_part_index];
-				}
-				else
-				{
-					/*
-					 * Otherwise, just remember the original unconverted
-					 * tuple, to avoid a needless round trip conversion.
-					 */
-					cstate->transition_capture->tcs_original_insert_tuple = tuple;
-					cstate->transition_capture->tcs_map = NULL;
-				}
-			}
-			/*
-			 * We might need to convert from the parent rowtype to the
-			 * partition rowtype.
-			 */
-			map = cstate->partition_tupconv_maps[leaf_part_index];
-			if (map)
-			{
-				Relation	partrel = resultRelInfo->ri_RelationDesc;
-
-				tuple = do_convert_tuple(tuple, map);
-
-				/*
-				 * We must use the partition's tuple descriptor from this
-				 * point on.  Use a dedicated slot from this point on until
-				 * we're finished dealing with the partition.
-				 */
-				slot = cstate->partition_tuple_slot;
-				Assert(slot != NULL);
-				ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
-				ExecStoreTuple(tuple, slot, InvalidBuffer, true);
-			}
+        next_cf_state = NextCopyFrom(cstate, econtext, values, nulls, &loaded_oid);
 
-			tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
-		}
-
-		skip_tuple = false;
-
-		/* BEFORE ROW INSERT Triggers */
-		if (resultRelInfo->ri_TrigDesc &&
-			resultRelInfo->ri_TrigDesc->trig_insert_before_row)
-		{
-			slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
-
-			if (slot == NULL)	/* "do nothing" */
-				skip_tuple = true;
-			else				/* trigger might have changed tuple */
-				tuple = ExecMaterializeSlot(slot);
+		if (!next_cf_state) {
+			break;
 		}
+        else if (next_cf_state == NCF_SUCCESS)
+        {
+    		/* And now we can form the input tuple. */
+    		tuple = heap_form_tuple(tupDesc, values, nulls);
+
+    		if (loaded_oid != InvalidOid)
+    			HeapTupleSetOid(tuple, loaded_oid);
+
+    		/*
+    		 * Constraints might reference the tableoid column, so initialize
+    		 * t_tableOid before evaluating them.
+    		 */
+    		tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+    		/* Triggers and stuff need to be invoked in query context. */
+    		MemoryContextSwitchTo(oldcontext);
+
+    		/* Place tuple in tuple slot --- but slot shouldn't free it */
+    		slot = myslot;
+    		ExecStoreTuple(tuple, slot, InvalidBuffer, false);
+
+    		/* Determine the partition to heap_insert the tuple into */
+    		if (cstate->partition_dispatch_info)
+    		{
+    			int			leaf_part_index;
+    			TupleConversionMap *map;
+
+    			/*
+    			 * Away we go ... If we end up not finding a partition after all,
+    			 * ExecFindPartition() does not return and errors out instead.
+    			 * Otherwise, the returned value is to be used as an index into
+    			 * arrays mt_partitions[] and mt_partition_tupconv_maps[] that
+    			 * will get us the ResultRelInfo and TupleConversionMap for the
+    			 * partition, respectively.
+    			 */
+    			leaf_part_index = ExecFindPartition(resultRelInfo,
+    												cstate->partition_dispatch_info,
+    												slot,
+    												estate);
+    			Assert(leaf_part_index >= 0 &&
+    				   leaf_part_index < cstate->num_partitions);
+
+    			/*
+    			 * If this tuple is mapped to a partition that is not same as the
+    			 * previous one, we'd better make the bulk insert mechanism gets a
+    			 * new buffer.
+    			 */
+    			if (prev_leaf_part_index != leaf_part_index)
+    			{
+    				ReleaseBulkInsertStatePin(bistate);
+    				prev_leaf_part_index = leaf_part_index;
+    			}
+
+    			/*
+    			 * Save the old ResultRelInfo and switch to the one corresponding
+    			 * to the selected partition.
+    			 */
+    			saved_resultRelInfo = resultRelInfo;
+    			resultRelInfo = cstate->partitions + leaf_part_index;
+
+    			/* We do not yet have a way to insert into a foreign partition */
+    			if (resultRelInfo->ri_FdwRoutine)
+    				ereport(ERROR,
+    						(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+    						 errmsg("cannot route inserted tuples to a foreign table")));
+
+    			/*
+    			 * For ExecInsertIndexTuples() to work on the partition's indexes
+    			 */
+    			estate->es_result_relation_info = resultRelInfo;
+
+    			/*
+    			 * If we're capturing transition tuples, we might need to convert
+    			 * from the partition rowtype to parent rowtype.
+    			 */
+    			if (cstate->transition_capture != NULL)
+    			{
+    				if (resultRelInfo->ri_TrigDesc &&
+    					(resultRelInfo->ri_TrigDesc->trig_insert_before_row ||
+    					 resultRelInfo->ri_TrigDesc->trig_insert_instead_row))
+    				{
+    					/*
+    					 * If there are any BEFORE or INSTEAD triggers on the
+    					 * partition, we'll have to be ready to convert their
+    					 * result back to tuplestore format.
+    					 */
+    					cstate->transition_capture->tcs_original_insert_tuple = NULL;
+    					cstate->transition_capture->tcs_map =
+    						cstate->transition_tupconv_maps[leaf_part_index];
+    				}
+    				else
+    				{
+    					/*
+    					 * Otherwise, just remember the original unconverted
+    					 * tuple, to avoid a needless round trip conversion.
+    					 */
+    					cstate->transition_capture->tcs_original_insert_tuple = tuple;
+    					cstate->transition_capture->tcs_map = NULL;
+    				}
+    			}
+    			/*
+    			 * We might need to convert from the parent rowtype to the
+    			 * partition rowtype.
+    			 */
+    			map = cstate->partition_tupconv_maps[leaf_part_index];
+    			if (map)
+    			{
+    				Relation	partrel = resultRelInfo->ri_RelationDesc;
+
+    				tuple = do_convert_tuple(tuple, map);
+
+    				/*
+    				 * We must use the partition's tuple descriptor from this
+    				 * point on.  Use a dedicated slot from this point on until
+    				 * we're finished dealing with the partition.
+    				 */
+    				slot = cstate->partition_tuple_slot;
+    				Assert(slot != NULL);
+    				ExecSetSlotDescriptor(slot, RelationGetDescr(partrel));
+    				ExecStoreTuple(tuple, slot, InvalidBuffer, true);
+    			}
+
+    			tuple->t_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+    		}
+
+    		skip_tuple = false;
+
+    		/* BEFORE ROW INSERT Triggers */
+    		if (resultRelInfo->ri_TrigDesc &&
+    			resultRelInfo->ri_TrigDesc->trig_insert_before_row)
+    		{
+    			slot = ExecBRInsertTriggers(estate, resultRelInfo, slot);
+
+    			if (slot == NULL)	/* "do nothing" */
+    				skip_tuple = true;
+    			else				/* trigger might have changed tuple */
+    				tuple = ExecMaterializeSlot(slot);
+    		}
+        }
+        else
+        {
+            skip_tuple = true;
+        }
 
 		if (!skip_tuple)
 		{
@@ -2985,6 +3006,7 @@ BeginCopyFrom(ParseState *pstate,
 	cstate->cur_lineno = 0;
 	cstate->cur_attname = NULL;
 	cstate->cur_attval = NULL;
+    cstate->ignore_errors = true;
 
 	/* Set up variables to avoid per-attribute overhead. */
 	initStringInfo(&cstate->attribute_buf);
@@ -3262,7 +3284,7 @@ NextCopyFromRawFields(CopyState cstate, char ***fields, int *nfields)
  * relation passed to BeginCopyFrom. This function fills the arrays.
  * Oid of the tuple is returned with 'tupleOid' separately.
  */
-bool
+int
 NextCopyFrom(CopyState cstate, ExprContext *econtext,
 			 Datum *values, bool *nulls, Oid *tupleOid)
 {
@@ -3280,11 +3302,20 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 	int		   *defmap = cstate->defmap;
 	ExprState **defexprs = cstate->defexprs;
 
+    int         error_level = ERROR;        /* Error level for COPY FROM input data errors */
+    int         exec_state = NCF_SUCCESS;   /* Return code */
+
 	tupDesc = RelationGetDescr(cstate->rel);
 	attr = tupDesc->attrs;
 	num_phys_attrs = tupDesc->natts;
 	attr_count = list_length(cstate->attnumlist);
 	nfields = file_has_oids ? (attr_count + 1) : attr_count;
+    
+    /* Set error level to WARNING, if errors handling is turned on */
+    if (cstate->ignore_errors)
+    {
+        error_level = WARNING;
+    }
 
 	/* Initialize all values for row to NULL */
 	MemSet(values, 0, num_phys_attrs * sizeof(Datum));
@@ -3300,13 +3331,17 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 
 		/* read raw fields in the next line */
 		if (!NextCopyFromRawFields(cstate, &field_strings, &fldct))
-			return false;
+			return NCF_STOP;
 
 		/* check for overflowing fields */
 		if (nfields > 0 && fldct > nfields)
-			ereport(ERROR,
+        {
+            exec_state = NCF_SKIP;
+			ereport(error_level,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("extra data after last expected column")));
+        }
+
 
 		fieldno = 0;
 
@@ -3314,15 +3349,22 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 		if (file_has_oids)
 		{
 			if (fieldno >= fldct)
-				ereport(ERROR,
+            {
+                exec_state = NCF_SKIP;
+				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("missing data for OID column")));
+            }
+
 			string = field_strings[fieldno++];
 
 			if (string == NULL)
-				ereport(ERROR,
+            {
+                exec_state = NCF_SKIP;
+				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("null OID in COPY data")));
+            }
 			else if (cstate->oids && tupleOid != NULL)
 			{
 				cstate->cur_attname = "oid";
@@ -3330,9 +3372,13 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 				*tupleOid = DatumGetObjectId(DirectFunctionCall1(oidin,
 																 CStringGetDatum(string)));
 				if (*tupleOid == InvalidOid)
-					ereport(ERROR,
+                {
+                    exec_state = NCF_SKIP;
+					ereport(error_level,
 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 							 errmsg("invalid OID in COPY data")));
+                }
+
 				cstate->cur_attname = NULL;
 				cstate->cur_attval = NULL;
 			}
@@ -3345,10 +3391,14 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 			int			m = attnum - 1;
 
 			if (fieldno >= fldct)
-				ereport(ERROR,
+            {
+                exec_state = NCF_SKIP;
+				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("missing data for column \"%s\"",
 								NameStr(attr[m]->attname))));
+            }
+
 			string = field_strings[fieldno++];
 
 			if (cstate->convert_select_flags &&
@@ -3431,14 +3481,17 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 				ereport(ERROR,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("received copy data after EOF marker")));
-			return false;
+			return NCF_STOP;
 		}
 
 		if (fld_count != attr_count)
-			ereport(ERROR,
+        {
+            exec_state = NCF_SKIP;
+			ereport(error_level,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("row field count is %d, expected %d",
 							(int) fld_count, attr_count)));
+        }
 
 		if (file_has_oids)
 		{
@@ -3453,9 +3506,12 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 														 -1,
 														 &isnull));
 			if (isnull || loaded_oid == InvalidOid)
-				ereport(ERROR,
+            {
+                exec_state = NCF_SKIP;
+				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("invalid OID in COPY data")));
+            }
 			cstate->cur_attname = NULL;
 			if (cstate->oids && tupleOid != NULL)
 				*tupleOid = loaded_oid;
@@ -3497,7 +3553,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 										 &nulls[defmap[i]]);
 	}
 
-	return true;
+	return exec_state;
 }
 
 /*
diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h
index 8b2971d287..bba64fc3d7 100644
--- a/src/include/commands/copy.h
+++ b/src/include/commands/copy.h
@@ -31,7 +31,7 @@ extern void ProcessCopyOptions(ParseState *pstate, CopyState cstate, bool is_fro
 extern CopyState BeginCopyFrom(ParseState *pstate, Relation rel, const char *filename,
 			  bool is_program, copy_data_source_cb data_source_cb, List *attnamelist, List *options);
 extern void EndCopyFrom(CopyState cstate);
-extern bool NextCopyFrom(CopyState cstate, ExprContext *econtext,
+extern int NextCopyFrom(CopyState cstate, ExprContext *econtext,
 			 Datum *values, bool *nulls, Oid *tupleOid);
 extern bool NextCopyFromRawFields(CopyState cstate,
 					  char ***fields, int *nfields);
-- 
2.11.0


From 9b19c9e365c019ae0e3e56d05502044c5683fd09 Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Sun, 11 Jun 2017 16:48:37 +0300
Subject: [PATCH 2/8] Report line number on each ERROR/WARNING

---
 src/backend/commands/copy.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 935ef960b0..4dd9acd899 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -3339,7 +3339,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
             exec_state = NCF_SKIP;
 			ereport(error_level,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
-					 errmsg("extra data after last expected column")));
+					 errmsg("extra data after last expected column at line %d", cstate->cur_lineno)));
         }
 
 
@@ -3353,7 +3353,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
                 exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
-						 errmsg("missing data for OID column")));
+						 errmsg("missing data for OID column at line %d", cstate->cur_lineno)));
             }
 
 			string = field_strings[fieldno++];
@@ -3363,7 +3363,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
                 exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
-						 errmsg("null OID in COPY data")));
+						 errmsg("null OID in COPY data at line %d", cstate->cur_lineno)));
             }
 			else if (cstate->oids && tupleOid != NULL)
 			{
@@ -3376,7 +3376,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
                     exec_state = NCF_SKIP;
 					ereport(error_level,
 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
-							 errmsg("invalid OID in COPY data")));
+							 errmsg("invalid OID in COPY data at line %d", cstate->cur_lineno)));
                 }
 
 				cstate->cur_attname = NULL;
@@ -3395,8 +3395,8 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
                 exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
-						 errmsg("missing data for column \"%s\"",
-								NameStr(attr[m]->attname))));
+						 errmsg("missing data for column \"%s\" at line %d",
+								NameStr(attr[m]->attname), cstate->cur_lineno)));
             }
 
 			string = field_strings[fieldno++];
-- 
2.11.0


From 9e395007d1b35dcaaa90fe884514091b6dec5dd9 Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Tue, 13 Jun 2017 14:57:59 +0300
Subject: [PATCH 3/8] Catch errors at InputFunctionCall inside NextCopyFrom

---
 src/backend/commands/copy.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 4dd9acd899..be76c0df1e 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -3304,6 +3304,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 
     int         error_level = ERROR;        /* Error level for COPY FROM input data errors */
     int         exec_state = NCF_SUCCESS;   /* Return code */
+    MemoryContext oldcontext = CurrentMemoryContext;
 
 	tupDesc = RelationGetDescr(cstate->rel);
 	attr = tupDesc->attrs;
@@ -3434,10 +3435,38 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 
 			cstate->cur_attname = NameStr(attr[m]->attname);
 			cstate->cur_attval = string;
+
+            PG_TRY();
+            {
 			values[m] = InputFunctionCall(&in_functions[m],
 										  string,
 										  typioparams[m],
 										  attr[m]->atttypmod);
+            }
+            PG_CATCH();
+            {
+            if (cstate->ignore_errors)
+            {
+        		ErrorData  *edata;
+
+        		/* Save error info */
+                MemoryContextSwitchTo(oldcontext);
+        		edata = CopyErrorData();
+                FlushErrorState();
+                
+                /* TODO Find an appropriate errcode */
+				ereport(WARNING,
+						(errcode(ERRCODE_TOO_MANY_COLUMNS),
+                        errmsg("%s at line %d col %d", edata->message, cstate->cur_lineno, attnum)));
+                return NCF_SKIP;
+            }
+            else
+            {
+                PG_RE_THROW();
+            }
+            }
+            PG_END_TRY();
+
 			if (string != NULL)
 				nulls[m] = false;
 			cstate->cur_attname = NULL;
-- 
2.11.0


From c5074cf2b4d7595a770c9ac891bf78386532b9f8 Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Tue, 13 Jun 2017 16:30:50 +0300
Subject: [PATCH 4/8] Tabs vs spaces consistency

---
 src/backend/commands/copy.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index be76c0df1e..f1042f2f41 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -3447,17 +3447,17 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
             {
             if (cstate->ignore_errors)
             {
-        		ErrorData  *edata;
+                ErrorData  *edata;
 
-        		/* Save error info */
+                /* Save error info */
                 MemoryContextSwitchTo(oldcontext);
-        		edata = CopyErrorData();
+                edata = CopyErrorData();
                 FlushErrorState();
                 
                 /* TODO Find an appropriate errcode */
-				ereport(WARNING,
-						(errcode(ERRCODE_TOO_MANY_COLUMNS),
-                        errmsg("%s at line %d col %d", edata->message, cstate->cur_lineno, attnum)));
+                ereport(WARNING,
+                        (errcode(ERRCODE_TOO_MANY_COLUMNS),
+                         errmsg("%s at line %d col %d", edata->message, cstate->cur_lineno, attnum)));
                 return NCF_SKIP;
             }
             else
-- 
2.11.0


From 9a8c8ebed9f6722738bc3bb3132d488074ef9654 Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Thu, 15 Jun 2017 20:35:51 +0300
Subject: [PATCH 5/8] Fixed possible Segmentation fault during malformed lines
 processing

---
 src/backend/commands/copy.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index f1042f2f41..cb5b4ad302 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -3303,7 +3303,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 	ExprState **defexprs = cstate->defexprs;
 
     int         error_level = ERROR;        /* Error level for COPY FROM input data errors */
-    int         exec_state = NCF_SUCCESS;   /* Return code */
+    // int         exec_state = NCF_SUCCESS;   /* Return code */
     MemoryContext oldcontext = CurrentMemoryContext;
 
 	tupDesc = RelationGetDescr(cstate->rel);
@@ -3337,10 +3337,10 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 		/* check for overflowing fields */
 		if (nfields > 0 && fldct > nfields)
         {
-            exec_state = NCF_SKIP;
 			ereport(error_level,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("extra data after last expected column at line %d", cstate->cur_lineno)));
+            return NCF_SKIP;
         }
 
 
@@ -3351,20 +3351,20 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 		{
 			if (fieldno >= fldct)
             {
-                exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("missing data for OID column at line %d", cstate->cur_lineno)));
+                return NCF_SKIP;
             }
 
 			string = field_strings[fieldno++];
 
 			if (string == NULL)
             {
-                exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("null OID in COPY data at line %d", cstate->cur_lineno)));
+                return NCF_SKIP;
             }
 			else if (cstate->oids && tupleOid != NULL)
 			{
@@ -3374,10 +3374,10 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 																 CStringGetDatum(string)));
 				if (*tupleOid == InvalidOid)
                 {
-                    exec_state = NCF_SKIP;
 					ereport(error_level,
 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 							 errmsg("invalid OID in COPY data at line %d", cstate->cur_lineno)));
+                    return NCF_SKIP;
                 }
 
 				cstate->cur_attname = NULL;
@@ -3393,11 +3393,11 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 
 			if (fieldno >= fldct)
             {
-                exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("missing data for column \"%s\" at line %d",
 								NameStr(attr[m]->attname), cstate->cur_lineno)));
+                return NCF_SKIP;
             }
 
 			string = field_strings[fieldno++];
@@ -3486,7 +3486,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 		if (!CopyGetInt16(cstate, &fld_count))
 		{
 			/* EOF detected (end of file, or protocol-level EOF) */
-			return false;
+			return NCF_STOP;
 		}
 
 		if (fld_count == -1)
@@ -3515,11 +3515,11 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 
 		if (fld_count != attr_count)
         {
-            exec_state = NCF_SKIP;
 			ereport(error_level,
 					(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 					 errmsg("row field count is %d, expected %d",
 							(int) fld_count, attr_count)));
+            return NCF_SKIP;
         }
 
 		if (file_has_oids)
@@ -3536,10 +3536,10 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 														 &isnull));
 			if (isnull || loaded_oid == InvalidOid)
             {
-                exec_state = NCF_SKIP;
 				ereport(error_level,
 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
 						 errmsg("invalid OID in COPY data")));
+                return NCF_SKIP;
             }
 			cstate->cur_attname = NULL;
 			if (cstate->oids && tupleOid != NULL)
@@ -3582,7 +3582,7 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 										 &nulls[defmap[i]]);
 	}
 
-	return exec_state;
+	return NCF_SUCCESS;
 }
 
 /*
-- 
2.11.0


From c4c93114eae5bbe6850e07c79f21be51f3f9b76c Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Fri, 16 Jun 2017 13:14:29 +0300
Subject: [PATCH 6/8] Propagate catched SQL error code to warning

---
 src/backend/commands/copy.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index cb5b4ad302..6305b3b0c5 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -3302,8 +3302,8 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 	int		   *defmap = cstate->defmap;
 	ExprState **defexprs = cstate->defexprs;
 
-    int         error_level = ERROR;        /* Error level for COPY FROM input data errors */
-    // int         exec_state = NCF_SUCCESS;   /* Return code */
+    /* Error level for COPY FROM input data errors */
+    int         error_level = ERROR;
     MemoryContext oldcontext = CurrentMemoryContext;
 
 	tupDesc = RelationGetDescr(cstate->rel);
@@ -3436,6 +3436,10 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
 			cstate->cur_attname = NameStr(attr[m]->attname);
 			cstate->cur_attval = string;
 
+            /*
+             * Catch errors inside InputFunctionCall to handle
+             * errors due to the type invalid syntax.
+             */
             PG_TRY();
             {
 			values[m] = InputFunctionCall(&in_functions[m],
@@ -3453,15 +3457,16 @@ NextCopyFrom(CopyState cstate, ExprContext *econtext,
                 MemoryContextSwitchTo(oldcontext);
                 edata = CopyErrorData();
                 FlushErrorState();
-                
-                /* TODO Find an appropriate errcode */
+
+                /* Propagate catched ERROR sqlerrcode and message as WARNING */
                 ereport(WARNING,
-                        (errcode(ERRCODE_TOO_MANY_COLUMNS),
+                        (errcode(edata->sqlerrcode),
                          errmsg("%s at line %d col %d", edata->message, cstate->cur_lineno, attnum)));
                 return NCF_SKIP;
             }
             else
             {
+                /* Propagate ERROR as is if errors handling is not turned on */
                 PG_RE_THROW();
             }
             }
-- 
2.11.0


From cc2c804122693ca9caa7f49dc19bbb7c43dbe87c Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Mon, 19 Jun 2017 20:59:34 +0300
Subject: [PATCH 7/8] IGNORE_ERRORS COPY SQL option added (only new syntax
 supported)

---
 doc/src/sgml/ref/copy.sgml  | 12 ++++++++++++
 src/backend/commands/copy.c | 10 +++++++++-
 src/backend/parser/gram.y   |  6 +++++-
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml
index 8de1150dfb..5e322308d6 100644
--- a/doc/src/sgml/ref/copy.sgml
+++ b/doc/src/sgml/ref/copy.sgml
@@ -44,6 +44,7 @@ COPY { <replaceable class="parameter">table_name</replaceable> [ ( <replaceable
     FORCE_NOT_NULL ( <replaceable class="parameter">column_name</replaceable> [, ...] )
     FORCE_NULL ( <replaceable class="parameter">column_name</replaceable> [, ...] )
     ENCODING '<replaceable class="parameter">encoding_name</replaceable>'
+    IGNORE_ERRORS [ <replaceable class="parameter">boolean</replaceable> ]
 </synopsis>
  </refsynopsisdiv>
 
@@ -249,6 +250,17 @@ COPY { <replaceable class="parameter">table_name</replaceable> [ ( <replaceable
    </varlistentry>
 
    <varlistentry>
+    <term><literal>IGNORE_ERRORS</literal></term>
+    <listitem>
+     <para>
+      Specifies ignoring errors in the input data. If TRUE, then all
+      errors due to the extra or missing columns and inappropriate type
+      format will be ignored with WARNING.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry>
     <term><literal>NULL</literal></term>
     <listitem>
      <para>
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 6305b3b0c5..29c5376b2d 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1214,6 +1214,15 @@ ProcessCopyOptions(ParseState *pstate,
 								defel->defname),
 						 parser_errposition(pstate, defel->location)));
 		}
+		else if (strcmp(defel->defname, "ignore_errors") == 0)
+		{
+			if (cstate->ignore_errors)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("conflicting or redundant options"),
+						 parser_errposition(pstate, defel->location)));
+			cstate->ignore_errors = defGetBoolean(defel);
+		}
 		else
 			ereport(ERROR,
 					(errcode(ERRCODE_SYNTAX_ERROR),
@@ -3006,7 +3015,6 @@ BeginCopyFrom(ParseState *pstate,
 	cstate->cur_lineno = 0;
 	cstate->cur_attname = NULL;
 	cstate->cur_attval = NULL;
-    cstate->ignore_errors = true;
 
 	/* Set up variables to avoid per-attribute overhead. */
 	initStringInfo(&cstate->attribute_buf);
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 0f3998ff89..83ff1ab72b 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -636,7 +636,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P
 	INCLUDING INCREMENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
 	INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
-	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
+	INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION IGNORE_ERRORS
 
 	JOIN
 
@@ -2977,6 +2977,10 @@ copy_opt_item:
 				{
 					$$ = makeDefElem("encoding", (Node *)makeString($2), @1);
 				}
+			| IGNORE_ERRORS
+				{
+					$$ = makeDefElem("ignore_errors", (Node *)makeInteger(TRUE), @1);
+				}
 		;
 
 /* The following exist for backward compatibility with very old versions */
-- 
2.11.0


From 659aec15f7fcc391daa084f43dc1e1a66bdbe48e Mon Sep 17 00:00:00 2001
From: Alex K <alex.lumir@gmail.com>
Date: Wed, 21 Jun 2017 14:32:44 +0300
Subject: [PATCH 8/8] Regression test updated to handle new COPY functionality

---
 src/test/regress/expected/alter_table.out |  2 +-
 src/test/regress/expected/copy2.out       | 35 +++++++++++++++++++++++++++----
 src/test/regress/sql/copy2.sql            | 23 ++++++++++++++++++++
 3 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out
index 13d6a4b747..078ab27359 100644
--- a/src/test/regress/expected/alter_table.out
+++ b/src/test/regress/expected/alter_table.out
@@ -1431,7 +1431,7 @@ ERROR:  column "a" of relation "test" does not exist
 copy test("........pg.dropped.1........") to stdout;
 ERROR:  column "........pg.dropped.1........" of relation "test" does not exist
 copy test from stdin;
-ERROR:  extra data after last expected column
+ERROR:  extra data after last expected column at line 1
 CONTEXT:  COPY test, line 1: "10	11	12"
 select * from test;
  b | c 
diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out
index 65e9c626b3..ab4d47b351 100644
--- a/src/test/regress/expected/copy2.out
+++ b/src/test/regress/expected/copy2.out
@@ -36,15 +36,26 @@ COPY x from stdin;
 ERROR:  invalid input syntax for integer: ""
 CONTEXT:  COPY x, line 1, column a: ""
 COPY x from stdin;
-ERROR:  missing data for column "e"
+ERROR:  missing data for column "e" at line 1
 CONTEXT:  COPY x, line 1: "2000	230	23	23"
 COPY x from stdin;
-ERROR:  missing data for column "e"
+ERROR:  missing data for column "e" at line 1
 CONTEXT:  COPY x, line 1: "2001	231	\N	\N"
 -- extra data: should fail
 COPY x from stdin;
-ERROR:  extra data after last expected column
+ERROR:  extra data after last expected column at line 1
 CONTEXT:  COPY x, line 1: "2002	232	40	50	60	70	80"
+-- missing data: should be able to ignore errors
+COPY x from stdin with (ignore_errors);
+WARNING:  missing data for column "e" at line 1
+COPY x from stdin with (ignore_errors);
+WARNING:  missing data for column "e" at line 1
+-- extra data: should be able to ignore errors
+COPY x from stdin with (ignore_errors);
+WARNING:  extra data after last expected column at line 1
+-- data type inappropriate format: should be able to ignore errors
+COPY x from stdin with (ignore_errors);
+WARNING:  invalid input syntax for integer: "'24'" at line 2 col 2
 -- various COPY options: delimiters, oids, NULL string, encoding
 COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
 COPY x from stdin WITH DELIMITER AS ';' NULL AS '';
@@ -60,6 +71,10 @@ SELECT * FROM x;
  10003 | 24 | 34         | 44     | before trigger fired
  10004 | 25 | 35         | 45     | before trigger fired
  10005 | 26 | 36         | 46     | before trigger fired
+  2004 | 27 | 37         | 47     | before trigger fired
+  2006 | 28 | 38         | 48     | before trigger fired
+  2008 | 29 | 39         | 49     | before trigger fired
+  2009 | 30 | 40         | 50     | before trigger fired
      6 |    | 45         | 80     | before trigger fired
      7 |    | x          | \x     | before trigger fired
      8 |    | ,          | \,     | before trigger fired
@@ -78,7 +93,7 @@ SELECT * FROM x;
      3 |  3 | stuff      | test_3 | after trigger fired
      4 |  4 | stuff      | test_4 | after trigger fired
      5 |  5 | stuff      | test_5 | after trigger fired
-(25 rows)
+(29 rows)
 
 -- COPY w/ oids on a table w/o oids should fail
 CREATE TABLE no_oids (
@@ -101,6 +116,10 @@ COPY x TO stdout;
 10003	24	34	44	before trigger fired
 10004	25	35	45	before trigger fired
 10005	26	36	46	before trigger fired
+2004	27	37	47	before trigger fired
+2006	28	38	48	before trigger fired
+2008	29	39	49	before trigger fired
+2009	30	40	50	before trigger fired
 6	\N	45	80	before trigger fired
 7	\N	x	\\x	before trigger fired
 8	\N	,	\\,	before trigger fired
@@ -127,6 +146,10 @@ COPY x (c, e) TO stdout;
 34	before trigger fired
 35	before trigger fired
 36	before trigger fired
+37	before trigger fired
+38	before trigger fired
+39	before trigger fired
+40	before trigger fired
 45	before trigger fired
 x	before trigger fired
 ,	before trigger fired
@@ -153,6 +176,10 @@ I'm null	before trigger fired
 24	before trigger fired
 25	before trigger fired
 26	before trigger fired
+27	before trigger fired
+28	before trigger fired
+29	before trigger fired
+30	before trigger fired
 I'm null	before trigger fired
 I'm null	before trigger fired
 I'm null	before trigger fired
diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql
index f3a6d228fa..692f4d2f29 100644
--- a/src/test/regress/sql/copy2.sql
+++ b/src/test/regress/sql/copy2.sql
@@ -72,6 +72,29 @@ COPY x from stdin;
 2002	232	40	50	60	70	80
 \.
 
+-- missing data: should be able to ignore errors
+COPY x from stdin with (ignore_errors);
+2003	230	23	23
+2004	27	37	47	57
+\.
+COPY x from stdin with (ignore_errors);
+2005	231	\N	\N
+2006	28	38	48	58
+\.
+
+-- extra data: should be able to ignore errors
+COPY x from stdin with (ignore_errors);
+2007	232	40	50	60	70	80
+2008	29	39	49	59
+\.
+
+-- data type inappropriate format: should be able to ignore errors
+
+COPY x from stdin with (ignore_errors);
+2009	30	40	50	60
+2010	'24'	34	44	54
+\.
+
 -- various COPY options: delimiters, oids, NULL string, encoding
 COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x';
 500000,x,45,80,90
-- 
2.11.0