Fix rtree and contrib/rtree_gist search behavior for the 1-D box and

polygon operators (<<, &<, >>, &>).  Per ideas originally put forward
by andrew@supernews and later rediscovered by moi.  This patch just
fixes the existing opclasses, and does not add any new behavior as I
proposed earlier; that can be sorted out later.  In principle this
could be back-patched, since it changes only search behavior and not
system catalog entries nor rtree index contents.  I'm not currently
planning to do that, though, since I think it could use more testing.

6 files changed, 69 insertions, 27 deletions

diff --git a/contrib/rtree_gist/rtree_gist.c b/contrib/rtree_gist/rtree_gist.c
index 0b3588351f..53f673856c 100644
--- a/contrib/rtree_gist/rtree_gist.c
+++ b/contrib/rtree_gist/rtree_gist.c
@@ -2,8 +2,8 @@
  *
  * rtree_gist.c
  *	  pg_amproc entries for GiSTs over 2-D boxes.
- * This gives R-tree behavior, with Guttman's poly-time split algorithm.
  *
+ * This gives R-tree behavior, with Guttman's poly-time split algorithm.
  *
  *
  * IDENTIFICATION
@@ -78,7 +78,7 @@ gbox_consistent(PG_FUNCTION_ARGS)
 	StrategyNumber strategy = (StrategyNumber) PG_GETARG_UINT16(2);
 
 	/*
-	 * * if entry is not leaf, use gbox_internal_consistent, * else use
+	 * if entry is not leaf, use rtree_internal_consistent, else use
 	 * gbox_leaf_consistent
 	 */
 	if (!(DatumGetPointer(entry->key) != NULL && query))
@@ -509,8 +509,9 @@ gpoly_consistent(PG_FUNCTION_ARGS)
 	bool		result;
 
 	/*
-	 * * if entry is not leaf, use gbox_internal_consistent, * else use
-	 * gbox_leaf_consistent
+	 * Since the operators are marked lossy anyway, we can just use
+	 * rtree_internal_consistent even at leaf nodes.  (This works
+	 * in part because the index entries are bounding Boxes not polygons.)
 	 */
 	if (!(DatumGetPointer(entry->key) != NULL && query))
 		PG_RETURN_BOOL(FALSE);
@@ -536,15 +537,19 @@ rtree_internal_consistent(BOX *key,
 	switch (strategy)
 	{
 		case RTLeftStrategyNumber:
+			retval = !DatumGetBool(DirectFunctionCall2(box_overright, PointerGetDatum(key), PointerGetDatum(query)));
+			break;
 		case RTOverLeftStrategyNumber:
-			retval = DatumGetBool(DirectFunctionCall2(box_overleft, PointerGetDatum(key), PointerGetDatum(query)));
+			retval = !DatumGetBool(DirectFunctionCall2(box_right, PointerGetDatum(key), PointerGetDatum(query)));
 			break;
 		case RTOverlapStrategyNumber:
 			retval = DatumGetBool(DirectFunctionCall2(box_overlap, PointerGetDatum(key), PointerGetDatum(query)));
 			break;
 		case RTOverRightStrategyNumber:
+			retval = !DatumGetBool(DirectFunctionCall2(box_left, PointerGetDatum(key), PointerGetDatum(query)));
+			break;
 		case RTRightStrategyNumber:
-			retval = DatumGetBool(DirectFunctionCall2(box_right, PointerGetDatum(key), PointerGetDatum(query)));
+			retval = !DatumGetBool(DirectFunctionCall2(box_overleft, PointerGetDatum(key), PointerGetDatum(query)));
 			break;
 		case RTSameStrategyNumber:
 		case RTContainsStrategyNumber:
diff --git a/src/backend/access/common/indexvalid.c b/src/backend/access/common/indexvalid.c
index aea84c614c..d687deda65 100644
--- a/src/backend/access/common/indexvalid.c
+++ b/src/backend/access/common/indexvalid.c
@@ -59,8 +59,16 @@ index_keytest(IndexTuple tuple,
 
 		test = FunctionCall2(&key->sk_func, datum, key->sk_argument);
 
-		if (!DatumGetBool(test))
-			return false;
+		if (key->sk_flags & SK_NEGATE)
+		{
+			if (DatumGetBool(test))
+				return false;
+		}
+		else
+		{
+			if (!DatumGetBool(test))
+				return false;
+		}
 
 		key++;
 		scanKeySize--;
diff --git a/src/backend/access/rtree/rtscan.c b/src/backend/access/rtree/rtscan.c
index b155fb7cc5..79049b1330 100644
--- a/src/backend/access/rtree/rtscan.c
+++ b/src/backend/access/rtree/rtscan.c
@@ -125,27 +125,36 @@ rtrescan(PG_FUNCTION_ARGS)
 		 * Scans on internal pages use different operators than they do on
 		 * leaf pages.	For example, if the user wants all boxes that
 		 * exactly match (x1,y1,x2,y2), then on internal pages we need to
-		 * find all boxes that contain (x1,y1,x2,y2).
+		 * find all boxes that contain (x1,y1,x2,y2).  rtstrat.c knows
+		 * how to pick the opclass member to use for internal pages.
+		 * In some cases we need to negate the result of the opclass member.
 		 */
 		for (i = 0; i < s->numberOfKeys; i++)
 		{
 			AttrNumber	attno = s->keyData[i].sk_attno;
 			Oid			opclass;
+			Oid			subtype;
+			StrategyNumber orig_strategy;
 			StrategyNumber int_strategy;
 			Oid			int_oper;
 			RegProcedure int_proc;
+			int			int_flags;
 
 			opclass = s->indexRelation->rd_indclass->values[attno - 1];
-			int_strategy = RTMapToInternalOperator(s->keyData[i].sk_strategy);
-			int_oper = get_opclass_member(opclass,
-										  s->keyData[i].sk_subtype,
-										  int_strategy);
+			subtype = s->keyData[i].sk_subtype;
+			orig_strategy = s->keyData[i].sk_strategy;
+			int_strategy = RTMapToInternalOperator(orig_strategy);
+			int_oper = get_opclass_member(opclass, subtype, int_strategy);
+			Assert(OidIsValid(int_oper));
 			int_proc = get_opcode(int_oper);
+			int_flags = s->keyData[i].sk_flags;
+			if (RTMapToInternalNegate(orig_strategy))
+				int_flags |= SK_NEGATE;
 			ScanKeyEntryInitialize(&(p->s_internalKey[i]),
-								   s->keyData[i].sk_flags,
+								   int_flags,
 								   attno,
 								   int_strategy,
-								   s->keyData[i].sk_subtype,
+								   subtype,
 								   int_proc,
 								   s->keyData[i].sk_argument);
 		}
diff --git a/src/backend/access/rtree/rtstrat.c b/src/backend/access/rtree/rtstrat.c
index 84788cb659..82c1add8ff 100644
--- a/src/backend/access/rtree/rtstrat.c
+++ b/src/backend/access/rtree/rtstrat.c
@@ -28,20 +28,31 @@
  *	the leaf we search for equality.
  *
  *	This array maps leaf search operators to the internal search operators.
- *	We assume the normal ordering on operators:
- *
- *		left, left-or-overlap, overlap, right-or-overlap, right, same,
- *		contains, contained-by
  */
 static const StrategyNumber RTOperMap[RTNStrategies] = {
-	RTOverLeftStrategyNumber,
-	RTOverLeftStrategyNumber,
-	RTOverlapStrategyNumber,
-	RTOverRightStrategyNumber,
-	RTOverRightStrategyNumber,
-	RTContainsStrategyNumber,
-	RTContainsStrategyNumber,
-	RTOverlapStrategyNumber
+	RTOverRightStrategyNumber,	/* left */
+	RTRightStrategyNumber,		/* overleft */
+	RTOverlapStrategyNumber,	/* overlap */
+	RTLeftStrategyNumber,		/* overright */
+	RTOverLeftStrategyNumber,	/* right */
+	RTContainsStrategyNumber,	/* same */
+	RTContainsStrategyNumber,	/* contains */
+	RTOverlapStrategyNumber		/* contained-by */
+};
+
+/*
+ * We may need to negate the result of the selected operator.  (This could
+ * be avoided by expanding the set of operators required for an opclass.)
+ */
+static const bool RTNegateMap[RTNStrategies] = {
+	true,						/* left */
+	true,						/* overleft */
+	false,						/* overlap */
+	true,						/* overright */
+	true,						/* right */
+	false,						/* same */
+	false,						/* contains */
+	false						/* contained-by */
 };
 
 
@@ -51,3 +62,10 @@ RTMapToInternalOperator(StrategyNumber strat)
 	Assert(strat > 0 && strat <= RTNStrategies);
 	return RTOperMap[strat - 1];
 }
+
+bool
+RTMapToInternalNegate(StrategyNumber strat)
+{
+	Assert(strat > 0 && strat <= RTNStrategies);
+	return RTNegateMap[strat - 1];
+}
diff --git a/src/include/access/rtree.h b/src/include/access/rtree.h
index c183385579..51f3fade87 100644
--- a/src/include/access/rtree.h
+++ b/src/include/access/rtree.h
@@ -136,5 +136,6 @@ extern void ReleaseResources_rtree(void);
 
 /* rtstrat.c */
 extern StrategyNumber RTMapToInternalOperator(StrategyNumber strat);
+extern bool RTMapToInternalNegate(StrategyNumber strat);
 
 #endif   /* RTREE_H */
diff --git a/src/include/access/skey.h b/src/include/access/skey.h
index b7a1b4e183..4c54f7c177 100644
--- a/src/include/access/skey.h
+++ b/src/include/access/skey.h
@@ -72,6 +72,7 @@ typedef ScanKeyData *ScanKey;
 /* ScanKeyData sk_flags */
 #define SK_ISNULL		0x0001	/* sk_argument is NULL */
 #define SK_UNARY		0x0002	/* unary operator (currently unsupported) */
+#define SK_NEGATE		0x0004	/* must negate the function result */
 
 
 /*