Further patch rangetypes_selfuncs.c's statistics slot management.

Values in a STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM slot are float8,
not of the type of the column the statistics are for.

This bug is at least partly the fault of sloppy specification comments
for get_attstatsslot()/free_attstatsslot(): the type OID they want is that
of the stavalues entries, not of the underlying column.  (I double-checked
other callers and they seem to get this right.)  Adjust the comments to be
more correct.

Per buildfarm.

Security: CVE-2017-7484

Author: tglsfdc

src/backend/utils/adt/rangetypes_selfuncs.c

@@ -20,6 +20,7 @@
 #include "access/htup_details.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_statistic.h"
+#include "catalog/pg_type.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 #include "utils/rangetypes.h"
@@ -246,16 +247,17 @@ calc_rangesel(TypeCacheEntry *typcache, VariableStatData *vardata,
 
 		/* Try to get fraction of empty ranges */
 		if (get_attstatsslot(vardata->statsTuple,
-							 vardata->atttype, vardata->atttypmod,
-						   STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM, InvalidOid,
+							 FLOAT8OID, -1,
+							 STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM,
+							 InvalidOid,
 							 NULL,
 							 NULL, NULL,
 							 &numbers, &nnumbers))
 		{
 			if (nnumbers != 1)
 				elog(ERROR, "invalid empty fraction statistic");		/* shouldn't happen */
 			empty_frac = numbers[0];
-			free_attstatsslot(vardata->atttype, NULL, 0, numbers, nnumbers);
+			free_attstatsslot(FLOAT8OID, NULL, 0, numbers, nnumbers);
 		}
 		else
 		{
@@ -424,7 +426,7 @@ calc_hist_selectivity(TypeCacheEntry *typcache, VariableStatData *vardata,
 	{
 		if (!(HeapTupleIsValid(vardata->statsTuple) &&
 			  get_attstatsslot(vardata->statsTuple,
-							   vardata->atttype, vardata->atttypmod,
+							   FLOAT8OID, -1,
 							   STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM,
 							   InvalidOid,
 							   NULL,
@@ -438,7 +440,7 @@ calc_hist_selectivity(TypeCacheEntry *typcache, VariableStatData *vardata,
 		/* check that it's a histogram, not just a dummy entry */
 		if (length_nhist < 2)
 		{
-			free_attstatsslot(vardata->atttype,
+			free_attstatsslot(FLOAT8OID,
 							  length_hist_values, length_nhist, NULL, 0);
 			free_attstatsslot(vardata->atttype, hist_values, nhist, NULL, 0);
 			return -1.0;
@@ -578,7 +580,7 @@ calc_hist_selectivity(TypeCacheEntry *typcache, VariableStatData *vardata,
 			break;
 	}
 
-	free_attstatsslot(vardata->atttype,
+	free_attstatsslot(FLOAT8OID,
 					  length_hist_values, length_nhist, NULL, 0);
 	free_attstatsslot(vardata->atttype, hist_values, nhist, NULL, 0);
 

src/backend/utils/cache/lsyscache.c

@@ -2865,16 +2865,16 @@ get_attavgwidth(Oid relid, AttrNumber attnum)
  * pg_statistic.
  *
  * statstuple: pg_statistic tuple to be examined.
- * atttype: type OID of attribute (can be InvalidOid if values == NULL).
- * atttypmod: typmod of attribute (can be 0 if values == NULL).
+ * atttype: type OID of slot's stavalues (can be InvalidOid if values == NULL).
+ * atttypmod: typmod of slot's stavalues (can be 0 if values == NULL).
  * reqkind: STAKIND code for desired statistics slot kind.
  * reqop: STAOP value wanted, or InvalidOid if don't care.
  * actualop: if not NULL, *actualop receives the actual STAOP value.
  * values, nvalues: if not NULL, the slot's stavalues are extracted.
  * numbers, nnumbers: if not NULL, the slot's stanumbers are extracted.
  *
  * If assigned, values and numbers are set to point to palloc'd arrays.
- * If the attribute type is pass-by-reference, the values referenced by
+ * If the stavalues datatype is pass-by-reference, the values referenced by
  * the values array are themselves palloc'd.  The palloc'd stuff can be
  * freed by calling free_attstatsslot.
  *
@@ -3004,7 +3004,8 @@ get_attstatsslot(HeapTuple statstuple,
  * free_attstatsslot
  *		Free data allocated by get_attstatsslot
  *
- * atttype need be valid only if values != NULL.
+ * atttype is the type of the individual values in values[].
+ * It need be valid only if values != NULL.
  */
 void
 free_attstatsslot(Oid atttype,

src/include/catalog/pg_statistic.h

@@ -275,7 +275,7 @@ typedef FormData_pg_statistic *Form_pg_statistic;
  * fraction of empty ranges. stavalues is a histogram of non-empty lengths, in
  * a format similar to STATISTIC_KIND_HISTOGRAM: it contains M (>=2) range
  * values that divide the column data values into M-1 bins of approximately
- * equal population. The lengths are stores as float8s, as measured by the
+ * equal population. The lengths are stored as float8s, as measured by the
  * range type's subdiff function. Only non-null rows are considered.
  */
 #define STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM  6