Add reliability factor (rfactor) into interface of learning procedures.

Author: danolivo

aqo.c

@@ -71,21 +71,6 @@ int			auto_tuning_infinite_loop = 8;
 
 /* Machine learning parameters */
 
-/*
- * Defines where we do not perform learning procedure
- */
-const double	object_selection_prediction_threshold = 0.3;
-
-/*
- * This parameter tell us that the new learning sample object has very small
- * distance from one whose features stored in matrix already.
- * In this case we will not to add new line in matrix, but will modify this
- * nearest neighbor features and cardinality with linear smoothing by
- * learning_rate coefficient.
- */
-const double	object_selection_threshold = 0.1;
-const double	learning_rate = 1e-1;
-
 /* The number of nearest neighbors which will be chosen for ML-operations */
 int			aqo_k = 3;
 double		log_selectivity_lower_bound = -30;

aqo.h

@@ -144,6 +144,7 @@
 #include "utils/fmgroids.h"
 #include "utils/snapmgr.h"
 
+#include "machine_learning.h"
 
 /* Check PostgreSQL version (9.6.0 contains important changes in planner) */
 #if PG_VERSION_NUM < 90600
@@ -237,12 +238,6 @@ extern double auto_tuning_convergence_error;
 
 /* Machine learning parameters */
 
-/* Max number of matrix rows - max number of possible neighbors. */
-#define	aqo_K	(30)
-
-extern const double object_selection_prediction_threshold;
-extern const double object_selection_threshold;
-extern const double learning_rate;
 extern int	aqo_k;
 extern double log_selectivity_lower_bound;
 
@@ -285,17 +280,13 @@ extern bool find_query(uint64 qhash, QueryContextData *ctx);
 extern bool update_query(uint64 qhash, uint64 fhash,
 						 bool learn_aqo, bool use_aqo, bool auto_tuning);
 extern bool add_query_text(uint64 query_hash, const char *query_string);
-extern bool load_fss_ext(uint64 fs, int fss,
-						 int ncols, double **matrix, double *targets, int *rows,
+extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data,
 						 List **relids, bool isSafe);
-extern bool load_fss(uint64 fhash, int fss_hash,
-					 int ncols, double **matrix, double *targets, int *rows,
-					 List **relids);
-extern bool update_fss_ext(uint64 fhash, int fsshash, int nrows, int ncols,
-						   double **matrix, double *targets, List *relids,
-						   bool isTimedOut);
-extern bool update_fss(uint64 fhash, int fss_hash, int nrows, int ncols,
-					   double **matrix, double *targets, List *relids);
+extern bool load_fss(uint64 fhash, int fss_hash, OkNNrdata *data, List **relids);
+extern bool update_fss_ext(uint64 fhash, int fsshash, OkNNrdata *data,
+						   List *relids, bool isTimedOut);
+extern bool update_fss(uint64 fhash, int fss_hash, OkNNrdata *data,
+					   List *relids);
 QueryStat *get_aqo_stat(uint64 query_hash);
 void update_aqo_stat(uint64 query_hash, QueryStat * stat);
 extern bool my_index_insert(Relation indexRelation,	Datum *values, bool *isnull,
@@ -324,14 +315,6 @@ void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
 					 uint64 count, bool execute_once);
 void aqo_ExecutorEnd(QueryDesc *queryDesc);
 
-/* Machine learning techniques */
-extern double OkNNr_predict(int nrows, int ncols,
-							double **matrix, const double *targets,
-							double *features);
-extern int OkNNr_learn(int matrix_rows, int matrix_cols,
-			double **matrix, double *targets,
-			double *features, double target);
-
 /* Automatic query tuning */
 extern void automatical_query_tuning(uint64 query_hash, QueryStat * stat);
 

cardinality_estimation.c

@@ -21,6 +21,7 @@
 
 #include "aqo.h"
 #include "hash.h"
+#include "machine_learning.h"
 
 #ifdef AQO_DEBUG_PRINT
 static void
@@ -59,15 +60,12 @@ predict_debug_output(List *clauses, List *selectivities,
  */
 double
 predict_for_relation(List *clauses, List *selectivities,
-					 List *relids, int *fss_hash)
+					 List *relids, int *fss)
 {
-	int		nfeatures;
-	double	*matrix[aqo_K];
-	double	targets[aqo_K];
-	double	*features;
-	double	result;
-	int		rows;
-	int		i;
+	double	   *features;
+	double		result;
+	int			i;
+	OkNNrdata	data;
 
 	if (relids == NIL)
 		/*
@@ -76,16 +74,15 @@ predict_for_relation(List *clauses, List *selectivities,
 		 */
 		return -4.;
 
-	*fss_hash = get_fss_for_object(relids, clauses,
-								   selectivities, &nfeatures, &features);
+	*fss = get_fss_for_object(relids, clauses,
+								   selectivities, &data.cols, &features);
 
-	if (nfeatures > 0)
+	if (data.cols > 0)
 		for (i = 0; i < aqo_K; ++i)
-			matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
+			data.matrix[i] = palloc0(sizeof(double) * data.cols);
 
-	if (load_fss_ext(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
-					 targets, &rows, NULL, true))
-		result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
+	if (load_fss_ext(query_context.fspace_hash, *fss, &data, NULL, true))
+		result = OkNNr_predict(&data, features);
 	else
 	{
 		/*
@@ -100,10 +97,10 @@ predict_for_relation(List *clauses, List *selectivities,
 	predict_debug_output(clauses, selectivities, relids, *fss_hash, result);
 #endif
 	pfree(features);
-	if (nfeatures > 0)
+	if (data.cols > 0)
 	{
 		for (i = 0; i < aqo_K; ++i)
-			pfree(matrix[i]);
+			pfree(data.matrix[i]);
 	}
 
 	if (result < 0)

cardinality_hooks.c

@@ -30,6 +30,7 @@
 #include "aqo.h"
 #include "cardinality_hooks.h"
 #include "hash.h"
+#include "machine_learning.h"
 #include "path_utils.h"
 
 estimate_num_groups_hook_type prev_estimate_num_groups_hook = NULL;
@@ -137,12 +138,12 @@ default_estimate_num_groups(PlannerInfo *root, List *groupExprs,
 void
 aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
 {
-	double		predicted;
-	Oid			relid;
-	List	   *relids = NIL;
-	List	   *selectivities = NULL;
-	List	*clauses;
-	int fss = 0;
+	double	predicted;
+	Oid		relid;
+	List   *relids = NIL;
+	List   *selectivities = NULL;
+	List   *clauses;
+	int		fss = 0;
 
 	if (IsQueryDisabled())
 		/* Fast path. */
@@ -411,10 +412,9 @@ static double
 predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
 				   int *fss)
 {
-	int child_fss = 0;
-	double prediction;
-	int rows;
-	double target;
+	int			child_fss = 0;
+	double		prediction;
+	OkNNrdata	data;
 
 	if (subpath->parent->predicted_cardinality > 0.)
 		/* A fast path. Here we can use a fss hash of a leaf. */
@@ -431,13 +431,13 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
 	}
 
 	*fss = get_grouped_exprs_hash(child_fss, group_exprs);
+	memset(&data, 0, sizeof(OkNNrdata));
 
-	if (!load_fss_ext(query_context.fspace_hash, *fss, 0, NULL,
-					  &target, &rows, NULL, true))
+	if (!load_fss_ext(query_context.fspace_hash, *fss, &data, NULL, true))
 		return -1;
 
-	Assert(rows == 1);
-	prediction = exp(target);
+	Assert(data.rows == 1);
+	prediction = exp(data.targets[0]);
 	return (prediction <= 0) ? -1 : prediction;
 }
 

learn_cache.c

@@ -60,8 +60,7 @@ lc_init(void)
 }
 
 bool
-lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
-			  double **matrix, double *targets, List *relids)
+lc_update_fss(uint64 fs, int fss, OkNNrdata *data, List *relids)
 {
 	htab_key		key = {fs, fss};
 	htab_entry	   *entry;
@@ -81,15 +80,15 @@ lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
 		list_free(entry->relids);
 	}
 
-	entry->nrows = nrows;
-	entry->ncols = ncols;
+	entry->nrows = data->rows;
+	entry->ncols = data->cols;
 	for (i = 0; i < entry->nrows; ++i)
 	{
-		entry->matrix[i] = palloc(sizeof(double) * ncols);
-		memcpy(entry->matrix[i], matrix[i], sizeof(double) * ncols);
+		entry->matrix[i] = palloc(sizeof(double) * data->cols);
+		memcpy(entry->matrix[i], data->matrix[i], sizeof(double) * data->cols);
 	}
-	entry->targets = palloc(sizeof(double) * nrows);
-	memcpy(entry->targets, targets, sizeof(double) * nrows);
+	entry->targets = palloc(sizeof(double) * data->rows);
+	memcpy(entry->targets, data->targets, sizeof(double) * data->rows);
 	entry->relids = list_copy(relids);
 
 	MemoryContextSwitchTo(memctx);
@@ -116,8 +115,7 @@ lc_has_fss(uint64 fs, int fss)
  * XXX That to do with learning tails, living in the cache?
  */
 bool
-lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
-			double *targets, int *nrows, List **relids)
+lc_load_fss(uint64 fs, int fss, OkNNrdata *data, List **relids)
 {
 	htab_key	key = {fs, fss};
 	htab_entry	*entry;
@@ -134,11 +132,11 @@ lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
 		elog(NOTICE, "[AQO] Load ML data for fs %lu, fss %d from the cache",
 			 fs, fss);
 
-	*nrows = entry->nrows;
-	Assert(entry->ncols == ncols);
+	data->rows = entry->nrows;
+	Assert(entry->ncols == data->cols);
 	for (i = 0; i < entry->nrows; ++i)
-		memcpy(matrix[i], entry->matrix[i], sizeof(double) * ncols);
-	memcpy(targets, entry->targets, sizeof(double) * entry->nrows);
+		memcpy(data->matrix[i], entry->matrix[i], sizeof(double) * data->cols);
+	memcpy(data->targets, entry->targets, sizeof(double) * entry->nrows);
 	if (relids)
 		*relids = list_copy(entry->relids);
 	return true;

learn_cache.h

@@ -3,15 +3,16 @@
 
 #include "nodes/pg_list.h"
 
+#include "machine_learning.h"
+
 extern bool aqo_learn_statement_timeout;
 
 extern void lc_init(void);
-extern bool lc_update_fss(uint64 fhash, int fsshash, int nrows, int ncols,
-							  double **matrix, double *targets, List *relids);
+extern bool lc_update_fss(uint64 fhash, int fsshash, OkNNrdata *data,
+						  List *relids);
 extern bool lc_has_fss(uint64 fhash, int fss);
-extern bool lc_load_fss(uint64 fhash, int fsshash, int ncols,
-							double **matrix, double *targets, int *nrows,
-							List **relids);
+extern bool lc_load_fss(uint64 fhash, int fsshash, OkNNrdata *data,
+						List **relids);
 extern void lc_remove_fss(uint64 fhash, int fss_hash);
 extern void lc_assign_hook(bool newval, void *extra);