Query time is improved, can now filter by score, system now predicts ratings for items already rated

Author: Jim Avery

PostgreSQL/recdb_regression_test.sql

@@ -0,0 +1,38 @@
+/* ItemCosCF. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING itemcoscf;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itemcoscf WHERE userid = 1;
+DROP RECOMMENDER MovieRec;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itemcoscf WHERE userid = 1;
+
+/* ItemPearCF. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING itempearcf;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itempearcf WHERE userid = 1;
+DROP RECOMMENDER MovieRec;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itempearcf WHERE userid = 1;
+
+/* UserCosCF. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING usercoscf;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING usercoscf WHERE userid = 1;
+DROP RECOMMENDER MovieRec;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING usercoscf WHERE userid = 1;
+
+/* UserPearCF. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING userpearcf;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING userpearcf WHERE userid = 1;
+DROP RECOMMENDER MovieRec;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING userpearcf WHERE userid = 1;
+
+/* SVD. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING svd;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING svd WHERE userid = 1;
+DROP RECOMMENDER MovieRec;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING svd WHERE userid = 1;
+
+/* Miscellaneous. */
+CREATE RECOMMENDER MovieRec ON ml_ratings USERS FROM userid ITEMS FROM itemid EVENTS FROM ratingval USING itemcoscf;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itemcoscf WHERE userid IN (1,2,3,5,9) AND itemid < 7;
+SELECT r.itemid,r.ratingval,i.name,i.genre FROM ml_ratings r, ml_items i RECOMMEND r.itemid TO r.userid ON r.ratingval USING itemcoscf WHERE r.userid = 1 AND r.itemid = i.itemid AND i.genre ILIKE '%drama%';
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itemcoscf WHERE userid = 1 ORDER BY ratingval DESC LIMIT 10;
+SELECT r.itemid,r.ratingval,i.name,i.genre FROM ml_ratings r, ml_items i RECOMMEND r.itemid TO r.userid ON r.ratingval USING itemcoscf WHERE r.userid = 1 AND r.itemid = i.itemid AND i.genre ILIKE '%action%' ORDER BY ratingval DESC LIMIT 5;
+SELECT * FROM ml_ratings RECOMMEND itemid TO userid ON ratingval USING itemcoscf WHERE userid = 1 AND ratingval >= 4.5;
+DROP RECOMMENDER MovieRec;

PostgreSQL/src/backend/executor/execRecommend.c

@@ -305,7 +305,7 @@ ExecFilterRecommend(RecScanState *recnode,
 					 ExecScanRecheckMtd recheckMtd)
 {
 	ExprContext *econtext;
-	List	   *qual;
+	List	   *qual, *userqual;
 	ProjectionInfo *projInfo;
 	ExprDoneCond isDone;
 	TupleTableSlot *resultSlot;
@@ -319,6 +319,7 @@ ExecFilterRecommend(RecScanState *recnode,
 	 * Fetch data from node
 	 */
 	qual = node->ps.qual;
+	userqual = recnode->userqual;
 	projInfo = node->ps.ps_ProjInfo;
 	econtext = node->ps.ps_ExprContext;
 
@@ -373,7 +374,7 @@ ExecFilterRecommend(RecScanState *recnode,
 		if (recnode->finished) {
 			recnode->finished = false;
 			recnode->userNum = 0;
-			recnode->itemNum = 0;
+			recnode->fullItemNum = 0;
 			return NULL;
 		}
 
@@ -434,51 +435,36 @@ ExecFilterRecommend(RecScanState *recnode,
 
 		/*
 		 * We now have a blank tuple slot that we need to fill with data.
-		 * We have a working user ID, but not a valid item list. We'd like to
-		 * use the filter to determine if this is a good user, but we can't
-		 * do that without an item, in many cases. The solution is to add in
-		 * dummy items, then compare it against the filter. If a given user ID
-		 * doesn't make it past the filter with any item ID, then that user is
-		 * being filtered out, and we'll move on to the next.
+		 * We have a working user ID, but not a valid item list. This is where
+		 * we use our custom user-focused WHERE clause to filter out users that
+		 * do not pass our qualifications.
 		 */
 		if (recnode->newUser) {
-			recnode->fullItemNum = 0;
-			itemindex = recnode->fullItemNum;
-			itemID = recnode->fullItemList[itemindex];
-
 			slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
-			slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
+			slot->tts_values[recnode->itematt] = Int32GetDatum(-1);
 			slot->tts_values[recnode->eventatt] = Int32GetDatum(-1);
 
 			/* We have a preliminary slot - let's test it. */
-			while (qual && !ExecQual(qual, econtext, false)) {
-				/* We failed the test. Try the next item. */
-				recnode->fullItemNum++;
-				if (recnode->fullItemNum >= recnode->fullTotalItems) {
-					/* If we've reached the last item, move onto the next user.
-					 * If we've reached the last user, we're done. */
-					InstrCountFiltered1(node, recnode->fullTotalItems);
-					recnode->userNum++;
-					recnode->newUser = true;
+			while (userqual && !ExecQual(userqual, econtext, false)) {
+				/* We failed the test. Try the next user.
+				 * If we've reached the last user, we're done. */
+				InstrCountFiltered1(node, recnode->fullTotalItems);
+				recnode->userNum++;
+				recnode->newUser = true;
+
+				if (recnode->userNum >= recnode->totalUsers) {
+					recnode->userNum = 0;
 					recnode->fullItemNum = 0;
-					if (recnode->userNum >= recnode->totalUsers) {
-						recnode->userNum = 0;
-						recnode->itemNum = 0;
-						return NULL;
-					}
-					userindex = recnode->userNum;
-					userID = recnode->userList[userindex];
+					return NULL;
 				}
 
-				itemindex = recnode->fullItemNum;
-				itemID = recnode->fullItemList[itemindex];
+				userindex = recnode->userNum;
+				userID = recnode->userList[userindex];
 				slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
-				slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
 			}
 
 			/* If we get here, then we found a user who will be actually
-			 * returned in the results. One quick reset here. */
-			recnode->fullItemNum = 0;
+			 * returned in the results. */
 		}
 
 		/* Mark the user ID and index. */
@@ -495,14 +481,26 @@ ExecFilterRecommend(RecScanState *recnode,
 		}
 
 		/* Now replace the item ID, if the user is valid. Otherwise,
-		 * leave the item ID as is, as it doesn't matter what it is. */
-		if (recnode->validUser)
+		 * leave the item ID as is, as it doesn't matter what it is. We'll
+		 * move on to the next user, as well. */
+		if (recnode->validUser) {
+			itemID = recnode->fullItemList[recnode->fullItemNum];
+			itemindex = recnode->fullItemNum;
+		} else {
+			recnode->userNum++;
+			recnode->newUser = true;
+			recnode->fullItemNum = 0;
+			if (recnode->userNum >= recnode->totalUsers)
+				recnode->finished = true;
+			continue;
+		}
+/*		if (recnode->validUser)
 			itemID = recnode->itemList[recnode->itemNum];
 		while (recnode->fullItemList[recnode->fullItemNum] < itemID)
 			recnode->fullItemNum++;
 		itemindex = recnode->fullItemNum;
 		if (recnode->fullItemList[itemindex] > itemID)
-			elog(ERROR, "critical item mismatch in ExecRecommend");
+			elog(ERROR, "critical item mismatch in ExecRecommend");*/
 
 		/* Plug in the data, marking those columns full. We also need to
 		 * mark the rating column with something temporary. */
@@ -514,21 +512,20 @@ ExecFilterRecommend(RecScanState *recnode,
 		 * If that's the case, we need to calculate it before we do the
 		 * qual filtering. Also, if we're doing a JoinRecommend, we should
 		 * not calculate the RecScore in this node. In the current version
-		 * of RecDB, an OP_NOFILTER shouldn't be allowed. */
-		if (attributes->opType == OP_NOFILTER)
+		 * of RecDB, special joins don't exist, so that's no problem. */
+		if (attributes->noFilter)
 			applyRecScore(recnode, slot, itemID, itemindex);
 
 		/* Move onto the next item, for next time. If we're doing a RecJoin,
 		 * though, we'll move onto the next user instead. */
-		recnode->itemNum++;
-		if (recnode->itemNum >= recnode->totalItems ||
+		recnode->fullItemNum++;
+		if (recnode->fullItemNum >= recnode->fullTotalItems ||
 			attributes->opType == OP_JOIN ||
 			attributes->opType == OP_GENERATEJOIN) {
 			/* If we've reached the last item, move onto the next user.
 			 * If we've reached the last user, we're done. */
 			recnode->userNum++;
 			recnode->newUser = true;
-			recnode->itemNum = 0;
 			recnode->fullItemNum = 0;
 			if (recnode->userNum >= recnode->totalUsers)
 				recnode->finished = true;
@@ -558,7 +555,7 @@ ExecFilterRecommend(RecScanState *recnode,
 			 * Found a satisfactory scan tuple. This is usually when
 			 * we will calculate and apply the RecScore.
 			 */
-			if (attributes->opType == OP_FILTER || attributes->opType == OP_GENERATE)
+			if (!attributes->noFilter)
 				applyRecScore(recnode, slot, itemID, itemindex);
 
 			if (projInfo)
@@ -701,18 +698,16 @@ InitializeRecommender(RecScanState *recstate) {
 		recstate->totalUsers = getTupleInt(hslot,"count");
 		recathon_queryEnd(queryDesc,recathoncontext);
 
-		/* In the event that there are no user IDs, our ratings table is empty, so
-		 * we can't do anything. */
+		/* In the event that there are no user IDs, we can't do anything. */
 		if (recstate->totalUsers <= 0)
-			elog(ERROR, "no ratings in table %s, cannot predict ratings",
-				attributes->eventtable);
+			elog(ERROR, "no users found, cannot predict ratings");
 
 		recstate->userList = (int*) palloc(recstate->totalUsers*sizeof(int));
 		recstate->userNum = 0;
 
 		/* Now for the actual query. */
-		sprintf(querystring,"select distinct %s from %s order by %s;",
-			attributes->userkey,attributes->eventtable,attributes->userkey);
+		sprintf(querystring,"select distinct %s from %s;",
+			attributes->userkey,attributes->eventtable);
 		queryDesc = recathon_queryStart(querystring,&recathoncontext);
 		planstate = queryDesc->planstate;
 
@@ -734,16 +729,14 @@ InitializeRecommender(RecScanState *recstate) {
 		/* Quick error protection. */
 		recstate->totalUsers = i;
 		if (recstate->totalUsers <= 0)
-			elog(ERROR, "no ratings in table %s, cannot predict ratings",
-				attributes->eventtable);
+			elog(ERROR, "no users found, cannot predict ratings");
 
 		/* Lastly, initialize the attributes->userID. */
 		attributes->userID = recstate->userList[0] - 1;
 	}
 
-	/* Next, for annoying and convoluted reasons, we need a full list of all the
-	 * items in the rating table. This will help us circumvent some filter issues
-	 * while remaining as efficient as we can manage. */
+	/* Next, we need a full list of all the items in the rating table. This will tell
+	 * us what items to generate ratings for. */
 	if ((attributes->opType != OP_GENERATE && attributes->opType != OP_GENERATEJOIN) ||
 	    attributes->method == userCosCF ||
 	    attributes->method == userPearCF) {
@@ -755,11 +748,9 @@ InitializeRecommender(RecScanState *recstate) {
 		recstate->fullTotalItems = getTupleInt(hslot,"count");
 		recathon_queryEnd(queryDesc,recathoncontext);
 
-		/* In the event that there are no item IDs, our ratings table is empty, so
-		 * we can't do anything. */
+		/* In the event that there are no item IDs, we can't do anything. */
 		if (recstate->fullTotalItems <= 0)
-			elog(ERROR, "no ratings in table %s, cannot predict ratings",
-				attributes->eventtable);
+			elog(ERROR, "no items found, cannot predict ratings");
 
 		recstate->fullItemList = (int*) palloc(recstate->fullTotalItems*sizeof(int));
 		recstate->fullItemNum = 0;
@@ -788,8 +779,7 @@ InitializeRecommender(RecScanState *recstate) {
 		/* Quick error protection. */
 		recstate->fullTotalItems = i;
 		if (recstate->fullTotalItems <= 0)
-			elog(ERROR, "no ratings in table %s, cannot predict ratings",
-				attributes->eventtable);
+			elog(ERROR, "no items found, cannot predict ratings");
 	}
 
 	recstate->finished = false;
@@ -803,7 +793,6 @@ InitializeRecommender(RecScanState *recstate) {
 	recstate->ratedTable = NULL;
 	recstate->pendingTable = NULL;
 	recstate->simTable = NULL;
-	recstate->itemList = NULL;
 	recstate->userFeatures = NULL;
 
 	/* In case we don't have a pre-built recommender, we need to assemble
@@ -903,6 +892,10 @@ ExecInitRecScan(RecScan *node, EState *estate, int eflags)
 	 * stuff out of Init and into Execute, to make EXPLAIN go faster. */
 	recstate->initialized = false;
 
+	/* Next we need to prep our user WHERE clause. */
+	recstate->userqual = (List *)
+		ExecInitExpr((Expr *) attributes->userWhereClause, NULL);
+
 	/* Code for a future version of RecDB. */
 /*	switch(attributes->cellType) {
 		case CELL_ALPHA:
@@ -992,8 +985,6 @@ ExecEndRecScan(RecScanState *node)
 	}
 
 	/* Now for extra stuff. */
-	if (node->itemList)
-		pfree(node->itemList);
 	if (node->fullItemList)
 		pfree(node->fullItemList);
 	if (node->userFeatures)

PostgreSQL/src/backend/executor/nodeRecjoin.c

@@ -152,9 +152,9 @@ ExecRecJoin(RecJoinState *recjoin)
 			/* Otherwise, we need to construct our hash table, since
 			 * we need info from the previous operator to do so. */
 			freeHash(recjoin->itemTable);
-			recjoin->itemTable = hashCreate(recjoin->recnode->totalItems);
-			for (i = 0; i < recjoin->recnode->totalItems; i++) {
-				int currentItem = recjoin->recnode->itemList[i];
+			recjoin->itemTable = hashCreate(recjoin->recnode->fullTotalItems);
+			for (i = 0; i < recjoin->recnode->fullTotalItems; i++) {
+				int currentItem = recjoin->recnode->fullItemList[i];
 
 				tempItem = (GenRating*) palloc(sizeof(GenRating));
 				tempItem->ID = currentItem;

PostgreSQL/src/backend/nodes/copyfuncs.c

@@ -1016,6 +1016,7 @@ _copyRangeVar(const RangeVar *from)
 	COPY_SCALAR_FIELD(relpersistence);
 	COPY_NODE_FIELD(alias);
 	COPY_LOCATION_FIELD(location);
+	COPY_NODE_FIELD(recommender);
 
 	return newnode;
 }
@@ -2422,7 +2423,7 @@ _copyQuery(const Query *from)
 	COPY_SCALAR_FIELD(hasRecursive);
 	COPY_SCALAR_FIELD(hasModifyingCTE);
 	COPY_SCALAR_FIELD(hasForUpdate);
-	COPY_SCALAR_FIELD(isRecommendStmt);
+	COPY_NODE_FIELD(recommendStmt);
 	COPY_NODE_FIELD(cteList);
 	COPY_NODE_FIELD(rtable);
 	COPY_NODE_FIELD(jointree);
@@ -2525,7 +2526,6 @@ _copyRecommendInfo(const RecommendInfo *from)
 	COPY_NODE_FIELD(recommender);
 	COPY_NODE_FIELD(attributes);
 	COPY_SCALAR_FIELD(opType);
-	COPY_NODE_FIELD(next);
 
 	return newnode;
 }
@@ -2548,13 +2548,11 @@ _copyAttributeInfo(const AttributeInfo *from)
 	COPY_STRING_FIELD(recModelName);
 	COPY_STRING_FIELD(recModelName2);
 	COPY_STRING_FIELD(recViewName);
-	COPY_SCALAR_FIELD(numAtts);
-	COPY_POINTER_FIELD(attNames, from->numAtts*sizeof(char*));
-	COPY_POINTER_FIELD(attValues, from->numAtts*sizeof(char*));
-	COPY_NODE_FIELD(target_val);
+	COPY_NODE_FIELD(userWhereClause);
 	COPY_SCALAR_FIELD(IDfound);
 	COPY_SCALAR_FIELD(cellType);
 	COPY_SCALAR_FIELD(opType);
+	COPY_SCALAR_FIELD(noFilter);
 
 	return newnode;
 }

PostgreSQL/src/backend/nodes/equalfuncs.c

@@ -107,6 +107,7 @@ _equalRangeVar(const RangeVar *a, const RangeVar *b)
 	COMPARE_SCALAR_FIELD(relpersistence);
 	COMPARE_NODE_FIELD(alias);
 	COMPARE_LOCATION_FIELD(location);
+	COMPARE_NODE_FIELD(recommender);
 
 	return true;
 }
@@ -909,7 +910,7 @@ _equalQuery(const Query *a, const Query *b)
 	COMPARE_SCALAR_FIELD(hasRecursive);
 	COMPARE_SCALAR_FIELD(hasModifyingCTE);
 	COMPARE_SCALAR_FIELD(hasForUpdate);
-	COMPARE_SCALAR_FIELD(isRecommendStmt);
+	COMPARE_NODE_FIELD(recommendStmt);
 	COMPARE_NODE_FIELD(cteList);
 	COMPARE_NODE_FIELD(rtable);
 	COMPARE_NODE_FIELD(jointree);
@@ -1002,7 +1003,6 @@ _equalRecommendInfo(const RecommendInfo *a, const RecommendInfo *b)
 	COMPARE_NODE_FIELD(recommender);
 	COMPARE_NODE_FIELD(attributes);
 	COMPARE_SCALAR_FIELD(opType);
-	COMPARE_NODE_FIELD(next);
 
 	return true;
 }
@@ -1023,13 +1023,11 @@ _equalAttributeInfo(const AttributeInfo *a, const AttributeInfo *b)
 	COMPARE_STRING_FIELD(recModelName);
 	COMPARE_STRING_FIELD(recModelName2);
 	COMPARE_STRING_FIELD(recViewName);
-	COMPARE_SCALAR_FIELD(numAtts);
-	COMPARE_POINTER_FIELD(attNames, a->numAtts*sizeof(char*));
-	COMPARE_POINTER_FIELD(attValues, a->numAtts*sizeof(char*));
-	COMPARE_NODE_FIELD(target_val);
+	COMPARE_NODE_FIELD(userWhereClause);
 	COMPARE_SCALAR_FIELD(IDfound);
 	COMPARE_SCALAR_FIELD(cellType);
 	COMPARE_SCALAR_FIELD(opType);
+	COMPARE_SCALAR_FIELD(noFilter);
 
 	return true;
 }