[#24123] YSQL: Make a new ANN Access Method that redirects to the bas…

…e table

Summary:
This change introduces infrastructure for creating indexes that are "covered" by the main table much like a primary key index. This is achieved by adding a boolean variable to the `IndexAmRoutine` struct by the name of `yb_amiscoveredbymaintable` to denote whether or not the index is inlined with the underlying table. YB code that has traditionally checked `indisprimary` to avoid doing operations to the main table is replaced by checks to a utility function `YBIsCoveredByMainTable(Relation)` that checks `indisprimary` along with `yb_amiscoveredbymaintable`.

This work is done in preparation for vector indexes that are colocated with an underlying table. This change introduces the index access method `yblocalann` as an example of such an index type. This index AM can only be created on colocated tables for now as there isn't infrastructure to read ordered results from a multi-tablet vector index yet.

**Upgrade/Rollback safety:**
This change moves where `PgVectorReadOptionsPB` is defined and adds a protobuf field to that `vector_column_id` to signify what column is being used as the vector key. This shouldn't affect any user of the previous format. This is also for a use case that is not GA or EA.
Jira: DB-13014

Test Plan: ./yb_build.sh --java-test 'org.yb.pgsql.TestPgRegressThirdPartyExtensionsPgvector'

Reviewers: fizaa, tfoucher

Reviewed By: fizaa

Subscribers: yql, ybase

Differential Revision: https://phorge.dev.yugabyte.com/D38176

src/postgres/src/backend/access/yb_access/yb_scan.c

@@ -647,7 +647,7 @@ ybcFetchNextIndexTuple(YbScanDesc ybScan, ScanDirection dir)
 			 * Return the IndexTuple. If this is a primary key, reorder the
 			 * values first as expected in the index's column order first.
 			 */
-			if (index->rd_index->indisprimary)
+			if (YBIsCoveredByMainTable(index))
 			{
 				Assert(index->rd_index->indnatts <= INDEX_MAX_KEYS);
 
@@ -766,18 +766,19 @@ ybcSetupScanPlan(bool xs_want_itup, YbScanDesc ybScan, YbScanPlan scan_plan)
 	if (index)
 	{
 		ybScan->prepare_params.index_relfilenode_oid =
-			ybScan->prepare_params.fetch_ybctids_only && index->rd_index->indisprimary
-				? YbGetRelfileNodeId(relation)
-				: YbGetRelfileNodeId(index);
+			ybScan->prepare_params.fetch_ybctids_only &&
+				YBIsCoveredByMainTable(index)
+					? YbGetRelfileNodeId(relation)
+					: YbGetRelfileNodeId(index);
 		ybScan->prepare_params.index_only_scan = xs_want_itup;
 		ybScan->prepare_params.use_secondary_index =
-			!index->rd_index->indisprimary ||
+			!YBIsCoveredByMainTable(index) ||
 			(ybScan->prepare_params.fetch_ybctids_only &&
 			 !ybScan->prepare_params.querying_colocated_table);
 	}
 
 	/* Setup descriptors for target and bind. */
-	if (!index || index->rd_index->indisprimary)
+	if (!index || YBIsCoveredByMainTable(index))
 	{
 		/*
 		 * SequentialScan or PrimaryIndexScan or BitmapIndexScan on the primary index
@@ -852,7 +853,7 @@ ybcSetupScanPlan(bool xs_want_itup, YbScanDesc ybScan, YbScanPlan scan_plan)
 			ybScan->target_key_attnums[i] = key->sk_attno;
 			scan_plan->bind_key_attnums[i] = key->sk_attno;
 		}
-		else if (index->rd_index->indisprimary)
+		else if (YBIsCoveredByMainTable(index))
 		{
 			/*
 			 * PrimaryIndex scan: This is a special case in YugaByte. There is no PrimaryIndexTable.
@@ -2394,8 +2395,8 @@ YbCollectHashKeyComponents(YbScanDesc ybScan, YbScanPlan scan_plan,
 {
 	Relation index = ybScan->index;
 	Relation secondary_index =
-		index && !index->rd_index->indisprimary && !is_index_only_scan ? index :
-																		 NULL;
+		index && !YBIsCoveredByMainTable(index) &&
+			!is_index_only_scan ? index : NULL;
 	int idx = -1;
 	while ((idx = bms_next_member(scan_plan->hash_key, idx)) >= 0)
 	{
@@ -2579,7 +2580,7 @@ ybcSetupTargets(YbScanDesc ybScan, YbScanPlan scan_plan, Scan *pg_scan_plan)
 	 */
 	if (all_attrs_required)
 	{
-		if (is_index_only_scan && index->rd_index->indisprimary)
+		if (is_index_only_scan && YBIsCoveredByMainTable(index))
 		{
 			/*
 			 * Special case: For Primary-Key-ONLY-Scan, we select ONLY the
@@ -2632,13 +2633,13 @@ ybcSetupTargets(YbScanDesc ybScan, YbScanPlan scan_plan, Scan *pg_scan_plan)
 	}
 
 	// If we are using a Bitmap Index scan on non-colocated primary keys
-	if (index && index->rd_index->indisprimary &&
+	if (index && YBIsCoveredByMainTable(index) &&
 		ybScan->prepare_params.fetch_ybctids_only &&
 		!ybScan->prepare_params.querying_colocated_table)
 		YbDmlAppendTargetSystem(YBIdxBaseTupleIdAttributeNumber,
 								ybScan->handle);
 
-	if (!is_index_only_scan && index && !index->rd_index->indisprimary)
+	if (!is_index_only_scan && index && !YBIsCoveredByMainTable(index))
 		YbDmlAppendTargetSystem(YBIdxBaseTupleIdAttributeNumber,
 								ybScan->handle);
 
@@ -2745,7 +2746,7 @@ YbDmlAppendTargetsAggregate(List *aggrefs, TupleDesc tupdesc, Relation index,
 	}
 
 	/* Set ybbasectid in case of non-primary secondary index scan. */
-	if (index && !xs_want_itup && !index->rd_index->indisprimary)
+	if (index && !xs_want_itup && !YBIsCoveredByMainTable(index))
 		YbDmlAppendTargetSystem(YBIdxBaseTupleIdAttributeNumber,
 								handle);
 }
@@ -3502,7 +3503,7 @@ void ybcIndexCostEstimate(struct PlannerInfo *root, IndexPath *path,
 							Cost *total_cost)
 {
 	Relation	index = RelationIdGetRelation(path->indexinfo->indexoid);
-	bool		isprimary = index->rd_index->indisprimary;
+	bool		isprimary = YBIsCoveredByMainTable(index);
 	Relation	relation = isprimary ? RelationIdGetRelation(index->rd_index->indrelid) : NULL;
 	RelOptInfo *baserel = path->path.parent;
 	List	   *qinfos = NIL;
@@ -3515,8 +3516,8 @@ void ybcIndexCostEstimate(struct PlannerInfo *root, IndexPath *path,
 	List	   *clauses = NIL;
 	double 		baserel_rows_estimate;
 
-	/* Primary-index scans are always covered in Yugabyte (internally) */
-	bool       is_uncovered_idx_scan = !index->rd_index->indisprimary &&
+	/* Primary/Inlined-index scans are always covered in Yugabyte (internally) */
+	bool       is_uncovered_idx_scan = !YBIsCoveredByMainTable(index) &&
 	                                   path->path.pathtype != T_IndexOnlyScan;
 
 	YbScanPlanData	scan_plan;

src/postgres/src/backend/catalog/dependency.c

@@ -1179,7 +1179,7 @@ doDeletion(const ObjectAddress *object, int flags)
 
 					Relation index = RelationIdGetRelation(object->objectId);
 
-					if (IsYBRelation(index) && !index->rd_index->indisprimary)
+					if (IsYBRelation(index) && !YBIsCoveredByMainTable(index))
 						YBCDropIndex(index);
 
 					RelationClose(index);

src/postgres/src/backend/catalog/index.c

@@ -1009,10 +1009,13 @@ index_create(Relation heapRelation,
 	Assert(indexRelationId == RelationGetRelid(indexRelation));
 
 	/*
-	 * Create index in YugaByte only if it is a secondary index. Primary key is
+	 * Create index in YugaByte only if it is a secondary non-covered index. Primary key is
 	 * an implicit part of the base table in YugaByte and doesn't need to be created.
 	 */
-	if (IsYBRelation(indexRelation) && !isprimary)
+	 IndexAmRoutine *amroutine =
+		GetIndexAmRoutineByAmId(indexInfo->ii_Am, true);
+	if (IsYBRelation(indexRelation) &&
+		!isprimary && !(amroutine && amroutine->yb_amiscoveredbymaintable))
 	{
 		YBCCreateIndex(indexRelationName,
 					   indexInfo,
@@ -4074,11 +4077,11 @@ reindex_index(Oid indexId, bool skip_constraint_checks, char persistence,
 				 errmsg("cannot reindex temporary tables of other sessions")));
 
 	/*
-	 * YB pk indexes share the same storage as their tables, so it is not
+	 * YB covered indexes share the same storage as their tables, so it is not
 	 * possible to reindex them. However, this code-path may be internally
 	 * invoked by table rewrite, and we need to reset the index's reltuples.
 	 */
-	if (!is_yb_table_rewrite && iRel->rd_index->indisprimary &&
+	if (!is_yb_table_rewrite && YBIsCoveredByMainTable(iRel) &&
 		IsYBRelation(iRel))
 		ereport(ERROR,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -4419,14 +4422,15 @@ reindex_relation(Oid relid, int flags, int options, bool is_yb_table_rewrite,
 				RelationSetIndexList(rel, doneIndexes, InvalidOid);
 			if (IsYBRelation(iRel))
 			{
-				if (!is_yb_table_rewrite && !iRel->rd_index->indisprimary)
+				if (!is_yb_table_rewrite && !YBIsCoveredByMainTable(iRel))
 					/*
 					* Drop the old DocDB table associated with this index.
-					* This is only required for secondary indexes, because a
-					* primary index in YB doesn't have a DocDB table separate
-					* from the base relation's table.
-					* If this is a table rewrite, the indexes on the table
-					* will automatically be dropped when the table is dropped.
+					* This is only required for uncovered
+					* secondary indexes, because a primary index in YB doesn't
+					* have a DocDB table separate from the base relation's
+					* table. If this is a table rewrite, the indexes on the
+					* table will automatically be dropped when the table is
+					* dropped.
 					* Note: The drop isn't finalized until after the txn
 					* commits/aborts.
 					*/

src/postgres/src/backend/commands/indexcmds.c

@@ -1001,6 +1001,13 @@ DefineIndex(Oid relationId,
 				 errmsg("access method \"%s\" does not support exclusion constraints",
 						accessMethodName)));
 
+	/* YB: Inlined indexes are only supported in colocated mode right now. */
+	if (!MyDatabaseColocated && amRoutine->yb_amiscoveredbymaintable)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("access method \"%s\" requires colocation",
+						accessMethodName)));
+
 	amcanorder = amRoutine->amcanorder;
 	amoptions = amRoutine->amoptions;
 
@@ -1654,8 +1661,9 @@ DefineIndex(Oid relationId,
 
 	/* TODO(jason): handle exclusion constraints, possibly not here. */
 
-	/* Do backfill. */
-	HandleYBStatus(YBCPgBackfillIndex(databaseId, indexRelationId));
+	/* YB: Do backfill if this is a separate DocDB table from the main table. */
+	if (!YBIsOidCoveredByMainTable(indexRelationId))
+		HandleYBStatus(YBCPgBackfillIndex(databaseId, indexRelationId));
 
 	YbTestGucFailIfStrEqual(yb_test_fail_index_state_change, "postbackfill");
 

src/postgres/src/backend/commands/tablecmds.c

@@ -3435,11 +3435,11 @@ RenameRelation(RenameStmt *stmt, bool yb_is_internal_clone_rename)
 
 	RenameRelationInternal(relid, stmt->newname, false);
 
-	/* YB rename is not needed for a primary key dummy index. */
+	/* YB rename is not needed for a covered dummy index. */
 	rel             = RelationIdGetRelation(relid);
 	needs_yb_rename = IsYBRelation(rel) &&
 					  !(rel->rd_rel->relkind == RELKIND_INDEX &&
-						rel->rd_index->indisprimary) &&
+						YBIsCoveredByMainTable(rel)) &&
 					  rel->rd_rel->relkind != RELKIND_PARTITIONED_INDEX
 					  && !yb_is_internal_clone_rename;
 	RelationClose(rel);
@@ -12146,9 +12146,9 @@ ATPrepSetTableSpace(AlteredTableInfo *tab, Relation rel,
 				 errmsg("cannot set tablespaces for temporary tables")));
 	}
 
-	if (IsYugaByteEnabled() && tablespacename && 
+	if (IsYugaByteEnabled() && tablespacename &&
 		rel->rd_index &&
-		rel->rd_index->indisprimary) {
+		YBIsCoveredByMainTable(rel)) {
 		/*
 		 * Disable setting tablespaces for primary key indexes in Yugabyte
 		 * clusters.
@@ -12836,23 +12836,23 @@ AlterTableMoveAll(AlterTableMoveAllStmt *stmt)
 			continue;
 
 		/*
-		 * In YB, a primary key index is an intrinsic part of its base table.
-		 * For a primary key index, we only need to update the
+		 * In YB, a covered index is an intrinsic part of its base table.
+		 * For a covered index, we only need to update the
 		 * new_tablespaceoid field in pg_class.
 		 */
 		if (relForm->relkind == RELKIND_INDEX ||
 			relForm->relkind == RELKIND_PARTITIONED_INDEX)
 		{
 			yb_index_rel = RelationIdGetRelation(relOid);
-			bool isPrimaryIndex = (yb_index_rel != NULL &&
-								   yb_index_rel->rd_index->indisprimary);
+			bool isCoveredByMainTable = (yb_index_rel != NULL &&
+				YBIsCoveredByMainTable(yb_index_rel));
 
 			RelationClose(yb_index_rel);
 
-			if (isPrimaryIndex)
+			if (isCoveredByMainTable)
 			{
 				/*
-				 * We move the primary key indexes along with the tables that
+				 * We move the covered indexes along with the tables that
 				 * they are associated with when using the following commands
 				 * ALTER TABLE/INDEX/MATERIALIZED VIEW ... SET TABLESPACE ...
 				 */
@@ -14761,10 +14761,11 @@ AlterRelationNamespaceInternal(Relation classRel, Oid relOid,
 
 		/*
 		 * Call SetSchema handler for the related internal YB DocDB table.
-		 * No YB DocDB table for a primary key dummy index.
+		 * No YB DocDB table for a covered index.
 		 */
 		const Relation rel = RelationIdGetRelation(relOid);
-		if (IsYBRelation(rel) && !(rel->rd_index && rel->rd_index->indisprimary))
+		if (IsYBRelation(rel) && !(rel->rd_index &&
+			YBIsCoveredByMainTable(rel)))
 			YBCAlterTableNamespace(classForm, relOid);
 
 		RelationClose(rel);

src/postgres/src/backend/executor/execIndexing.c

@@ -515,13 +515,13 @@ ExecInsertIndexTuplesOptimized(TupleTableSlot *slot,
 			   indexRelation->rd_index->indisready);
 
 		/*
-		 * No need to update YugaByte primary key which is intrinic part of
-		 * the base table.
+		 * No need to update YugaByte primary key or a covered index
+		 * since they are share the same storage as the base table.
 		 *
 		 * TODO(neil) The following YB check might not be needed due to later work on indexes.
 		 * We keep this check for now as this bugfix will be backported to ealier releases.
 		 */
-		if (isYBRelation && indexRelation->rd_index->indisprimary &&
+		if (isYBRelation && YBIsCoveredByMainTable(indexRelation) &&
 			!YbIsInsertOnConflictReadBatchingEnabled(resultRelInfo))
 			continue;
 
@@ -677,7 +677,7 @@ ExecDeleteIndexTuplesOptimized(Datum ybctid,
 		 * - As a result, we don't need distinguish between Postgres and YugaByte here.
 		 *   I update this code only for clarity.
 		 */
-		if (isYBRelation && indexRelation->rd_index->indisprimary &&
+		if (isYBRelation && YBIsCoveredByMainTable(indexRelation) &&
 			!YbIsInsertOnConflictReadBatchingEnabled(resultRelInfo))
 			continue;
 

src/postgres/src/backend/executor/ybcModifyTable.c

@@ -1333,12 +1333,23 @@ void YBCUpdateSysCatalogTupleForDb(Oid dboid, Relation rel, HeapTuple oldtuple,
 	YBCApplyWriteStmt(update_stmt, rel);
 }
 
+/*
+ * This checks if a YB relation has any separate index DocDB tables.
+ * Covered/primary indexes do not have separate DocDB tables.
+ */
 bool
 YBCRelInfoHasSecondaryIndices(ResultRelInfo *resultRelInfo)
 {
-	return resultRelInfo->ri_NumIndices > 1 ||
-			(resultRelInfo->ri_NumIndices == 1 &&
-			 !resultRelInfo->ri_IndexRelationDescs[0]->rd_index->indisprimary);
+	for (int i = 0; i < resultRelInfo->ri_NumIndices; i++)
+	{
+		Relation index = resultRelInfo->ri_IndexRelationDescs[i];
+		if (YBIsCoveredByMainTable(index))
+			continue;
+
+		return true;
+	}
+
+	return false;
 }
 
 /*

src/postgres/src/backend/optimizer/plan/createplan.c

@@ -3844,14 +3844,14 @@ create_indexscan_plan(PlannerInfo *root,
 		if (bitmapindex)
 			need_idx_remote = true;
 		/*
-		 * For hypothetical index where primary index isn't involved, there is
+		 * For hypothetical index where no covered index is involved, there is
 		 * no Relation. Hence don't make change to need_idx_remote.
 		 */
 		else if (!indexonly && !best_path->indexinfo->hypothetical)
 		{
 			Relation index;
 			index = RelationIdGetRelation(best_path->indexinfo->indexoid);
-			need_idx_remote = !index->rd_index->indisprimary;
+			need_idx_remote = !YBIsCoveredByMainTable(index);
 			RelationClose(index);
 		}
 		else

src/postgres/src/backend/optimizer/plan/planner.c

@@ -1687,7 +1687,7 @@ yb_is_main_table(IndexOptInfo *indexinfo)
 
 	indrel = RelationIdGetRelation(indexinfo->indexoid);
 	if (indrel != NULL && indrel->rd_index != NULL)
-		is_main_table = indrel->rd_index->indisprimary;
+		is_main_table = YBIsCoveredByMainTable(indrel);
 	if (indrel != NULL)
 		RelationClose(indrel);
 	return is_main_table;

src/postgres/src/backend/optimizer/util/ybcplan.c

@@ -264,7 +264,7 @@ is_index_only_attribute_nums(List *colrefs, IndexOptInfo *indexinfo,
 				{
 					Relation index;
 					index = RelationIdGetRelation(indexinfo->indexoid);
-					bool is_primary = index->rd_index->indisprimary;
+					bool is_primary = YBIsCoveredByMainTable(index);
 					RelationClose(index);
 
 					if (is_primary)

src/postgres/src/backend/utils/adt/dbsize.c

@@ -397,8 +397,8 @@ calculate_table_size(Relation rel)
 
 	if (IsYBRelation(rel))
 	{
-		/* Primary index relation doesn't have dedicated table in DocDB */
-		if (rel->rd_index && rel->rd_index->indisprimary)
+		/* Covered index relations don't have a dedicated table in DocDB */
+		if (rel->rd_index && YBIsCoveredByMainTable(rel))
 			return -1;
 
 		/* Colcoated tables do not have size info */

src/postgres/src/backend/utils/adt/ri_triggers.c

@@ -261,7 +261,7 @@ YBCBuildYBTupleIdDescriptor(const RI_ConstraintInfo *riinfo, HeapTuple tup)
 	bool using_index = false;
 	Relation idx_rel = RelationIdGetRelation(riinfo->conindid);
 	Relation source_rel = idx_rel;
-	if (idx_rel->rd_index != NULL && !idx_rel->rd_index->indisprimary)
+	if (idx_rel->rd_index != NULL && !YBIsCoveredByMainTable(idx_rel))
 	{
 		Assert(IndexRelationGetNumberOfKeyAttributes(idx_rel) == riinfo->nkeys);
 		using_index = true;

src/postgres/src/backend/utils/adt/ruleutils.c

@@ -1504,7 +1504,7 @@ pg_get_indexdef_worker(Oid indexrelid, int colno,
 		appendStringInfoChar(&buf, ')');
 
 		if (includeYbMetadata && IsYBRelation(indexrel) &&
-			!idxrec->indisprimary)
+			!YBIsCoveredByMainTable(indexrel))
 		{
 			YbAppendIndexReloptions(&buf, indexrelid, YbGetTableProperties(indexrel));
 		}

src/postgres/src/backend/utils/cache/relcache.c

@@ -5447,7 +5447,7 @@ RelationSetNewRelfilenode(Relation relation, char persistence,
 			   relation->rd_rel->relkind == RELKIND_RELATION);
 
 		if (relation->rd_rel->relkind == RELKIND_INDEX &&
-			!relation->rd_index->indisprimary)
+			!YBIsCoveredByMainTable(relation))
 			/*
 			 * Note: caller is responsible for dropping the old DocDB table
 			 * associated with the index, if required.