/*-------------------------------------------------------------------------
 * Copyright (c) Microsoft Corporation.  All rights reserved.
 *
 * src/planner/bson_aggregation_nested_pipeline.c
 *
 * Implementation of the backend query generation for pipelines that have
 * nested pipelines (such as $lookup, $facet, $inverseMatch).
 *
 *-------------------------------------------------------------------------
 */


#include <postgres.h>
#include <float.h>
#include <fmgr.h>
#include <miscadmin.h>
#include <utils/lsyscache.h>

#include <catalog/pg_operator.h>
#include <optimizer/planner.h>
#include <nodes/nodes.h>
#include <nodes/makefuncs.h>
#include <nodes/nodeFuncs.h>
#include <parser/parser.h>
#include <parser/parse_agg.h>
#include <parser/parse_clause.h>
#include <parser/parse_param.h>
#include <parser/analyze.h>
#include <parser/parse_oper.h>
#include <utils/ruleutils.h>
#include <utils/builtins.h>
#include <catalog/pg_aggregate.h>
#include <catalog/pg_class.h>
#include <rewrite/rewriteSearchCycle.h>
#include <utils/version_utils.h>

#include "io/bson_core.h"
#include "metadata/metadata_cache.h"
#include "query/query_operator.h"
#include "planner/documentdb_planner.h"
#include "aggregation/bson_aggregation_pipeline.h"
#include "commands/parse_error.h"
#include "commands/commands_common.h"
#include "utils/feature_counter.h"
#include "utils/version_utils.h"
#include "operators/bson_expression.h"

#include "aggregation/bson_aggregation_pipeline_private.h"

/* We use this flag to determine if an specialized VAR in $lookup whether let or document VAR needs varlevelsup adjustment
 * The last 2 bytes are reserved to store the nested pipeline level, since the
 * maximum nested pipeline level is 20, we can use 2 bytes to store it.
 */
#define NESTED_PIPELINE_VAR_FLAG 0x0F000000

const int MaximumLookupPipelineDepth = 20;
extern bool EnableOperatorVariablesInLookup;

/*
 * Struct having parsed view of the
 * arguments to Lookup.
 */
typedef struct
{
	/*
	 * The name of the collection to lookup
	 */
	StringView from;

	/*
	 * The alias where the results of the lookup are placed
	 */
	StringView lookupAs;

	/*
	 * The remote field to match (if applicable)
	 */
	StringView foreignField;

	/*
	 * The local field to match (if applicable)
	 */
	StringView localField;

	/*
	 * A pipeline to execute (for uncorrelated lookups)
	 */
	bson_value_t pipeline;

	/*
	 * let variables for $lookup
	 */
	pgbson *let;

	/*
	 * has a join (foreign/local field).
	 */
	bool hasLookupMatch;
} LookupArgs;

typedef struct
{
	bool isLookupUnwind;

	/*
	 * Only applicable for Lookup Unwind combination.
	 */
	bool preserveNullAndEmptyArrays;
} LookupContext;

typedef struct LookupOptimizationArgs
{
	/*
	 * Is the query agnostic
	 */
	bool isLookupAgnostic;

	/*
	 * Whether or not the $lookup is uncorrelated.
	 */
	bool isLookupUncorrelated;

	/*
	 * The segment of the pipeline stages that can be inlined.
	 */
	List *inlinedPipelineStages;

	/*
	 * The segment of the pipeline stages that cannot be inlined and needs
	 * to be applied post-join.
	 */
	List *nonInlinedPipelineStages;

	/*
	 * Can the join be done on the right query's _id?
	 */
	bool isLookupJoinOnRightId;

	/*
	 * Can the join be done on the left query's _id?
	 */
	bool isLookupJoinOnLeftId;

	/*
	 * Right query's subqueryContext
	 */
	AggregationPipelineBuildContext rightQueryContext;

	/*
	 * The base query on the rightquery.
	 */
	Query *rightBaseQuery;

	/*
	 * Whether or not the query has Let.
	 * This is true if the lookupArgs has let *or* the parent context
	 * has a let that is non const.
	 */
	bool hasLet;

	/*
	 * Referrence to the first non-inlined $match stage with $expr qualifiers which
	 * was not inlined due to the presence of a `let`.
	 * We do a best effort inlining of this stage close to the right query to avoid
	 * the stage post-JOIN
	 */
	AggregationStage *nonInlinedMatchStage;

	/*
	 * The attrNum for the lookup let in left query
	 */
	AttrNumber lookupLetAttrNum;
} LookupOptimizationArgs;


/*
 * The walker state to replace the Let variable in queries.
 */
typedef struct LevelsUpQueryTreeWalkerState
{
	/* numLevels up - modified during modify state*/
	int numLevels;

	/* the let variable in the query */
	Var *originalVariable;

	/* Used by the RTE CTE LevelsUp Walker */
	const char *cteName;
} LevelsUpQueryTreeWalkerState;


/*
 * Args processed from the $graphLookup stage.
 */
typedef struct
{
	/* the input expression must start with the specified input */
	bson_value_t inputExpression;

	/* The target collection */
	StringView fromCollection;

	/* the connectFrom field */
	StringView connectFromField;

	/* the connectTo field */
	StringView connectToField;

	/* the field it should be written to */
	StringView asField;

	/* How many times to recurse */
	int32_t maxDepth;

	/* optional depth Field to write it into */
	StringView depthField;

	/* A match clause to restrict search */
	bson_value_t restrictSearch;

	/* connectFrom field as an expression */
	bson_value_t connectFromFieldExpression;
} GraphLookupArgs;

/*
 * Args processed from the $inverseMatch stage.
 */
typedef struct InverseMatchArgs
{
	/* The specified path. */
	StringView path;

	/* The from collection to run the aggregation pipeline provided in the args. */
	StringView fromCollection;

	/* The specified input. */
	bson_value_t input;

	/* The aggregation pipeline specified to run on the fromCollection. */
	bson_value_t pipeline;

	/* The default result to use if a query in a document is not found. */
	bson_value_t defaultResult;
} InverseMatchArgs;


static int ValidateFacet(const bson_value_t *facetValue);
static Query * BuildFacetUnionAllQuery(int numStages, const bson_value_t *facetValue,
									   CommonTableExpr *baseCte, QuerySource querySource,
									   const bson_value_t *sortSpec,
									   AggregationPipelineBuildContext *parentContext);
static Query * AddBsonArrayAggFunction(Query *baseQuery,
									   AggregationPipelineBuildContext *context,
									   ParseState *parseState, const char *fieldPath,
									   uint32_t fieldPathLength, bool migrateToSubQuery,
									   Aggref **aggrefPtr);
static Query * AddBsonObjectAggFunction(Query *baseQuery,
										AggregationPipelineBuildContext *context);
static void ParseLookupStage(const bson_value_t *existingValue, LookupArgs *args);
static void ParseGraphLookupStage(const bson_value_t *existingValue,
								  GraphLookupArgs *args);
static Query * CreateInverseMatchFromCollectionQuery(InverseMatchArgs *inverseMatchArgs,
													 AggregationPipelineBuildContext *
													 context, ParseState *parseState);
static bool ParseInverseMatchSpec(const bson_value_t *spec, InverseMatchArgs *args);
static Query * CreateCteSelectQuery(CommonTableExpr *baseCte, const char *prefix,
									int stageNum, int levelsUp);
static Query * ProcessLookupCoreWithLet(Query *query,
										AggregationPipelineBuildContext *context,
										LookupArgs *lookupArgs,
										LookupContext *lookupContext);
static void ValidatePipelineForShardedLookupWithLet(const bson_value_t *pipeline);
static Query * ProcessGraphLookupCore(Query *query,
									  AggregationPipelineBuildContext *context,
									  GraphLookupArgs *lookupArgs);
static Query * BuildGraphLookupCteQuery(QuerySource parentSource,
										CommonTableExpr *baseCteExpr,
										GraphLookupArgs *args,
										AggregationPipelineBuildContext *parentContext);
static Query * BuildRecursiveGraphLookupQuery(QuerySource parentSource,
											  GraphLookupArgs *args,
											  AggregationPipelineBuildContext *
											  parentContext,
											  CommonTableExpr *baseCteExpr, int levelsUp);
static void ValidateUnionWithPipeline(const bson_value_t *pipeline, bool hasCollection);

static void ValidateVariableIsDefined(AggregationExpressionData *parsedData,
									  HTAB *operatorVariables);
static void WalkQueryAndSetLevelsUp(Query *query, Var *varToCheck,
									int varLevelsUpBase);
static void WalkQueryAndSetCteLevelsUp(Query *query, const char *cteName,
									   int varLevelsUpBase);
static Query * HandleLookupCore(const bson_value_t *existingValue, Query *query,
								AggregationPipelineBuildContext *context,
								LookupContext *lookupContext);
static Query * AddLookupRightQueryExpressionOrArrayAgg(Query *rightQuery,
													   AggregationPipelineBuildContext *
													   context,
													   ParseState *parseState,
													   LookupArgs *lookupArgs,
													   LookupContext *lookupContext,
													   bool requiresSubQuery);

/*
 * Validates and returns a given pipeline stage: Used in validations for facet/lookup/unionWith
 */
inline static pgbsonelement
GetPipelineStage(bson_iter_t *pipelineIter, const char *parentStage, const
				 char *pipelineKey)
{
	const bson_value_t *pipelineStage = bson_iter_value(pipelineIter);
	if (!BSON_ITER_HOLDS_DOCUMENT(pipelineIter))
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
						errmsg(
							"Pipeline stage for %s %s must be a document",
							parentStage, pipelineKey)));
	}

	pgbsonelement stageElement;
	if (!TryGetBsonValueToPgbsonElement(pipelineStage, &stageElement))
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40323),
						errmsg(
							"Pipeline stage must have a single field.")));
	}

	return stageElement;
}


/*
 * Processes the $_internalInhibitOptimization Pipeine stage.
 * Injects a CTE into the pipeline with the MaterializeAlways flag
 * generating a break in the pipeline.
 */
Query *
HandleInternalInhibitOptimization(const bson_value_t *existingValue, Query *query,
								  AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_INTERNAL_INHIBIT_OPTIMIZATION);

	/* First step, move the current query into a CTE */
	CommonTableExpr *baseCte = makeNode(CommonTableExpr);
	baseCte->ctename = "internalinhibitoptimization";
	baseCte->ctequery = (Node *) query;

	/* Mark it as materialize always */
	baseCte->ctematerialized = CTEMaterializeAlways;
	int levelsUp = 0;
	query = CreateCteSelectQuery(baseCte, "inhibit", context->stageNum, levelsUp);
	query->cteList = list_make1(baseCte);
	return query;
}


/*
 * Processes the $facet Pipeine stage.
 * Injects a CTE for the current query.
 * Then builds a UNION ALL query that has the N pipelines
 * selecting from the common CTE.
 * Finally aggregates all of it into a BSON_OBJECT_AGG
 */
Query *
HandleFacet(const bson_value_t *existingValue, Query *query,
			AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_FACET);
	if (list_length(query->targetList) > 1)
	{
		/* if we have multiple projectors, push to a subquery (Facet needs 1 projector) */
		/* TODO: Can we get rid of this Subquery? */
		query = MigrateQueryToSubQuery(query, context);
	}

	int numStages = ValidateFacet(existingValue);

	/* First step, move the current query into a CTE */
	CommonTableExpr *baseCte = makeNode(CommonTableExpr);

	/* Adding the stage number to the CTE alias (which is done in CreateCteSelectQuery()) is not enough to avoid conflict
	 * when there are multiple top level facets due to the facet stage is planned (see facet explains)*/
	baseCte->ctename = psprintf("facet_base_%d_%d", context->stageNum,
								context->nestedPipelineLevel);
	baseCte->ctequery = (Node *) query;

	/* Second step: Build UNION ALL query */
	Query *finalQuery = BuildFacetUnionAllQuery(numStages, existingValue, baseCte,
												query->querySource,
												&context->sortSpec,
												context);

	/* Finally, add bson_object_agg */
	finalQuery = AddBsonObjectAggFunction(finalQuery, context);
	finalQuery->cteList = list_make1(baseCte);
	WalkQueryAndSetCteLevelsUp(finalQuery, baseCte->ctename, 0);
	context->requiresSubQuery = true;
	return finalQuery;
}


/*
 * Top level method handling processing of the lookup stage.
 */
Query *
HandleLookup(const bson_value_t *existingValue, Query *query,
			 AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_LOOKUP);

	LookupContext lookupContext = { 0 };
	return HandleLookupCore(existingValue, query, context, &lookupContext);
}


/*
 * Top level method handling processing of the lookup and unwind stage combination.
 */
Query *
HandleLookupUnwind(const bson_value_t *existingValue, Query *query,
				   AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_LOOKUP);
	ReportFeatureUsage(FEATURE_STAGE_UNWIND);

	/* The spec for lookup unwind has 2 fields, `lookup` spec and `preserveNullAndEmptyArrays` from inlined unwind */
	bson_value_t lookupSpec;
	bool preserveNullAndEmptyArrays = false;

	bson_iter_t iter;
	BsonValueInitIterator(existingValue, &iter);
	while (bson_iter_next(&iter))
	{
		const char *key = bson_iter_key(&iter);
		if (strcmp(key, "lookup") == 0)
		{
			lookupSpec = *bson_iter_value(&iter);
		}
		else if (strcmp(key, "preserveNullAndEmptyArrays") == 0)
		{
			preserveNullAndEmptyArrays = bson_iter_as_bool(&iter);
		}
	}

	LookupContext lookupContext = {
		.isLookupUnwind = true,
		.preserveNullAndEmptyArrays = preserveNullAndEmptyArrays
	};
	return HandleLookupCore(&lookupSpec, query, context, &lookupContext);
}


/*
 * CanInlineLookupWithUnwind checks if the lookup stage can be inlined with the unwind stage.
 * Iff the lookup stage is a simple lookup with no pipeline and the aggregated field of lookup is same
 * as unwinding path, then we can inline the lookup stage with the unwind stage.
 */
bool
CanInlineLookupWithUnwind(const bson_value_t *lookUpStageValue,
						  const bson_value_t *unwindStageValue,
						  bool *isPreserveEmptyAndNullArrays)
{
	LookupArgs lookupArgs = { 0 };
	ParseLookupStage(lookUpStageValue, &lookupArgs);

	if (lookupArgs.lookupAs.length == 0)
	{
		return false;
	}

	if (lookupArgs.pipeline.value_type != BSON_TYPE_EOD)
	{
		/*
		 * We can't inline the lookup stage with unwind stage if the lookup stage has a pipeline.
		 * because today we run the pipeline against all the documents that are matched post JOIN
		 * which requires us to aggregate the matched docs anywat, so inlining for this case is tricky plus
		 * calls for a lookup rewrite so better drop the inlining
		 */
		return false;
	}

	UnwindArgs unwindArgs = { 0 };
	UnwindParseErrorHandler onErrorIgnored = NULL;
	if (!TryParseUnwindStage(unwindStageValue, &unwindArgs, onErrorIgnored))
	{
		return false;
	}

	/* includeArrayIndex requires synthesizing an extra field per row;
	 * the fused op has no place to put it. */
	if (unwindArgs.includeArrayIndex.length > 0)
	{
		return false;
	}

	*isPreserveEmptyAndNullArrays = unwindArgs.preserveNullAndEmptyArrays;

	/* TryParseUnwindStage guarantees the path is "$<field>", but validate
	 * defensively before skipping the leading '$' for the comparison. */
	const char *unwindPath = unwindArgs.pathValue.value.v_utf8.str;
	if (unwindPath == NULL || unwindPath[0] != '$')
	{
		return false;
	}

	/* Compare the trailing field name to the lookup's "as" target. */
	return strcmp(unwindPath + 1, lookupArgs.lookupAs.string) == 0;
}


/*
 * Top level method handling processing of the $graphLookup Stage.
 */
Query *
HandleGraphLookup(const bson_value_t *existingValue, Query *query,
				  AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_GRAPH_LOOKUP);

	GraphLookupArgs graphArgs;
	memset(&graphArgs, 0, sizeof(GraphLookupArgs));
	ParseGraphLookupStage(existingValue, &graphArgs);

	return ProcessGraphLookupCore(query, context, &graphArgs);
}


/*
 * Top level method handling processing of the $documents stage.
 */
Query *
HandleDocumentsStage(const bson_value_t *existingValue, Query *query,
					 AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_DOCUMENTS);

	if (list_length(query->rtable) != 0 || context->stageNum != 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
						errmsg(
							"$documents can only be used as the first stage in a pipeline.")));
	}

	/* Documents is an expression */
	StringView documentsPath =
	{
		.string = "$documents",
		.length = 10
	};

	pgbson_writer writer;
	PgbsonWriterInit(&writer);

	ParseAggregationExpressionContext parseContext = { 0 };
	parseContext.collationString = context->collationString;

	AggregationExpressionData expressionData;
	memset(&expressionData, 0, sizeof(AggregationExpressionData));

	ParseAggregationExpressionData(&expressionData, existingValue, &parseContext);

	ExpressionVariableContext *variableContext = NULL;
	bool isNullOnEmpty = false;
	EvaluateAggregationExpressionDataToWriter(&expressionData, PgbsonInitEmpty(),
											  documentsPath, &writer, variableContext,
											  isNullOnEmpty);

	/* Validate documents */
	pgbson *targetBson = PgbsonWriterGetPgbson(&writer);
	pgbsonelement fetchedElement = { 0 };
	if (!TryGetSinglePgbsonElementFromPgbson(targetBson, &fetchedElement) ||
		fetchedElement.bsonValue.value_type != BSON_TYPE_ARRAY)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION5858203),
						errmsg(
							"An array was expected.")));
	}

	/* Create a distinct unwind - to expand arrays and such */
	Const *unwindValue = MakeTextConst(documentsPath.string,
									   documentsPath.length);
	List *args = list_make2(MakeBsonConst(targetBson), unwindValue);
	FuncExpr *resultExpr = makeFuncExpr(
		BsonLookupUnwindFunctionOid(), BsonTypeId(), args, InvalidOid,
		InvalidOid, COERCE_EXPLICIT_CALL);
	resultExpr->funcretset = true;

	RangeTblFunction *tblFunction = makeNode(RangeTblFunction);
	tblFunction->funcexpr = (Node *) resultExpr;
	tblFunction->funccolcount = 1;
	tblFunction->funccoltypes = list_make1_oid(BsonTypeId());
	tblFunction->funccolcollations = list_make1_oid(InvalidOid);
	tblFunction->funccoltypmods = list_make1_int(-1);

	RangeTblEntry *rte = makeNode(RangeTblEntry);
	rte->rtekind = RTE_FUNCTION;
	rte->self_reference = false;
	rte->lateral = false;
	rte->inh = false;
	rte->inFromCl = true;
	rte->functions = list_make1(tblFunction);

	List *colnames = list_make1(makeString("documents"));
	rte->alias = makeAlias("documents", NIL);
	rte->eref = makeAlias("documents", colnames);

	Var *queryOutput = makeVar(1, 1, BsonTypeId(),
							   -1, InvalidOid, 0);
	bool resJunk = false;
	TargetEntry *upperEntry = makeTargetEntry((Expr *) queryOutput, 1,
											  "documents_aggregate",
											  resJunk);
	query->targetList = list_make1(upperEntry);
	query->rtable = list_make1(rte);

	RangeTblRef *rtr = makeNode(RangeTblRef);
	rtr->rtindex = 1;
	query->jointree = makeFromExpr(list_make1(rtr), NULL);

	context->requiresSubQuery = true;
	return query;
}


/* Handles the $inverseMatch stage.
 * It generates a query like:
 * SELECT document FROM collection WHERE
 * bson_dollar_inverse_match(document, '{"path": <path>, "input": <input>}')
 */
Query *
HandleInverseMatch(const bson_value_t *existingValue, Query *query,
				   AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_INVERSEMATCH);

	if (existingValue->value_type != BSON_TYPE_DOCUMENT)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
						errmsg(
							"$inverseMatch requires a document as an input instead got: %s",
							BsonTypeName(
								existingValue->value_type)),
						errdetail_log(
							"$inverseMatch requires a document as an input instead got: %s",
							BsonTypeName(
								existingValue->value_type))));
	}

	if (query->limitOffset != NULL || query->limitCount != NULL ||
		query->sortClause != NULL)
	{
		query = MigrateQueryToSubQuery(query, context);
	}

	InverseMatchArgs arguments;
	memset(&arguments, 0, sizeof(InverseMatchArgs));

	bool useFromCollection = ParseInverseMatchSpec(existingValue, &arguments);

	/* The first projector is the document */
	TargetEntry *firstEntry = linitial(query->targetList);

	Expr *currentProjection = firstEntry->expr;

	/* build WHERE ApiSchemaName.bson_dollar_inverse_match clause */
	Expr *specArgument = NULL;
	ParseState *parseState = make_parsestate(NULL);
	parseState->p_expr_kind = EXPR_KIND_SELECT_TARGET;
	parseState->p_next_resno = list_length(query->targetList) + 1;

	if (!useFromCollection)
	{
		pgbson *specBson = PgbsonInitFromBuffer(
			(char *) existingValue->value.v_doc.data,
			existingValue->value.v_doc.data_len);
		specArgument = (Expr *) MakeBsonConst(specBson);
	}
	else
	{
		/* if from collection is specified we generate a query like:
		 * SELECT document
		 * FROM ApiDataSchemaName.documents_963002 collection
		 * WHERE ApiInternalSchemaName.bson_dollar_inverse_match(
		 *  document,
		 *  (
		 *      SELECT ApiCatalogSchema.bson_dollar_add_fields(
		 *          '{ "path" : "rule", "defaultResult" : false }'::ApiCatalogSchema.bson,
		 *          (
		 *              SELECT COALESCE(
		 *                  ApiCatalogSchema.bson_array_agg(collection_0_1.document, 'input'::text),
		 *                  '{ "input" : [  ] }'::CoreSchema.bson
		 *              ) AS document
		 *              FROM ApiDataSchemaName.documents_963001_9630019 collection_0_1
		 *              WHERE ApiCatalogSchema.bson_dollar_ne(
		 *                  collection_0_1.document,
		 *                  '{ "user_id" : { "$numberInt" : "200" } }'::CoreSchema.bson
		 *              )
		 *              AND collection_0_1.shard_key_value OPERATOR(pg_catalog.=) '963001'::bigint
		 *              LIMIT '1'::bigint
		 *          )
		 *      ) AS spec
		 *  )
		 * )
		 * AND shard_key_value OPERATOR(pg_catalog.=) '963002'::bigint;
		 */
		if (StringViewEquals(&context->collectionNameView, &arguments.fromCollection))
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
							errmsg(
								"'from' collection should be a different collection than the base collection the inverseMatch is running against.")));
		}

		/* Generate add_fields subquery to pass down as an argument. */
		Query *addFieldsQuery = CreateInverseMatchFromCollectionQuery(&arguments, context,
																	  parseState);

		SubLink *addFieldsSubLink = makeNode(SubLink);
		addFieldsSubLink->subLinkType = EXPR_SUBLINK;
		addFieldsSubLink->subLinkId = 1; /* addFieldsQuery has a sublink already. */
		addFieldsSubLink->subselect = (Node *) addFieldsQuery;

		query->hasSubLinks = true;
		specArgument = (Expr *) addFieldsSubLink;
	}

	FuncExpr *inverseMatchFuncExpr = makeFuncExpr(
		BsonDollarInverseMatchFunctionId(), BOOLOID,
		list_make2(currentProjection, specArgument),
		InvalidOid, InvalidOid,
		COERCE_EXPLICIT_CALL);

	List *quals = list_make1(inverseMatchFuncExpr);
	if (query->jointree->quals != NULL)
	{
		quals = lappend(quals, query->jointree->quals);
	}

	free_parsestate(parseState);

	query->jointree->quals = (Node *) make_ands_explicit(quals);
	return query;
}


/* Helper method to generate the sub query used as an argument to the bson_inverse_match UDF.
 * When from collection is specified we need to generate the spec we pass down to inverse match,
 * using the result of the pipeline executed on the from collection. For that we need to generate
 * a subquery used as the parameter to inverse match, i.e:
 * (SELECT bson_dollar_add_fields(spec, (SELECT COALESCE(bson_array_agg(document, "input"), '{"input": []}')) as spec FROM collection))
 *
 * The query passed to bson_dollar_add_fields will vary depending on  the pipeline specified.
 */
static Query *
CreateInverseMatchFromCollectionQuery(InverseMatchArgs *inverseMatchArgs,
									  AggregationPipelineBuildContext *context,
									  ParseState *parseState)
{
	pg_uuid_t *collectionUuid = NULL;
	AggregationPipelineBuildContext subPipelineContext = { 0 };
	subPipelineContext.nestedPipelineLevel = context->nestedPipelineLevel + 1;
	subPipelineContext.databaseNameDatum = context->databaseNameDatum;
	subPipelineContext.variableSpec = context->variableSpec;
	subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
	subPipelineContext.parentStageName = ParentStageName_INVERSEMATCH;
	strncpy((char *) subPipelineContext.collationString, context->collationString,
			MAX_ICU_COLLATION_LENGTH);

	bson_value_t *indexHint = NULL;
	Query *nestedPipeline = GenerateBaseTableQuery(context->databaseNameDatum,
												   &inverseMatchArgs->fromCollection,
												   collectionUuid, indexHint,
												   &subPipelineContext);

	if (subPipelineContext.mongoCollection == NULL)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_NAMESPACENOTFOUND),
						errmsg(
							"The 'from' collection '%s.%s' could not be found.",
							text_to_cstring(context->databaseNameDatum),
							inverseMatchArgs->fromCollection.string)));
	}

	List *stages = ExtractAggregationStages(&inverseMatchArgs->pipeline,
											&subPipelineContext);
	nestedPipeline = MutateQueryWithPipeline(nestedPipeline, stages,
											 &subPipelineContext);

	bool requiresSubQuery = false;
	Aggref **aggrefPtr = NULL;
	nestedPipeline = AddBsonArrayAggFunction(nestedPipeline, &subPipelineContext,
											 parseState, "input", 5, requiresSubQuery,
											 aggrefPtr);

	/* once we have the bson with the input, we should append this to the spec with bson_add_fields. */
	Query *addFieldsQuery = makeNode(Query);
	addFieldsQuery->commandType = CMD_SELECT;
	addFieldsQuery->querySource = QSRC_ORIGINAL;
	addFieldsQuery->canSetTag = true;
	addFieldsQuery->hasSubLinks = true;

	SubLink *subLink = makeNode(SubLink);
	subLink->subLinkType = EXPR_SUBLINK;
	subLink->subLinkId = 0;
	subLink->subselect = (Node *) nestedPipeline;

	/* Write new spec with the path and defaultResult arguments. */
	pgbson_writer writer;
	PgbsonWriterInit(&writer);
	PgbsonWriterAppendUtf8(&writer, "path", -1, inverseMatchArgs->path.string);
	PgbsonWriterAppendValue(&writer, "defaultResult", -1,
							&inverseMatchArgs->defaultResult);

	pgbson *specBson = PgbsonWriterGetPgbson(&writer);

	/* Since no dotted path, no need to override array */
	bool overrideArray = false;
	List *addFieldsArgs = list_make3(MakeBsonConst(specBson), subLink,
									 MakeBoolValueConst(overrideArray));
	FuncExpr *projectorFunc = makeFuncExpr(
		BsonDollaMergeDocumentsFunctionOid(), BsonTypeId(),
		addFieldsArgs,
		InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL);

	TargetEntry *specProjector = makeTargetEntry((Expr *) projectorFunc,
												 1,
												 "spec", false);

	addFieldsQuery->targetList = list_make1(specProjector);
	addFieldsQuery->jointree = makeNode(FromExpr);

	return addFieldsQuery;
}


/* Parses and validates the inverse match spec.
 * The only validation it doesn't do is enforcing that input is a document or an array of documents.
 * We leave that validation to the bson_dollar_inverse_match UDF since a path expression is valid in this context
 * and we can't validate it at this stage.
 * Returns true if it is from collection is specified, false other-wise
 */
static bool
ParseInverseMatchSpec(const bson_value_t *spec, InverseMatchArgs *args)
{
	bson_iter_t docIter;
	BsonValueInitIterator(spec, &docIter);
	while (bson_iter_next(&docIter))
	{
		const char *key = bson_iter_key(&docIter);
		if (strcmp(key, "path") == 0)
		{
			EnsureTopLevelFieldType("$inverseMatch.path", &docIter, BSON_TYPE_UTF8);
			const bson_value_t pathValue = *bson_iter_value(&docIter);
			args->path.length = pathValue.value.v_utf8.len;
			args->path.string = pathValue.value.v_utf8.str;
		}
		else if (strcmp(key, "input") == 0)
		{
			args->input = *bson_iter_value(&docIter);
		}
		else if (strcmp(key, "defaultResult") == 0)
		{
			EnsureTopLevelFieldType("$inverseMatch.defaultResult", &docIter,
									BSON_TYPE_BOOL);
			args->defaultResult = *bson_iter_value(&docIter);
		}
		else if (strcmp(key, "from") == 0)
		{
			EnsureTopLevelFieldType("$inverseMatch.from", &docIter,
									BSON_TYPE_UTF8);
			args->fromCollection.string = bson_iter_utf8(&docIter,
														 &args->fromCollection.length);
		}
		else if (strcmp(key, "pipeline") == 0)
		{
			EnsureTopLevelFieldType("$inverseMatch.pipeline", &docIter,
									BSON_TYPE_ARRAY);
			args->pipeline = *bson_iter_value(&docIter);

			bson_iter_t pipelineArray;
			BsonValueInitIterator(&args->pipeline, &pipelineArray);

			/* These are the allowed nested stages in an $inverseMatch */
			while (bson_iter_next(&pipelineArray))
			{
				pgbsonelement stageElement = GetPipelineStage(&pipelineArray,
															  "inverseMatch",
															  "pipeline");
				const char *nestedPipelineStage = stageElement.path;
				if (strcmp(nestedPipelineStage, "$match") != 0 &&
					strcmp(nestedPipelineStage, "$project") != 0 &&
					strcmp(nestedPipelineStage, "$limit") != 0)
				{
					ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
									errmsg(
										"%s is not allowed to be used within an $inverseMatch stage, only $match, $project or $limit are allowed",
										nestedPipelineStage)));
				}
			}
		}
		else
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
							errmsg(
								"Unrecognized parameter supplied to $inverseMatch: '%s'",
								key)));
		}
	}

	if (args->path.length == 0 || args->path.string == NULL)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE), errmsg(
							"Path parameter is missing for the operator $inverseMatch")));
	}

	bool useFromCollection = false;
	if (args->fromCollection.length > 0)
	{
		if (args->input.value_type != BSON_TYPE_EOD)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
							errmsg(
								"'input' and 'from' can't be used together in an $inverseMatch stage.")));
		}

		if (args->pipeline.value_type == BSON_TYPE_EOD)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
							errmsg(
								"'pipeline' argument is required when 'from' is specified in an $inverseMatch stage.")));
		}

		useFromCollection = true;
	}
	else if (args->input.value_type == BSON_TYPE_EOD)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE), errmsg(
							"Missing 'input' and 'from' parameter to $inverseMatch, one should be provided.")));
	}

	if (args->defaultResult.value_type == BSON_TYPE_EOD)
	{
		/* If missing, it will automatically default to false. */
		args->defaultResult.value_type = BSON_TYPE_BOOL;
		args->defaultResult.value.v_bool = false;
	}

	return useFromCollection;
}


/*
 * Helper method to parse the union with arguments
 */
void
ParseUnionWith(const bson_value_t *existingValue, StringView *collectionFrom,
			   bson_value_t *pipeline)
{
	if (existingValue->value_type == BSON_TYPE_UTF8)
	{
		/* This is unionWith on the base collection */
		collectionFrom->length = existingValue->value.v_utf8.len;
		collectionFrom->string = existingValue->value.v_utf8.str;
	}
	else if (existingValue->value_type == BSON_TYPE_DOCUMENT)
	{
		bson_iter_t pipelineIterator;
		BsonValueInitIterator(existingValue, &pipelineIterator);

		while (bson_iter_next(&pipelineIterator))
		{
			const char *key = bson_iter_key(&pipelineIterator);
			const bson_value_t *value = bson_iter_value(&pipelineIterator);
			if (strcmp(key, "coll") == 0)
			{
				EnsureTopLevelFieldType("$unionWith.coll", &pipelineIterator,
										BSON_TYPE_UTF8);
				collectionFrom->length = value->value.v_utf8.len;
				collectionFrom->string = value->value.v_utf8.str;
			}
			else if (strcmp(key, "pipeline") == 0)
			{
				EnsureTopLevelFieldType("$unionWith.pipeline", &pipelineIterator,
										BSON_TYPE_ARRAY);
				*pipeline = *value;
			}
			else
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_UNKNOWNBSONFIELD),
								errmsg(
									"The BSON field '$unionWith.%s' is not recognized as a valid field.",
									key),
								errdetail_log(
									"The BSON field '$unionWith.%s' is not recognized as a valid field.",
									key)));
			}
		}

		if (collectionFrom->length == 0 && pipeline->value_type == BSON_TYPE_EOD)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
							errmsg(
								"A $unionWith stage without specifying a target collection must begin its pipeline with a $documents stage.")));
		}
	}
	else
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg(
							"The $unionWith stage requires its specification to be either an object or a string, but instead encountered %s",
							BsonTypeName(existingValue->value_type)),
						errdetail_log(
							"The $unionWith stage requires its specification to be either an object or a string, but instead encountered %s",
							BsonTypeName(existingValue->value_type))));
	}
}


/*
 * Top level method handling processing of the $unionWith stage.
 */
Query *
HandleUnionWith(const bson_value_t *existingValue, Query *query,
				AggregationPipelineBuildContext *context)
{
	ReportFeatureUsage(FEATURE_STAGE_UNIONWITH);

	if (context->nestedPipelineLevel >= MaximumLookupPipelineDepth)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_MAXSUBPIPELINEDEPTHEXCEEDED),
						errmsg(
							"The allowed limit for nested sub-pipelines has been surpassed, exceeding the maximum of %d.",
							MaximumLookupPipelineDepth)));
	}

	Query *leftQuery = query;
	Query *rightQuery = NULL;

	StringView collectionFrom = { 0 };
	bson_value_t pipelineValue = { 0 };
	ParseUnionWith(existingValue, &collectionFrom, &pipelineValue);

	if (pipelineValue.value_type == BSON_TYPE_EOD)
	{
		AggregationPipelineBuildContext subPipelineContext = { 0 };
		subPipelineContext.nestedPipelineLevel = context->nestedPipelineLevel + 1;
		subPipelineContext.databaseNameDatum = context->databaseNameDatum;
		subPipelineContext.variableSpec = context->variableSpec;
		subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
		subPipelineContext.parentStageName = ParentStageName_UNIONWITH;
		strncpy((char *) subPipelineContext.collationString, context->collationString,
				MAX_ICU_COLLATION_LENGTH);

		/* This is unionWith on the base collection */
		pg_uuid_t *collectionUuid = NULL;
		bson_value_t *indexHint = NULL;
		rightQuery = GenerateBaseTableQuery(context->databaseNameDatum,
											&collectionFrom,
											collectionUuid,
											indexHint,
											&subPipelineContext);
	}
	else
	{
		AggregationPipelineBuildContext subPipelineContext = { 0 };
		subPipelineContext.nestedPipelineLevel = context->nestedPipelineLevel + 1;
		subPipelineContext.databaseNameDatum = context->databaseNameDatum;
		subPipelineContext.variableSpec = context->variableSpec;
		subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
		subPipelineContext.parentStageName = ParentStageName_UNIONWITH;
		strncpy((char *) subPipelineContext.collationString, context->collationString,
				MAX_ICU_COLLATION_LENGTH);

		pg_uuid_t *collectionUuid = NULL;

		if (collectionFrom.length == 0)
		{
			rightQuery = GenerateBaseAgnosticQuery(context->databaseNameDatum,
												   &subPipelineContext);
		}
		else
		{
			bson_value_t *indexHint = NULL;
			rightQuery = GenerateBaseTableQuery(context->databaseNameDatum,
												&collectionFrom,
												collectionUuid,
												indexHint,
												&subPipelineContext);
		}

		if (pipelineValue.value_type != BSON_TYPE_EOD)
		{
			bool hasCollection = collectionFrom.length != 0;
			ValidateUnionWithPipeline(&pipelineValue, hasCollection);

			List *stages = ExtractAggregationStages(&pipelineValue,
													&subPipelineContext);
			rightQuery = MutateQueryWithPipeline(rightQuery, stages,
												 &subPipelineContext);
		}
	}

	bool includeAllColumns = false;
	RangeTblEntry *leftRte = MakeSubQueryRte(leftQuery, context->stageNum, 0,
											 "unionLeft",
											 includeAllColumns);
	RangeTblEntry *rightRte = MakeSubQueryRte(rightQuery, context->stageNum, 0,
											  "unionRight",
											  includeAllColumns);

	Query *modifiedQuery = makeNode(Query);
	modifiedQuery->commandType = CMD_SELECT;
	modifiedQuery->querySource = query->querySource;
	modifiedQuery->canSetTag = true;
	modifiedQuery->jointree = makeNode(FromExpr);

	modifiedQuery->rtable = list_make2(leftRte, rightRte);
	RangeTblRef *leftReference = makeNode(RangeTblRef);
	leftReference->rtindex = 1;
	RangeTblRef *rightReference = makeNode(RangeTblRef);
	rightReference->rtindex = 2;

	SetOperationStmt *setOpStatement = MakeBsonSetOpStatement();
	setOpStatement->larg = (Node *) leftReference;
	setOpStatement->rarg = (Node *) rightReference;

	/* Update the query with the SetOp statement */
	modifiedQuery->setOperations = (Node *) setOpStatement;

	/* Result column node */
	TargetEntry *leftMostTargetEntry = linitial(leftQuery->targetList);
	Var *var = makeVar(1, leftMostTargetEntry->resno,
					   BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *restle = makeTargetEntry((Expr *) var,
										  leftMostTargetEntry->resno,
										  leftMostTargetEntry->resname,
										  false);
	modifiedQuery->targetList = list_make1(restle);
	modifiedQuery = MigrateQueryToSubQuery(modifiedQuery, context);

	return modifiedQuery;
}


/*
 * Validates the facet pipeline definition.
 */
static int
ValidateFacet(const bson_value_t *facetValue)
{
	if (facetValue->value_type != BSON_TYPE_DOCUMENT)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION15947),
						errmsg(
							"Expected 'document' type for $facet but found '%s' type",
							BsonTypeName(facetValue->value_type))));
	}

	bson_iter_t facetIter;
	BsonValueInitIterator(facetValue, &facetIter);

	int numStages = 0;
	while (bson_iter_next(&facetIter))
	{
		const char *key = bson_iter_key(&facetIter);
		const bson_value_t *pipeline = bson_iter_value(&facetIter);
		EnsureTopLevelFieldValueType("$facet.pipeline", pipeline, BSON_TYPE_ARRAY);

		numStages++;

		bson_iter_t pipelineArray;
		BsonValueInitIterator(pipeline, &pipelineArray);

		/* These stages are not allowed when executing $facet */
		while (bson_iter_next(&pipelineArray))
		{
			pgbsonelement stageElement = GetPipelineStage(&pipelineArray, "facet", key);
			const char *nestedPipelineStage = stageElement.path;
			if (strcmp(nestedPipelineStage, "$collStats") == 0 ||
				strcmp(nestedPipelineStage, "$facet") == 0 ||
				strcmp(nestedPipelineStage, "$geoNear") == 0 ||
				strcmp(nestedPipelineStage, "$indexStats") == 0 ||
				strcmp(nestedPipelineStage, "$out") == 0 ||
				strcmp(nestedPipelineStage, "$merge") == 0 ||
				strcmp(nestedPipelineStage, "$planCacheStats") == 0 ||
				strcmp(nestedPipelineStage, "$search") == 0 ||
				strcmp(nestedPipelineStage, "$changeStream") == 0)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40600),
								errmsg(
									"%s cannot be utilized within an operators facet processing stage",
									nestedPipelineStage),
								errdetail_log(
									"%s cannot be utilized within an operators facet processing stage",
									nestedPipelineStage)));
			}
		}
	}

	if (numStages == 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40169),
						errmsg(
							"No stages are specified for $facet")));
	}

	return numStages;
}


/*
 * Given a specific facet value that corresponds to an object with
 * one or more fields containing pipelines, creates a query that has
 * the UNION ALL SELECT operators of all those pipelines.
 */
static Query *
BuildFacetUnionAllQuery(int numStages, const bson_value_t *facetValue,
						CommonTableExpr *baseCte, QuerySource querySource,
						const bson_value_t *sortSpec,
						AggregationPipelineBuildContext *parentContext)
{
	Query *modifiedQuery = NULL;

	ParseState *parseState = make_parsestate(NULL);
	parseState->p_expr_kind = EXPR_KIND_SELECT_TARGET;
	parseState->p_next_resno = 1;

	/* Special case, if there's only 1 facet stage, we can just treat it as a SELECT from the CTE
	 * with a Subquery and skip all the UNION ALL processing.
	 */
	if (numStages == 1)
	{
		pgbsonelement singleElement = { 0 };
		BsonValueToPgbsonElement(facetValue, &singleElement);

		/* Creates a CTE that selects from the base CTE */

		/* Levels up is unused as we reset it after we build the aggregation and the final levelsup
		 * is determined. */
		int levelsUpUnused = 0;
		Query *baseQuery = CreateCteSelectQuery(baseCte, "facetsub",
												parentContext->stageNum,
												levelsUpUnused);

		/* Mutate the Query to add the aggregation pipeline */
		AggregationPipelineBuildContext nestedContext = { 0 };
		nestedContext.nestedPipelineLevel = parentContext->nestedPipelineLevel + 1;
		nestedContext.databaseNameDatum = parentContext->databaseNameDatum;
		nestedContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
		nestedContext.sortSpec = *sortSpec;
		nestedContext.variableSpec = parentContext->variableSpec;
		nestedContext.parentStageName = ParentStageName_FACET;
		strncpy((char *) nestedContext.collationString, parentContext->collationString,
				MAX_ICU_COLLATION_LENGTH);

		nestedContext.mongoCollection = parentContext->mongoCollection;
		nestedContext.collectionNameView = parentContext->collectionNameView;

		List *stages = ExtractAggregationStages(&singleElement.bsonValue,
												&nestedContext);
		modifiedQuery = MutateQueryWithPipeline(baseQuery, stages,
												&nestedContext);

		/* Add bson_array_agg to the output */
		bool migrateToSubQuery = true;
		Aggref **aggrefPtr = NULL;
		modifiedQuery = AddBsonArrayAggFunction(modifiedQuery, &nestedContext, parseState,
												singleElement.path,
												singleElement.pathLength,
												migrateToSubQuery, aggrefPtr);
	}
	else
	{
		/* We need yet another layer thats going to hold the Set UNION ALL queries */
		modifiedQuery = makeNode(Query);
		modifiedQuery->commandType = CMD_SELECT;
		modifiedQuery->querySource = querySource;
		modifiedQuery->canSetTag = true;
		modifiedQuery->jointree = makeNode(FromExpr);
		modifiedQuery->hasAggs = true;

		List *rangeTableReferences = NIL;
		bson_iter_t facetIterator;
		BsonValueInitIterator(facetValue, &facetIterator);
		int nestedStage = 1;

		Query *firstInnerQuery = NULL;
		Index firstRangeTableIndex = 0;
		while (bson_iter_next(&facetIterator))
		{
			StringView pathView = bson_iter_key_string_view(&facetIterator);

			/* Same drill - create a CTE Scan */

			/* Note that levelsUp is overridden after the query is built.
			 * In this path because we create a wrapper for the SetOperation, the
			 * prior stage query is at least 1 level up from each nested pipeline query.
			 * Since we attach the CTE to the query with the obj_agg, we add one more level
			 * to the CTE levels up.
			 */
			Query *baseQuery = CreateCteSelectQuery(baseCte, "facetsub", nestedStage,
													0);

			/* Modify the pipeline */
			AggregationPipelineBuildContext nestedContext = { 0 };
			nestedContext.nestedPipelineLevel = 1;
			nestedContext.databaseNameDatum = parentContext->databaseNameDatum;
			nestedContext.sortSpec = *sortSpec;
			nestedContext.variableSpec = parentContext->variableSpec;
			nestedContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
			nestedContext.parentStageName = ParentStageName_FACET;
			strncpy((char *) nestedContext.collationString,
					parentContext->collationString, MAX_ICU_COLLATION_LENGTH);
			nestedContext.mongoCollection = parentContext->mongoCollection;
			nestedContext.collectionNameView = parentContext->collectionNameView;
			nestedContext.nestedPipelineLevel = parentContext->nestedPipelineLevel + 1;

			List *stages = ExtractAggregationStages(bson_iter_value(&facetIterator),
													&nestedContext);
			Query *innerQuery = MutateQueryWithPipeline(baseQuery, stages,
														&nestedContext);

			/* Add the BSON_ARRAY_AGG function */
			bool migrateToSubQuery = true;
			Aggref **aggrefPtr = NULL;
			innerQuery = AddBsonArrayAggFunction(innerQuery, &nestedContext, parseState,
												 pathView.string, pathView.length,
												 migrateToSubQuery, aggrefPtr);

			/* Build the RTE for the output and add it to the SetOperation wrapper */
			bool includeAllColumns = false;
			RangeTblEntry *entry = MakeSubQueryRte(innerQuery, nestedStage, 0,
												   "facetinput",
												   includeAllColumns);
			modifiedQuery->rtable = lappend(modifiedQuery->rtable, entry);

			/* Track the RangeTableRef to this RTE (Will be used later) */
			RangeTblRef *innerRangeTableRef = makeNode(RangeTblRef);
			innerRangeTableRef->rtindex = list_length(modifiedQuery->rtable);
			if (firstInnerQuery == NULL)
			{
				firstInnerQuery = innerQuery;
				firstRangeTableIndex = innerRangeTableRef->rtindex;
			}

			rangeTableReferences = lappend(rangeTableReferences, innerRangeTableRef);
			nestedStage++;
		}


		/* now build the SetOperationStmt tree: This is the UNION ALL */
		int i = 0;
		SetOperationStmt *setOpStatement = MakeBsonSetOpStatement();
		bool insertLeft = false;
		for (i = 0; i < list_length(rangeTableReferences); i++)
		{
			if (setOpStatement->larg == NULL)
			{
				setOpStatement->larg = (Node *) list_nth(rangeTableReferences, i);
			}
			else if (setOpStatement->rarg == NULL)
			{
				setOpStatement->rarg = (Node *) list_nth(rangeTableReferences, i);
			}
			else if (insertLeft)
			{
				/* Both left and right are full and we want to add a node
				 * Flip flop between inserting on the left and right
				 */
				SetOperationStmt *newStatement = MakeBsonSetOpStatement();
				newStatement->larg = setOpStatement->larg;
				newStatement->rarg = (Node *) list_nth(rangeTableReferences, i);
				setOpStatement->larg = (Node *) newStatement;
				insertLeft = false;
			}
			else
			{
				SetOperationStmt *newStatement = MakeBsonSetOpStatement();
				newStatement->larg = setOpStatement->rarg;
				newStatement->rarg = (Node *) list_nth(rangeTableReferences, i);
				setOpStatement->rarg = (Node *) newStatement;
				insertLeft = true;
			}
		}

		/* Update the query with the SetOp statement */
		modifiedQuery->setOperations = (Node *) setOpStatement;

		/* Result column node */
		TargetEntry *leftMostTargetEntry = linitial(firstInnerQuery->targetList);
		Var *var = makeVar(firstRangeTableIndex,
						   leftMostTargetEntry->resno,
						   BsonTypeId(), -1, InvalidOid, 0);
		TargetEntry *restle = makeTargetEntry((Expr *) var,
											  leftMostTargetEntry->resno,
											  leftMostTargetEntry->resname,
											  false);
		modifiedQuery->targetList = list_make1(restle);
	}

	pfree(parseState);
	return modifiedQuery;
}


/*
 * Creates the COALESCE(Expr, '{ "field": [] }'::bson)
 * Expression for an arbitrary expression.
 */
static Expr *
GetArrayAggCoalesce(Expr *innerExpr, const char *fieldPath, uint32_t fieldPathLength)
{
	pgbson_writer defaultValueWriter;
	PgbsonWriterInit(&defaultValueWriter);
	PgbsonWriterAppendEmptyArray(&defaultValueWriter, fieldPath, fieldPathLength);

	pgbson *bson = PgbsonWriterGetPgbson(&defaultValueWriter);

	/* Add COALESCE operator */
	CoalesceExpr *coalesce = makeNode(CoalesceExpr);
	coalesce->coalescetype = BsonTypeId();
	coalesce->coalescecollid = InvalidOid;
	coalesce->args = list_make2(innerExpr, MakeBsonConst(bson));

	return (Expr *) coalesce;
}


/*
 * Creates the COALESCE(Expr, '{ }'::bson)
 * Expression for an arbitrary expression.
 */
static Expr *
GetEmptyBsonCoalesce(Expr *innerExpr)
{
	pgbson_writer defaultValueWriter;
	PgbsonWriterInit(&defaultValueWriter);

	pgbson *emptyBson = PgbsonWriterGetPgbson(&defaultValueWriter);

	/* Add COALESCE operator */
	CoalesceExpr *coalesce = makeNode(CoalesceExpr);
	coalesce->coalescetype = BsonTypeId();
	coalesce->coalescecollid = InvalidOid;
	coalesce->args = list_make2(innerExpr, MakeBsonConst(emptyBson));

	return (Expr *) coalesce;
}


/*
 * Adds either the BSON_ARRAY_AGG or bson_expression_get function to a given lookup right query.
 * Also migrates the existing query to a subquery if required.
 */
static Query *
AddLookupRightQueryExpressionOrArrayAgg(Query *rightQuery,
										AggregationPipelineBuildContext *context,
										ParseState *parseState, LookupArgs *lookupArgs,
										LookupContext *lookupContext,
										bool requiresSubQuery)
{
	Query *modifiedQuery = requiresSubQuery ? MigrateQueryToSubQuery(rightQuery,
																	 context) :
						   rightQuery;
	requiresSubQuery = false;
	if (lookupContext->isLookupUnwind)
	{
		/* Don't aggregate the documents inside array for lookUp + Unwind, later we use an specialized
		 * merge operation to add the right document into the left docuement under the lookupAs field.
		 */
		return modifiedQuery;
	}
	else
	{
		Aggref **aggrefPtr = NULL;
		return AddBsonArrayAggFunction(modifiedQuery, context, parseState,
									   lookupArgs->lookupAs.string,
									   lookupArgs->lookupAs.length, requiresSubQuery,
									   aggrefPtr);
	}
}


/*
 * Adds the BSON_ARRAY_AGG function to a given query. Also migrates the existing query
 * to a subquery if required.
 */
static Query *
AddBsonArrayAggFunction(Query *baseQuery, AggregationPipelineBuildContext *context,
						ParseState *parseState, const char *fieldPath,
						uint32_t fieldPathLength, bool migrateToSubQuery,
						Aggref **aggrefPtr)
{
	/* Now add the bson_array_agg function */
	Query *modifiedQuery = migrateToSubQuery ? MigrateQueryToSubQuery(baseQuery,
																	  context) :
						   baseQuery;

	/* The first projector is the document */
	TargetEntry *firstEntry = linitial(modifiedQuery->targetList);

	List *aggregateArgs = list_make2(
		firstEntry->expr,
		MakeTextConst(fieldPath, fieldPathLength));
	List *argTypesList = list_make2_oid(BsonTypeId(), TEXTOID);
	Aggref *aggref = CreateMultiArgAggregate(BsonArrayAggregateFunctionOid(),
											 aggregateArgs,
											 argTypesList, parseState);
	if (aggrefPtr != NULL)
	{
		*aggrefPtr = aggref;
	}

	firstEntry->expr = GetArrayAggCoalesce((Expr *) aggref, fieldPath, fieldPathLength);
	modifiedQuery->hasAggs = true;
	return modifiedQuery;
}


/*
 * Adds the BSON_OBJECT_AGG function to a given query.
 */
static Query *
AddBsonObjectAggFunction(Query *baseQuery, AggregationPipelineBuildContext *context)
{
	/* We replace the targetEntry so we're fine to take the resno 1 slot */
	ParseState *parseState = make_parsestate(NULL);
	parseState->p_expr_kind = EXPR_KIND_SELECT_TARGET;
	parseState->p_next_resno = 1;

	/* Better safe than sorry (Move it to a subquery) */
	Query *modifiedQuery = MigrateQueryToSubQuery(baseQuery, context);

	/* The first projector is the document */
	TargetEntry *firstEntry = linitial(modifiedQuery->targetList);

	/* Now add the bson_object_agg function */
	List *aggregateArgs = list_make1(firstEntry->expr);
	List *argTypesList = list_make1_oid(BsonTypeId());
	Aggref *aggref = CreateMultiArgAggregate(BsonObjectAggregateFunctionOid(),
											 aggregateArgs,
											 argTypesList, parseState);
	firstEntry->expr = (Expr *) aggref;
	modifiedQuery->hasAggs = true;

	pfree(parseState);
	return modifiedQuery;
}


/*
 * Updates an RTE with the CTE information
 */
static void
UpdateCteRte(RangeTblEntry *rte, CommonTableExpr *baseCte)
{
	Assert(rte->rtekind == RTE_CTE);
	rte->ctename = baseCte->ctename;
	rte->lateral = false;
	rte->inh = false;
	rte->inFromCl = true;

	List *colnames = NIL;
	List *coltypes = NIL;
	List *coltypmods = NIL;
	ListCell *cell;
	Query *baseQuery = (Query *) baseCte->ctequery;
	foreach(cell, baseQuery->targetList)
	{
		TargetEntry *tle = (TargetEntry *) lfirst(cell);
		colnames = lappend(colnames, makeString(tle->resname ? tle->resname : ""));
		coltypes = lappend_oid(coltypes, exprType((Node *) tle->expr));
		coltypmods = lappend_int(coltypmods, exprTypmod((Node *) tle->expr));
	}

	rte->eref = makeAlias(rte->alias->aliasname, colnames);
	rte->alias = makeAlias(rte->alias->aliasname, NIL);

	rte->coltypes = coltypes;
	rte->coltypmods = coltypmods;

	baseCte->ctecolnames = colnames;
	baseCte->ctecoltypes = coltypes;
	baseCte->ctecoltypmods = coltypmods;
}


static RangeTblEntry *
CreateCteRte(CommonTableExpr *baseCte, const char *prefix, int stageNum, int levelsUp)
{
	StringInfo s = makeStringInfo();
	appendStringInfo(s, "%s_stage_%d", prefix, stageNum);

	RangeTblEntry *rte = makeNode(RangeTblEntry);
	rte->rtekind = RTE_CTE;
	rte->ctelevelsup = levelsUp;
	rte->alias = makeAlias(s->data, NIL);
	baseCte->cterefcount++;
	rte->self_reference = false;
	UpdateCteRte(rte, baseCte);

	return rte;
}


/*
 * Creates a base query that selects from a given FuncExpr.
 */
static Query *
CreateFunctionSelectorQuery(FuncExpr *funcExpr, const char *prefix,
							int subStageNum, QuerySource querySource)
{
	StringInfo s = makeStringInfo();
	appendStringInfo(s, "%s_substage_%d", prefix, subStageNum);

	RangeTblFunction *tblFunction = makeNode(RangeTblFunction);
	tblFunction->funcexpr = (Node *) funcExpr;
	tblFunction->funccolcount = 1;
	tblFunction->funccoltypes = list_make1_oid(BsonTypeId());
	tblFunction->funccolcollations = list_make1_oid(InvalidOid);
	tblFunction->funccoltypmods = list_make1_int(-1);

	RangeTblEntry *rte = makeNode(RangeTblEntry);
	rte->rtekind = RTE_FUNCTION;
	rte->self_reference = false;
	rte->lateral = false;
	rte->inh = false;
	rte->inFromCl = true;
	rte->functions = list_make1(tblFunction);

	List *colnames = list_make1(makeString("lookup_unwind"));
	rte->alias = makeAlias(s->data, NIL);
	rte->eref = makeAlias(s->data, colnames);

	Var *queryOutput = makeVar(1, 1, funcExpr->funcresulttype,
							   -1, InvalidOid, 0);
	bool resJunk = false;
	TargetEntry *upperEntry = makeTargetEntry((Expr *) queryOutput, 1,
											  "funcName",
											  resJunk);

	Query *newquery = makeNode(Query);
	newquery->commandType = CMD_SELECT;
	newquery->querySource = querySource;
	newquery->canSetTag = true;
	newquery->targetList = list_make1(upperEntry);
	newquery->rtable = list_make1(rte);

	RangeTblRef *rtr = makeNode(RangeTblRef);
	rtr->rtindex = 1;
	newquery->jointree = makeFromExpr(list_make1(rtr), NULL);
	return newquery;
}


/*
 * Creates a base query that selects from a given CTE.
 */
static Query *
CreateCteSelectQuery(CommonTableExpr *baseCte, const char *prefix, int stageNum,
					 int levelsUp)
{
	Assert(baseCte->ctequery != NULL);
	Query *baseQuery = (Query *) baseCte->ctequery;

	RangeTblEntry *rte = CreateCteRte(baseCte, prefix, stageNum, levelsUp);

	List *upperTargetList = NIL;
	Index rtIndex = 1;
	ListCell *cell;
	foreach(cell, baseQuery->targetList)
	{
		TargetEntry *tle = (TargetEntry *) lfirst(cell);
		Var *newQueryOutput = makeVar(rtIndex, tle->resno, BsonTypeId(), -1,
									  InvalidOid, 0);
		TargetEntry *upperEntry = makeTargetEntry((Expr *) newQueryOutput, tle->resno,
												  tle->resname,
												  tle->resjunk);
		upperTargetList = lappend(upperTargetList, upperEntry);
	}

	Query *newquery = makeNode(Query);
	newquery->commandType = CMD_SELECT;
	newquery->querySource = baseQuery->querySource;
	newquery->canSetTag = true;
	newquery->targetList = upperTargetList;
	newquery->rtable = list_make1(rte);

	RangeTblRef *rtr = makeNode(RangeTblRef);
	rtr->rtindex = 1;
	newquery->jointree = makeFromExpr(list_make1(rtr), NULL);

	return newquery;
}


/*
 * Parses the lookup aggregation value and extracts the from collection and pipeline.
 */
void
LookupExtractCollectionAndPipeline(const bson_value_t *lookupValue,
								   StringView *collection, bson_value_t *pipeline)
{
	LookupArgs args;
	memset(&args, 0, sizeof(LookupArgs));
	ParseLookupStage(lookupValue, &args);
	*collection = args.from;
	*pipeline = args.pipeline;
}


/*
 * Parses the graphLookup aggregation value and extracts the from collection.
 */
void
GraphLookupExtractCollection(const bson_value_t *lookupValue, StringView *collection)
{
	GraphLookupArgs args;
	memset(&args, 0, sizeof(LookupArgs));
	ParseGraphLookupStage(lookupValue, &args);
	*collection = args.fromCollection;
}


/*
 * Parses & validates the input lookup spec.
 * Parsed outputs are placed in the LookupArgs struct.
 */
static void
ParseLookupStage(const bson_value_t *existingValue, LookupArgs *args)
{
	if (existingValue->value_type != BSON_TYPE_DOCUMENT)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40319),
						errmsg(
							"The $lookup stage must be defined as an object, but instead a %s value was provided.",
							BsonTypeName(existingValue->value_type))));
	}

	bson_iter_t lookupIter;
	BsonValueInitIterator(existingValue, &lookupIter);

	while (bson_iter_next(&lookupIter))
	{
		const char *key = bson_iter_key(&lookupIter);
		const bson_value_t *value = bson_iter_value(&lookupIter);
		if (strcmp(key, "as") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
								errmsg(
									"lookup argument 'as' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->lookupAs = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
		}
		else if (strcmp(key, "foreignField") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
								errmsg(
									"lookup argument 'foreignField' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->foreignField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
		}
		else if (strcmp(key, "from") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40321),
								errmsg(
									"The 'from' parameter in lookup must be provided as a string, but a value of type %s was given instead.",
									BsonTypeName(value->value_type))));
			}

			args->from = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
		}
		else if (strcmp(key, "let") == 0)
		{
			if (value->value_type != BSON_TYPE_DOCUMENT)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
								errmsg(
									"lookup argument 'let' must be a document, is type %s",
									BsonTypeName(value->value_type))));
			}

			/* let's use bson_dollar_project to evalute expression just exclude _id field */
			if (!IsBsonValueEmptyDocument(value))
			{
				args->let = PgbsonInitFromDocumentBsonValue(value);
			}
		}
		else if (strcmp(key, "localField") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
								errmsg(
									"lookup argument 'localField' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->localField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
		}
		else if (strcmp(key, "pipeline") == 0)
		{
			EnsureTopLevelFieldValueType("$lookup.pipeline", value, BSON_TYPE_ARRAY);
			args->pipeline = *value;

			bson_iter_t pipelineArray;
			BsonValueInitIterator(value, &pipelineArray);

			/* These stages are not allowed when executing $lookup */
			while (bson_iter_next(&pipelineArray))
			{
				pgbsonelement stageElement = GetPipelineStage(&pipelineArray, "lookup",
															  "pipeline");
				const char *nestedPipelineStage = stageElement.path;
				if (strcmp(nestedPipelineStage, "$out") == 0 ||
					strcmp(nestedPipelineStage, "$merge") == 0 ||
					strcmp(nestedPipelineStage, "$changeStream") == 0)
				{
					ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION51047),
									errmsg(
										"%s usage is prohibited inside a $lookup stage",
										nestedPipelineStage)));
				}
			}
		}
		else
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
							errmsg("Unrecognized parameter provided to $lookup: %s",
								   key)));
		}
	}

	if (args->lookupAs.length == 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg("must specify 'as' field for a $lookup")));
	}

	bool isPipelineLookup = args->pipeline.value_type != BSON_TYPE_EOD;
	if (args->from.length == 0 && !isPipelineLookup)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg("must specify 'from' field for a $lookup")));
	}

	if (!isPipelineLookup &&
		(args->foreignField.length == 0 || args->localField.length == 0))
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg(
							"$lookup needs to be provided with either a 'pipeline' definition or both 'localField' and 'foreignField' parameters explicitly")));
	}

	if ((args->foreignField.length == 0) ^ (args->localField.length == 0))
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg(
							"$lookup needs either both 'localField' and 'foreignField' specified or neither of them provided")));
	}

	if (args->foreignField.length != 0)
	{
		args->hasLookupMatch = true;
	}
}


/*
 * Helper method to create Lookup's JOIN RTE - this is the entry in the RTE
 * That goes in the Lookup's FROM clause and ties the two tables together.
 *
 * Note: useInnerJoin makes use of the INNER JOIN instead of LEFT JOIN
 */
inline static RangeTblEntry *
MakeLookupJoinRte(List *joinVars, List *colNames, List *joinLeftCols, List *joinRightCols,
				  bool useInnerJoin)
{
	/* Add an RTE for the JoinExpr */
	RangeTblEntry *joinRte = makeNode(RangeTblEntry);

	joinRte->rtekind = RTE_JOIN;
	joinRte->relid = InvalidOid;
	joinRte->subquery = NULL;
	joinRte->jointype = useInnerJoin ? JOIN_INNER : JOIN_LEFT;
	joinRte->joinmergedcols = 0; /* No using clause */
	joinRte->joinaliasvars = joinVars;
	joinRte->joinleftcols = joinLeftCols;
	joinRte->joinrightcols = joinRightCols;
	joinRte->join_using_alias = NULL;
	joinRte->alias = makeAlias("lookup_join", colNames);
	joinRte->eref = joinRte->alias;
	joinRte->inh = false;           /* never true for joins */
	joinRte->inFromCl = true;

#if PG_VERSION_NUM >= 160000
	joinRte->perminfoindex = 0;
#else
	joinRte->requiredPerms = 0;
	joinRte->checkAsUser = InvalidOid;
	joinRte->selectedCols = NULL;
	joinRte->insertedCols = NULL;
	joinRte->updatedCols = NULL;
	joinRte->extraUpdatedCols = NULL;
#endif
	return joinRte;
}


/*
 * Given a query in a specified RTE index that returns only BSON values, updates 3 lists based on the query:
 * 1) OutputVars are going to be Var nodes that point to the (RTE, Output) position
 * 2) The names of the output columns across the query
 * 3) The integer result numbers of the Vars.
 * Used to build the JoinRTE for lookup
 */
inline static void
MakeBsonJoinVarsFromQuery(Index queryIndex, Query *query, List **outputVars,
						  List **outputColNames, List **joinCols)
{
	ListCell *cell;
	foreach(cell, query->targetList)
	{
		TargetEntry *entry = lfirst(cell);

		Var *outputVar = makeVar(queryIndex, entry->resno, BsonTypeId(), -1, InvalidOid,
								 0);
		*outputVars = lappend(*outputVars, outputVar);
		*outputColNames = lappend(*outputColNames, makeString(entry->resname));
		*joinCols = lappend_int(*joinCols, (int) entry->resno);
	}
}


/*
 * Before applying the lookup query, parses the lookup args & context
 * and builds an optimization plan including how to join, how to apply the
 * pipeline and such.
 * This also handles splitting a lookup pipeline into pushdown and
 * non-pushdown etc.
 */
static void
OptimizeLookup(LookupArgs *lookupArgs,
			   Query *leftQuery,
			   AggregationPipelineBuildContext *leftQueryContext,
			   LookupOptimizationArgs *optimizationArgs)
{
	optimizationArgs->rightQueryContext.nestedPipelineLevel =
		leftQueryContext->nestedPipelineLevel + 1;
	optimizationArgs->rightQueryContext.databaseNameDatum =
		leftQueryContext->databaseNameDatum;
	optimizationArgs->rightQueryContext.variableSpec = leftQueryContext->variableSpec;
	strncpy((char *) optimizationArgs->rightQueryContext.collationString,
			leftQueryContext->collationString, MAX_ICU_COLLATION_LENGTH);
	optimizationArgs->rightQueryContext.parentStageName = ParentStageName_LOOKUP;
	optimizationArgs->rightQueryContext.optimizePipelineStages =
		leftQueryContext->optimizePipelineStages;
	optimizationArgs->rightQueryContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;

	optimizationArgs->isLookupAgnostic = lookupArgs->from.length == 0;
	optimizationArgs->isLookupUncorrelated = !lookupArgs->hasLookupMatch;

	bson_value_t inlinedLookupPipeline = (bson_value_t) {
		0
	};
	bson_value_t nonInlinedLookupPipeline = (bson_value_t) {
		0
	};

	if (leftQueryContext->variableSpec != NULL &&
		!IsA(leftQueryContext->variableSpec, Const))
	{
		optimizationArgs->hasLet = true;
	}

	if (lookupArgs->let)
	{
		/* Validate the lookupArgs->let if the left query had no let variables specified in it. */
		/* Time system variables are stored in the left query's variableSpec even in the absence of let. */
		/* Thus, if the left query's variableSpec contains only the time system variables, */
		/* we perform a validation on the lookupArgs->let. */
		if (leftQueryContext->variableSpec == NULL)
		{
			bson_value_t varsValue = ConvertPgbsonToBsonValue(lookupArgs->let);
			ExpressionVariableContext *nullContext = NULL;

			ParseAggregationExpressionContext parseContext = {
				.validateParsedExpressionFunc = &ValidateVariableIsDefined,
			};

			ParseVariableSpec(&varsValue, nullContext, &parseContext);
		}
		else if (IsA(leftQueryContext->variableSpec, Const))
		{
			Node *specNode = (Node *) leftQueryContext->variableSpec;
			Const *specConst = (Const *) specNode;
			pgbson *specBson = DatumGetPgBson(specConst->constvalue);

			bson_iter_t iter;
			if (!PgbsonInitIteratorAtPath(specBson, "let", &iter))
			{
				bson_value_t varsValue = ConvertPgbsonToBsonValue(lookupArgs->let);
				ExpressionVariableContext *nullContext = NULL;

				ParseAggregationExpressionContext parseContext = {
					.validateParsedExpressionFunc = &ValidateVariableIsDefined,
				};

				GetTimeSystemVariablesFromVariableSpec(specBson,
													   &parseContext.
													   timeSystemVariables);

				ParseVariableSpec(&varsValue, nullContext, &parseContext);
			}
		}

		/* If there is lookup let, we will build the let in the left query and then the let expression
		 * should be used by the right query / post join stages wherever needed.
		 */
		Const *letConstValue = MakeBsonConst(lookupArgs->let);

		List *args;
		Oid funcOid;
		Expr *sourceVariableSpec = leftQueryContext->variableSpec;
		if (sourceVariableSpec == NULL)
		{
			sourceVariableSpec = (Expr *) MakeBsonConst(PgbsonInitEmpty());
		}

		Expr *documentExpr = linitial_node(TargetEntry, leftQuery->targetList)->expr;
		args = list_make3(documentExpr, letConstValue, sourceVariableSpec);
		funcOid = BsonDollarLookupExpressionEvalMergeOid();

		Expr *letExpr = (Expr *) makeFuncExpr(funcOid, BsonTypeId(), args, InvalidOid,
											  InvalidOid, COERCE_EXPLICIT_CALL);

		optimizationArgs->lookupLetAttrNum = list_length(leftQuery->targetList) + 1;
		TargetEntry *lookupLetEntry = makeTargetEntry(letExpr,
													  optimizationArgs->lookupLetAttrNum,
													  "let", false);
		leftQuery->targetList = lappend(leftQuery->targetList, lookupLetEntry);

		/*
		 * Make a VAR to access the above let in right query.
		 * Today the right query is pushed to a CTE for and this Var is
		 * used to support special case non-inline stages i.e. $match
		 * so varlevelsup is 1
		 */
		Index varLevelsUp = 1;
		int leftQueryVarNo = 1;
		Expr *letVarExpr = (Expr *) makeVar(leftQueryVarNo,
											optimizationArgs->lookupLetAttrNum,
											BsonTypeId(), -1, InvalidOid, varLevelsUp);
		optimizationArgs->rightQueryContext.variableSpec = letVarExpr;
		optimizationArgs->hasLet = true;
	}

	if (optimizationArgs->hasLet)
	{
		optimizationArgs->isLookupUncorrelated = false;

		if (optimizationArgs->isLookupAgnostic)
		{
			/* TODO Support agnostic queries with lookup with let */
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_COMMANDNOTSUPPORTED),
							errmsg(
								"$lookup with let with agnostic queries not supported yet")));
		}
	}

	/* For the right query, generate a base table query for the right collection */
	pg_uuid_t *collectionUuid = NULL;
	bson_value_t *indexHint = NULL;
	optimizationArgs->rightBaseQuery =
		optimizationArgs->isLookupAgnostic ?
		GenerateBaseAgnosticQuery(
			optimizationArgs->rightQueryContext.databaseNameDatum,
			&optimizationArgs->rightQueryContext) :
		GenerateBaseTableQuery(
			optimizationArgs->rightQueryContext.databaseNameDatum,
			&lookupArgs->from,
			collectionUuid,
			indexHint,
			&optimizationArgs->rightQueryContext);

	/* Now let's figure out if we can join on _id */
	optimizationArgs->isLookupJoinOnRightId =
		StringViewEquals(&lookupArgs->foreignField, &IdFieldStringView) &&
		!optimizationArgs->isLookupAgnostic;

	if (IsCollationApplicable(leftQueryContext->collationString))
	{
		/* Can't perform the _id join when collation is applicable. The optimization
		 * joins on the physical id via byte-wise equality, which is not
		 * collation-aware, so an _id that holds collation-sensitive values (UTF8
		 * strings or documents) would incorrectly drop matches that the collation
		 * considers equal. Fall back to the collation-aware filter path. */
		optimizationArgs->isLookupJoinOnRightId = false;
	}

	if (optimizationArgs->isLookupJoinOnRightId &&
		list_length(optimizationArgs->rightBaseQuery->rtable) == 1 &&
		list_length(optimizationArgs->rightBaseQuery->targetList) == 1 &&
		optimizationArgs->rightQueryContext.mongoCollection != NULL &&
		optimizationArgs->rightQueryContext.mongoCollection->shardKey == NULL)
	{
		RangeTblEntry *entry = linitial(optimizationArgs->rightBaseQuery->rtable);
		TargetEntry *firstEntry = linitial(optimizationArgs->rightBaseQuery->targetList);
		if (entry->rtekind != RTE_RELATION || !IsA(firstEntry->expr, Var))
		{
			/* These cases can't do lookup on _id */
			optimizationArgs->isLookupJoinOnRightId = false;
		}
	}
	else
	{
		/* Not a single RTE, or is a sharded collection or a collection that doesn't exist
		 * Can't do _id optimization.
		 */
		optimizationArgs->isLookupJoinOnRightId = false;
	}

	/* No point in inlining lookup pipeline on agnostic - it Has to be applied */
	if (lookupArgs->pipeline.value_type == BSON_TYPE_EOD)
	{
		inlinedLookupPipeline = lookupArgs->pipeline;
		nonInlinedLookupPipeline = (bson_value_t) {
			0
		};
	}
	else if (IsBsonValueEmptyArray(&lookupArgs->pipeline))
	{
		inlinedLookupPipeline = lookupArgs->pipeline;
		nonInlinedLookupPipeline = (bson_value_t) {
			0
		};
	}
	else if (optimizationArgs->isLookupUncorrelated ||
			 optimizationArgs->isLookupAgnostic)
	{
		inlinedLookupPipeline = lookupArgs->pipeline;
		nonInlinedLookupPipeline = (bson_value_t) {
			0
		};
	}
	else if (optimizationArgs->isLookupJoinOnRightId)
	{
		/* In the cases where we are joining on _id, we don't want to apply the pipeline here.
		 * This is because it's better to join on _id (index pushdown) then apply the pipeline on
		 * the result.
		 */
		inlinedLookupPipeline = (bson_value_t) {
			0
		};
		nonInlinedLookupPipeline = lookupArgs->pipeline;
	}
	else
	{
		StringView lookupJoinPipelineField = lookupArgs->foreignField;

		StringView prefix = StringViewFindPrefix(&lookupJoinPipelineField, '.');
		if (prefix.length != 0)
		{
			lookupJoinPipelineField = prefix;
		}

		bool isPipelineValid = false;
		pgbson *inlinedPipeline = NULL;
		pgbson *nonInlinedPipeline = NULL;
		bool canInlinePipelineCore =
			CanInlineLookupPipeline(&lookupArgs->pipeline, &lookupJoinPipelineField,
									optimizationArgs->hasLet,
									&inlinedPipeline, &nonInlinedPipeline,
									&isPipelineValid);
		if (!isPipelineValid)
		{
			/* Invalid pipeline - just make it inlined to trigger the error */
			inlinedLookupPipeline = lookupArgs->pipeline;
			nonInlinedLookupPipeline = (bson_value_t) {
				0
			};
		}
		else if (canInlinePipelineCore)
		{
			/* The full pipeline can be inlined */
			inlinedLookupPipeline = lookupArgs->pipeline;
			nonInlinedLookupPipeline = (bson_value_t) {
				0
			};
		}
		else
		{
			pgbsonelement pipelineElement = { 0 };
			if (!IsPgbsonEmptyDocument(inlinedPipeline))
			{
				PgbsonToSinglePgbsonElement(inlinedPipeline, &pipelineElement);
				if (!IsBsonValueEmptyArray(&pipelineElement.bsonValue))
				{
					inlinedLookupPipeline = pipelineElement.bsonValue;
				}
			}

			if (!IsPgbsonEmptyDocument(nonInlinedPipeline))
			{
				PgbsonToSinglePgbsonElement(nonInlinedPipeline, &pipelineElement);
				if (!IsBsonValueEmptyArray(&pipelineElement.bsonValue))
				{
					nonInlinedLookupPipeline = pipelineElement.bsonValue;
				}
			}
		}
	}

	optimizationArgs->isLookupJoinOnLeftId = false;
	if (optimizationArgs->isLookupJoinOnRightId &&
		StringViewEquals(&lookupArgs->localField, &IdFieldStringView) &&
		list_length(leftQuery->rtable) == 1 &&
		list_length(leftQuery->targetList) == 1 &&
		leftQueryContext->mongoCollection != NULL &&
		leftQueryContext->mongoCollection->shardKey == NULL)
	{
		RangeTblEntry *entry = linitial(leftQuery->rtable);
		TargetEntry *firstEntry = linitial(leftQuery->targetList);
		if (entry->rtekind == RTE_RELATION && IsA(firstEntry->expr, Var))
		{
			/*
			 *  These cases can't do lookup on _id on the left.
			 *
			 *  Additionally, optimizationArgs->isLookupJoinOnLeftId also needs
			 *  to be set to `false` when collation is applicable. But, if that's
			 *  the case optimizationArgs->isLookupJoinOnRightId will be already
			 *  set to false and optimizationArgs->isLookupJoinOnLeftId will remain
			 *  at default `false`.
			 */
			optimizationArgs->isLookupJoinOnLeftId = true;
		}
	}

	/* Last check - nested $lookup with let not supported on sharded (citus limit) */
	if (optimizationArgs->hasLet &&
		leftQueryContext->mongoCollection != NULL &&
		leftQueryContext->mongoCollection->shardKey != NULL &&
		lookupArgs->pipeline.value_type != BSON_TYPE_EOD)
	{
		ValidatePipelineForShardedLookupWithLet(&lookupArgs->pipeline);
	}

	/*
	 * Extract the inline and non inline pipeline stages after everythin is done.
	 */
	optimizationArgs->inlinedPipelineStages = ExtractAggregationStages(
		&inlinedLookupPipeline, &optimizationArgs->rightQueryContext);
	optimizationArgs->nonInlinedPipelineStages = ExtractAggregationStages(
		&nonInlinedLookupPipeline, &optimizationArgs->rightQueryContext);

	/*
	 * If the first non-inline stages is a $match (which contains let and $expr),
	 * and we don't have a better match plan e.g. join on righ collection _id then
	 * we need to handle it similar to hasLookupMatch
	 */
	Stage firstNonInlineStage = GetAggregationStageAtPosition(
		optimizationArgs->nonInlinedPipelineStages, 0);
	if (!optimizationArgs->isLookupJoinOnRightId &&
		firstNonInlineStage == Stage_Match)
	{
		optimizationArgs->nonInlinedMatchStage = (AggregationStage *) linitial(
			optimizationArgs->nonInlinedPipelineStages);
		optimizationArgs->nonInlinedPipelineStages =
			list_delete_first(optimizationArgs->nonInlinedPipelineStages);
	}
}


/*
 * Core JOIN building logic for $lookup. See comments within on logic
 * of each step within.
 */
static Query *
ProcessLookupCoreWithLet(Query *query, AggregationPipelineBuildContext *context,
						 LookupArgs *lookupArgs, LookupContext *lookupContext)
{
	if (list_length(query->targetList) > 1)
	{
		/* if we have multiple projectors, push to a subquery (Lookup needs 1 projector) */
		/* TODO: Can we do away with this */
		query = MigrateQueryToSubQuery(query, context);
	}

	/* The left query is just the base query */
	Query *leftQuery = query;

	LookupOptimizationArgs optimizationArgs = { 0 };
	OptimizeLookup(lookupArgs, leftQuery, context, &optimizationArgs);

	/* Generate the lookup query */
	/* Start with a fresh query */
	Query *lookupQuery = makeNode(Query);
	lookupQuery->commandType = CMD_SELECT;
	lookupQuery->querySource = query->querySource;
	lookupQuery->canSetTag = true;
	lookupQuery->jointree = makeNode(FromExpr);

	/* Mark that we're adding a nesting level */
	context->numNestedLevels++;

	const Index leftQueryRteIndex = 1;
	const Index rightQueryRteIndex = 2;
	const Index joinQueryRteIndex = 3;

	Query *rightQuery = optimizationArgs.rightBaseQuery;

	/* Create a parse_state for this session */
	ParseState *parseState = make_parsestate(NULL);
	parseState->p_expr_kind = EXPR_KIND_SELECT_TARGET;
	parseState->p_next_resno = 1;

	/* Process the right query where possible */
	if (list_length(optimizationArgs.inlinedPipelineStages) > 0)
	{
		rightQuery = MutateQueryWithPipeline(rightQuery,
											 optimizationArgs.inlinedPipelineStages,
											 &optimizationArgs.rightQueryContext);
	}

	if (list_length(rightQuery->targetList) > 1 ||
		optimizationArgs.rightQueryContext.requiresSubQueryAfterProject ||
		optimizationArgs.rightQueryContext.requiresSubQuery)
	{
		rightQuery = MigrateQueryToSubQuery(rightQuery,
											&optimizationArgs.rightQueryContext);
	}

	/* Check if the pipeline can be pushed to the inner query (right collection)
	 * If it can, then it's inlined. If not, we apply the pipeline post-join.
	 */
	if (lookupArgs->hasLookupMatch || optimizationArgs.nonInlinedMatchStage != NULL)
	{
		/* We can apply the optimization on this based on object_id if and only if
		 * The right table is pointing directly to an actual table (not a view)
		 * and we're an unsharded collection - or a view that just does a "filter"
		 * match.
		 */
		if (optimizationArgs.isLookupJoinOnRightId)
		{
			PG_USED_FOR_ASSERTS_ONLY RangeTblEntry *entry = linitial(rightQuery->rtable);
			PG_USED_FOR_ASSERTS_ONLY TargetEntry *firstEntry = linitial(
				rightQuery->targetList);

			/* Add the document object_id projector as well, if there is no projection on the document && it's a base table
			 * in the case of projections we can't be sure something like { "_id": "abc" } has been added
			 */
			Assert(entry->rtekind == RTE_RELATION && IsA(firstEntry->expr, Var));

			/* Add the object_id targetEntry */
			Var *objectIdVar = makeVar(1, DOCUMENT_DATA_TABLE_OBJECT_ID_VAR_ATTR_NUMBER,
									   BsonTypeId(), -1, InvalidOid, 0);
			TargetEntry *objectEntry = makeTargetEntry((Expr *) objectIdVar,
													   list_length(
														   rightQuery->targetList) +
													   1, "objectId", false);
			rightQuery->targetList = lappend(rightQuery->targetList, objectEntry);

			if (ShouldSkipShardKeyFilterOnBaseTable(&optimizationArgs.rightQueryContext))
			{
				/* Add the shard key filter to the right query. In the path where we are
				 * in ShouldSkipShardKeyFilterOnBaseTable, the base table doesn't automatically
				 * get the shard key and since we have a lookup right query on _id, we need to
				 * explicitly inject the shard_key_value here to use the _id index.
				 */
				Expr *zeroShardKeyFilter = CreateNonShardedShardKeyValueFilter(
					1,
					optimizationArgs.rightQueryContext.mongoCollection);
				if (rightQuery->jointree->quals == NULL)
				{
					rightQuery->jointree->quals = (Node *) zeroShardKeyFilter;
				}
				else
				{
					List *newQuals = make_ands_implicit(
						(Expr *) rightQuery->jointree->quals);
					newQuals = lappend(newQuals, zeroShardKeyFilter);
					rightQuery->jointree->quals = (Node *) make_ands_explicit(newQuals);
				}
			}
		}

		CommonTableExpr *rightTableExpr = makeNode(CommonTableExpr);
		rightTableExpr->ctename = "lookup_right_query";
		rightTableExpr->ctequery = (Node *) rightQuery;

		rightQuery = CreateCteSelectQuery(rightTableExpr, "lookup_right_query",
										  context->nestedPipelineLevel, 0);

		if (lookupArgs->hasLookupMatch)
		{
			/* It's a join on a field, first add a TargetEntry for left for bson_dollar_lookup_extract_filter_expression */
			TargetEntry *currentEntry = linitial(leftQuery->targetList);

			/* The extract query right arg is a simple bson of the form { "remoteField": "localField" } */
			pgbson_writer filterWriter;
			PgbsonWriterInit(&filterWriter);
			PgbsonWriterAppendUtf8(&filterWriter, lookupArgs->foreignField.string,
								   lookupArgs->foreignField.length,
								   lookupArgs->localField.string);

			if (IsCollationApplicable(context->collationString))
			{
				PgbsonWriterAppendUtf8(&filterWriter, "collation", 9,
									   context->collationString);
			}

			pgbson *filterBson = PgbsonWriterGetPgbson(&filterWriter);

			/* Create the bson_dollar_lookup_extract_filter_expression(document, 'filter') */
			List *extractFilterArgs = list_make2(currentEntry->expr, MakeBsonConst(
													 filterBson));

			Expr *projectorFunc;
			if (optimizationArgs.isLookupJoinOnLeftId)
			{
				/* If we can join on the left _id, then just use object_id */
				projectorFunc = (Expr *) makeVar(1,
												 DOCUMENT_DATA_TABLE_OBJECT_ID_VAR_ATTR_NUMBER,
												 BsonTypeId(), -1, InvalidOid, 0);
			}
			else if (optimizationArgs.isLookupJoinOnRightId)
			{
				projectorFunc = (Expr *) makeFuncExpr(
					BsonLookupExtractFilterArrayFunctionOid(), GetBsonArrayTypeOid(),
					extractFilterArgs,
					InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL);
			}
			else
			{
				Oid extractFunctionOid =
					DocumentDBApiInternalBsonLookupExtractFilterExpressionFunctionOid();

				projectorFunc = (Expr *) makeFuncExpr(
					extractFunctionOid, BsonTypeId(),
					extractFilterArgs,
					InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL);
			}

			AttrNumber newProjectorAttrNum = list_length(leftQuery->targetList) + 1;
			TargetEntry *extractFilterProjector = makeTargetEntry((Expr *) projectorFunc,
																  newProjectorAttrNum,
																  "lookup_filter", false);
			leftQuery->targetList = lappend(leftQuery->targetList,
											extractFilterProjector);

			/* now on the right query, add a filter referencing this projector */
			List *rightQuals = NIL;
			if (rightQuery->jointree->quals != NULL)
			{
				rightQuals = make_ands_implicit((Expr *) rightQuery->jointree->quals);
			}

			/* add the WHERE bson_dollar_in(t2.document, t1.match) */
			TargetEntry *currentRightEntry = linitial(rightQuery->targetList);
			Var *rightVar = (Var *) currentRightEntry->expr;
			int matchLevelsUp = 1;
			Node *inClause;
			if (optimizationArgs.isLookupJoinOnLeftId)
			{
				/* Do a direct equality against object_id */
				Assert(list_length(rightQuery->targetList) == 2);
				TargetEntry *rightObjectIdEntry = (TargetEntry *) lsecond(
					rightQuery->targetList);
				rightQuery->targetList = list_make1(currentRightEntry);

				Var *matchVar = makeVar(leftQueryRteIndex, newProjectorAttrNum,
										BsonTypeId(), -1,
										InvalidOid, matchLevelsUp);
				inClause = (Node *) make_opclause(BsonEqualOperatorId(), BOOLOID, false,
												  copyObject(
													  rightObjectIdEntry->expr),
												  (Expr *) matchVar, InvalidOid,
												  InvalidOid);
			}
			else if (optimizationArgs.isLookupJoinOnRightId)
			{
				Assert(list_length(rightQuery->targetList) == 2);
				TargetEntry *rightObjectIdEntry = (TargetEntry *) lsecond(
					rightQuery->targetList);
				rightQuery->targetList = list_make1(currentRightEntry);

				ScalarArrayOpExpr *inOperator = makeNode(ScalarArrayOpExpr);
				inOperator->useOr = true;
				inOperator->opno = BsonEqualOperatorId();
				inOperator->opfuncid = BsonEqualFunctionOid();
				Var *matchVar = makeVar(leftQueryRteIndex, newProjectorAttrNum,
										GetBsonArrayTypeOid(), -1,
										InvalidOid, matchLevelsUp);
				List *inArgs = list_make2(copyObject(rightObjectIdEntry->expr), matchVar);
				inOperator->args = inArgs;
				inClause = (Node *) inOperator;
			}
			else
			{
				Var *matchVar = makeVar(leftQueryRteIndex, newProjectorAttrNum,
										BsonTypeId(),
										-1,
										InvalidOid, matchLevelsUp);
				Const *textConst = MakeTextConst(lookupArgs->foreignField.string,
												 lookupArgs->foreignField.length);
				List *inArgs = list_make3(copyObject(rightVar), matchVar, textConst);
				inClause = (Node *) makeFuncExpr(BsonDollarLookupJoinFilterFunctionOid(),
												 BOOLOID, inArgs,
												 InvalidOid, InvalidOid,
												 COERCE_EXPLICIT_CALL);
			}

			if (rightQuals == NIL)
			{
				rightQuery->jointree->quals = inClause;
			}
			else
			{
				rightQuals = lappend(rightQuals, inClause);
				rightQuery->jointree->quals = (Node *) make_ands_explicit(rightQuals);
			}
		}

		/*
		 * Add the $match with $expr to the right query
		 */
		if (optimizationArgs.nonInlinedMatchStage != NULL)
		{
			HandleMatch(&optimizationArgs.nonInlinedMatchStage->stageValue, rightQuery,
						&optimizationArgs.rightQueryContext);
		}

		/* Add the bson_array_agg function */
		bool requiresSubQuery = false;
		rightQuery = AddLookupRightQueryExpressionOrArrayAgg(rightQuery,
															 &optimizationArgs.
															 rightQueryContext,
															 parseState,
															 lookupArgs,
															 lookupContext,
															 requiresSubQuery);
		rightQuery->cteList = list_make1(rightTableExpr);
	}
	else
	{
		/* If lookup is purely a pipeline (uncorrelated subquery) then
		 * modify the pipeline. */
		bool migrateToSubQuery = false;
		rightQuery = AddLookupRightQueryExpressionOrArrayAgg(rightQuery,
															 &optimizationArgs.
															 rightQueryContext,
															 parseState,
															 lookupArgs,
															 lookupContext,
															 migrateToSubQuery);
	}

	/* Due to citus query_pushdown_planning.JoinTreeContainsSubqueryWalker
	 * we can't just use SubQueries (however, CTEs work). So move the left
	 * query is pushed to a CTE which seems to work (Similar to what the GW)
	 * does.
	 */
	StringInfo cteStr = makeStringInfo();
	appendStringInfo(cteStr, "lookupLeftCte_%d", context->nestedPipelineLevel);
	CommonTableExpr *cteExpr = makeNode(CommonTableExpr);
	cteExpr->ctename = cteStr->data;
	cteExpr->ctequery = (Node *) leftQuery;
	lookupQuery->cteList = lappend(lookupQuery->cteList, cteExpr);

	int stageNum = 1;
	RangeTblEntry *leftTree = CreateCteRte(cteExpr, "lookup", stageNum, 0);

	/* The "from collection" becomes another RTE */
	bool includeAllColumns = true;
	RangeTblEntry *rightTree = MakeSubQueryRte(rightQuery, 1,
											   context->nestedPipelineLevel,
											   "lookupRight", includeAllColumns);

	/* Mark the Right RTE as a lateral join*/
	rightTree->lateral = true;

	/* Build the JOIN RTE joining the left and right RTEs */
	List *outputVars = NIL;
	List *outputColNames = NIL;
	List *leftJoinCols = NIL;
	List *rightJoinCols = NIL;
	MakeBsonJoinVarsFromQuery(leftQueryRteIndex, leftQuery, &outputVars, &outputColNames,
							  &leftJoinCols);
	MakeBsonJoinVarsFromQuery(rightQueryRteIndex, rightQuery, &outputVars,
							  &outputColNames, &rightJoinCols);

	bool useInnerJoin = !lookupContext->preserveNullAndEmptyArrays;
	RangeTblEntry *joinRte = MakeLookupJoinRte(outputVars, outputColNames, leftJoinCols,
											   rightJoinCols, useInnerJoin);


	lookupQuery->rtable = list_make3(leftTree, rightTree, joinRte);

	/* Now specify the "From" as a join */
	/* The query has a single 'FROM' which is a Join */
	JoinExpr *joinExpr = makeNode(JoinExpr);
	joinExpr->jointype = joinRte->jointype;
	joinExpr->rtindex = joinQueryRteIndex;

	/* Create RangeTblRef's to point to the left & right RTEs */
	RangeTblRef *leftRef = makeNode(RangeTblRef);
	leftRef->rtindex = leftQueryRteIndex;
	RangeTblRef *rightRef = makeNode(RangeTblRef);
	rightRef->rtindex = rightQueryRteIndex;
	joinExpr->larg = (Node *) leftRef;
	joinExpr->rarg = (Node *) rightRef;

	/* Join ON TRUE */
	joinExpr->quals = (Node *) makeConst(BOOLOID, -1, InvalidOid, 1, BoolGetDatum(true),
										 false, true);

	lookupQuery->jointree->fromlist = list_make1(joinExpr);

	/* Now add the lookup projector */
	/* The lookup projector is an addFields on the left doc */
	/* And the rightArg will be an addFieldsSpec already under a bson_array_agg(value, 'asField') */
	Expr *leftOutput = (Expr *) makeVar(leftRef->rtindex, (AttrNumber) 1, BsonTypeId(),
										-1,
										InvalidOid, 0);
	Expr *rightOutput = (Expr *) makeVar(rightRef->rtindex, (AttrNumber) 1, BsonTypeId(),
										 -1,
										 InvalidOid, 0);

	TargetEntry *leftTargetEntry = makeTargetEntry(leftOutput, 1, "left", false);
	TargetEntry *rightTargetEntry = makeTargetEntry(rightOutput, 2, "right", false);
	lookupQuery->targetList = list_make2(leftTargetEntry, rightTargetEntry);

	/* handle the post non-inlined query */

	/* If there's a pipeline, run it here as a subquery only if we
	 * couldn't inline it in the top level query.
	 * We do it here since it's easier to deal with the pipeline
	 * post join (fewer queries to think about and manage).
	 */
	if (list_length(optimizationArgs.nonInlinedPipelineStages) > 0)
	{
		resetStringInfo(cteStr);
		appendStringInfo(cteStr, "lookup_join_cte_%d", context->nestedPipelineLevel);
		CommonTableExpr *lookupCte = makeNode(CommonTableExpr);
		lookupCte->ctename = cteStr->data;
		lookupCte->ctequery = (Node *) lookupQuery;

		/*
		 * Before creating the CTE - add the $let from left query to be used by post join pipeline
		 * It's a deliberate choice to rebuild the let expression here instead of reusing the one that is available
		 * because the expressions are not CONST and can pose problems if not handled properly.
		 */
		if (lookupArgs->let)
		{
			/* Evaluate the let against the leftExpr */
			/* Let Expression Evaluation here */
			Const *letConstValue = MakeBsonConst(lookupArgs->let);

			List *args;
			Oid funcOid;
			Expr *sourceVariableSpec = context->variableSpec;
			if (sourceVariableSpec == NULL)
			{
				sourceVariableSpec = (Expr *) MakeBsonConst(PgbsonInitEmpty());
			}

			args = list_make3(leftOutput, letConstValue, sourceVariableSpec);
			funcOid = BsonDollarLookupExpressionEvalMergeOid();

			Expr *letExpr = (Expr *) makeFuncExpr(funcOid, BsonTypeId(), args, InvalidOid,
												  InvalidOid, COERCE_EXPLICIT_CALL);
			TargetEntry *lookupLetEntry = makeTargetEntry(letExpr, 3, "let", false);
			lookupQuery->targetList = lappend(lookupQuery->targetList, lookupLetEntry);
		}

		Query *cteLookupQuery = CreateCteSelectQuery(lookupCte, "lookup_non_inlined", 1,
													 0);
		cteLookupQuery->cteList = list_make1(lookupCte);

		TargetEntry *firstEntry = linitial(cteLookupQuery->targetList);
		TargetEntry *secondEntry = lsecond(cteLookupQuery->targetList);
		Var *secondVar = (Var *) copyObject(secondEntry->expr);

		/* varlevelsup is set to 0 and once the complete pipeline building is done, levelsup is adjusted again.
		 * If it is set to anything as a positive value here then we will end up with wrong expectation of levelsup while
		 * creating $group Aggregate queries within pipeline.
		 * Refer: check_agglevels_and_constraints in parse_aggs.c
		 * https://github.com/postgres/postgres/blob/2e66cae935c2e0f7ce9bab6b65ddeb7806f4de7c/src/backend/parser/parse_agg.c#L347C2-L349C27
		 */
		secondVar->varlevelsup = 0;
		secondVar->location = NESTED_PIPELINE_VAR_FLAG | context->nestedPipelineLevel;

		Expr *letExpr = NULL;
		Var *letVar = NULL;
		if (lookupArgs->let)
		{
			letExpr = copyObject(lthird_node(TargetEntry,
											 cteLookupQuery->targetList)->expr);
			letVar = (Var *) letExpr;
			letVar->varlevelsup = 0;
			letVar->location = NESTED_PIPELINE_VAR_FLAG | context->nestedPipelineLevel;
		}
		else
		{
			letExpr = context->variableSpec;
		}

		/*
		 * We need to do the apply the lookup pipeline. Before proceeding
		 * if we have a "let" then project the evaluated let here.
		 */
		List *unwindArgs = list_make2(secondVar,
									  MakeTextConst(lookupArgs->lookupAs.string,
													lookupArgs->lookupAs.length));
		FuncExpr *funcExpr = makeFuncExpr(BsonLookupUnwindFunctionOid(), BsonTypeId(),
										  unwindArgs, InvalidOid, InvalidOid,
										  COERCE_EXPLICIT_CALL);
		funcExpr->funcretset = true;
		Query *subSelectQuery = CreateFunctionSelectorQuery(funcExpr,
															"lookup_subpipeline",
															context->
															nestedPipelineLevel,
															query->querySource);

		AggregationPipelineBuildContext projectorQueryContext = { 0 };
		projectorQueryContext.nestedPipelineLevel = context->nestedPipelineLevel + 1;
		projectorQueryContext.databaseNameDatum =
			optimizationArgs.rightQueryContext.databaseNameDatum;
		projectorQueryContext.collectionNameView =
			optimizationArgs.rightQueryContext.collectionNameView;
		projectorQueryContext.mongoCollection =
			optimizationArgs.rightQueryContext.mongoCollection;
		projectorQueryContext.variableSpec = letExpr;
		projectorQueryContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
		projectorQueryContext.parentStageName = ParentStageName_LOOKUP;
		strncpy((char *) projectorQueryContext.collationString, context->collationString,
				MAX_ICU_COLLATION_LENGTH);

		subSelectQuery = MutateQueryWithPipeline(subSelectQuery,
												 optimizationArgs.nonInlinedPipelineStages,
												 &projectorQueryContext);

		if (list_length(subSelectQuery->targetList) > 1)
		{
			projectorQueryContext.requiresSubQuery = true;
		}

		/* Readd the aggregate */
		Aggref **aggrefPtr = NULL;
		subSelectQuery = AddBsonArrayAggFunction(subSelectQuery,
												 &projectorQueryContext, parseState,
												 lookupArgs->lookupAs.string,
												 lookupArgs->lookupAs.length,
												 projectorQueryContext.
												 requiresSubQuery, aggrefPtr);

		/* Fix up levelsup for the subselect query */
		WalkQueryAndSetLevelsUp(subSelectQuery, secondVar, 1);

		if (letVar)
		{
			WalkQueryAndSetLevelsUp(subSelectQuery, letVar, 1);
		}

		SubLink *subLink = makeNode(SubLink);
		subLink->subLinkType = EXPR_SUBLINK;
		subLink->subLinkId = 0;
		subLink->subselect = (Node *) subSelectQuery;

		secondEntry->expr = (Expr *) subLink;
		cteLookupQuery->hasSubLinks = true;
		lookupQuery = cteLookupQuery;
		leftOutput = firstEntry->expr;
		rightOutput = secondEntry->expr;
	}

	List *mergeDocumentsArgs = list_make2(leftOutput, rightOutput);
	Oid mergeDocumentsOid = BsonDollaMergeDocumentsFunctionOid();
	if (lookupContext->isLookupUnwind)
	{
		/*
		 * If we are processing a lookup unwind with `preserveNullAndEmptyArrays: true`
		 * that means we are performing a LEFT JOIN but if the left documnets don't match with anything
		 * on the right we still need to write the left documents. So we will need to replace the rightOutput
		 * expression to a coalesce(rightOutput, {}) expression.
		 *
		 * Similarly, if `preserveNullAndEmptyArrays: false` we will need to filter out all the non-matching
		 * NULL documents from the right query i.e. righQuery.document IS NOT NULL
		 */
		Expr *rightDocExpr = lsecond(mergeDocumentsArgs);
		if (lookupContext->preserveNullAndEmptyArrays)
		{
#if PG_VERSION_NUM >= 160000

			/*
			 * Starting PG 16, if we are preserving nulls, then we need to set the varnullingrels
			 * to the join RTE index so that document var can be replaced if NULL.
			 */
			if (IsA(rightDocExpr, Var))
			{
				Bitmapset *nullValIngRel = NULL;
				nullValIngRel = bms_add_member(nullValIngRel, joinQueryRteIndex);
				((Var *) rightDocExpr)->varnullingrels = nullValIngRel;
			}
#endif
			Expr *coalesceExpr = GetEmptyBsonCoalesce(rightDocExpr);
			list_nth_cell(mergeDocumentsArgs, 1)->ptr_value = coalesceExpr;
		}
		else if (!useInnerJoin)
		{
			NullTest *nullTest = makeNode(NullTest);
			nullTest->argisrow = false;
			nullTest->nulltesttype = IS_NOT_NULL;
			nullTest->arg = rightDocExpr;

			List *existingQuals = make_ands_implicit(
				(Expr *) lookupQuery->jointree->quals);
			existingQuals = lappend(existingQuals, nullTest);
			lookupQuery->jointree->quals = (Node *) make_ands_explicit(existingQuals);
		}

		mergeDocumentsArgs = lappend(mergeDocumentsArgs,
									 MakeTextConst(lookupArgs->lookupAs.string,
												   lookupArgs->lookupAs.length));
		mergeDocumentsOid = BsonDollarMergeDocumentAtPathFunctionOid();
	}
	else
	{
		bool overrideArrayInMerge = true;
		mergeDocumentsArgs = lappend(mergeDocumentsArgs,
									 MakeBoolValueConst(overrideArrayInMerge));
	}
	FuncExpr *addFields = makeFuncExpr(mergeDocumentsOid, BsonTypeId(),
									   mergeDocumentsArgs, InvalidOid, InvalidOid,
									   COERCE_EXPLICIT_CALL);

	TargetEntry *topTargetEntry = makeTargetEntry((Expr *) addFields, 1, "document",
												  false);

	lookupQuery->targetList = list_make1(topTargetEntry);

	pfree(parseState);

	/* TODO: is this needed? */
	context->requiresSubQuery = true;
	return lookupQuery;
}


/*
 * Validate that for Sharded collections, we track that lookup wtih Let
 * Doesn't have nested lookup due to citus limitations.
 */
static void
ValidatePipelineForShardedLookupWithLet(const bson_value_t *pipeline)
{
	if (pipeline->value_type != BSON_TYPE_ARRAY)
	{
		return;
	}

	bson_iter_t pipelineIter;
	BsonValueInitIterator(pipeline, &pipelineIter);
	while (bson_iter_next(&pipelineIter))
	{
		if (!BSON_ITER_HOLDS_DOCUMENT(&pipelineIter))
		{
			return;
		}

		pgbsonelement element = { 0 };
		if (!TryGetBsonValueToPgbsonElement(bson_iter_value(&pipelineIter), &element))
		{
			return;
		}

		if (strcmp(element.path, "$lookup") == 0)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_COMMANDNOTSUPPORTED),
							errmsg(
								"Nested lookup with let on sharded collections not supported yet")));
		}

		if (strcmp(element.path, "$facet") == 0)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_COMMANDNOTSUPPORTED),
							errmsg(
								"Nested facet with let on sharded collections not supported yet")));
		}
	}
}


/*
 * Helper for a nested $lookup stage on whether it can be inlined for a $lookup.
 * This is valid as long as the lookupAs does not intersect with the local field.
 */
bool
CanInlineLookupStageLookup(const bson_value_t *lookupStage,
						   const StringView *lookupPath,
						   bool hasLet)
{
	if (hasLet)
	{
		return false;
	}

	/* A lookup can be inlined if */
	LookupArgs nestedLookupArgs;
	memset(&nestedLookupArgs, 0, sizeof(LookupArgs));
	ParseLookupStage(lookupStage, &nestedLookupArgs);

	if (StringViewStartsWithStringView(&nestedLookupArgs.lookupAs, lookupPath) ||
		StringViewStartsWithStringView(lookupPath, &nestedLookupArgs.lookupAs))
	{
		return false;
	}

	return true;
}


/*
 * Validates the pipeline for a $unionwith query.
 */
static void
ValidateUnionWithPipeline(const bson_value_t *pipeline, bool hasCollection)
{
	bson_iter_t pipelineIter;
	BsonValueInitIterator(pipeline, &pipelineIter);

	if (IsBsonValueEmptyArray(pipeline) && !hasCollection)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
						errmsg(
							"A $unionWith stage without specifying a target collection must begin its pipeline with a $documents stage.")));
	}

	bool isFirstStage = true;
	while (bson_iter_next(&pipelineIter))
	{
		pgbsonelement stageElement = GetPipelineStage(&pipelineIter, "unionWith",
													  "pipeline");
		if (isFirstStage && !hasCollection)
		{
			if (strcmp(stageElement.path, "$documents") != 0)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_BADVALUE),
								errmsg(
									"A $unionWith stage without specifying a target collection must begin its pipeline with a $documents stage.")));
			}
		}

		isFirstStage = false;
		if (strcmp(stageElement.path, "$out") == 0)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION31441),
							errmsg(
								"$out is prohibited inside a sub-pipeline of $unionWith")));
		}
		else if (strcmp(stageElement.path, "$merge") == 0)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION31441),
							errmsg(
								"Using the $merge is prohibited inside a sub-pipeline of $unionWith.")));
		}
		else if (strcmp(stageElement.path, "$changeStream") == 0)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION31441),
							errmsg(
								"The use of $changeStream is prohibited inside the sub-pipeline of $unionWith.")));
		}
	}
}


/*
 * Parses & validates the input $graphLookup spec.
 * Parsed outputs are placed in the GraphLookupArgs struct.
 */
static void
ParseGraphLookupStage(const bson_value_t *existingValue, GraphLookupArgs *args)
{
	if (existingValue->value_type != BSON_TYPE_DOCUMENT)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg(
							"The $graphLookup stage specification is expected to be provided as an object, however, the system encountered %s instead.",
							BsonTypeName(existingValue->value_type)),
						errdetail_log(
							"The $graphLookup stage specification is expected to be provided as an object, however, the system encountered %s instead.",
							BsonTypeName(existingValue->value_type))));
	}

	bson_iter_t lookupIter;
	BsonValueInitIterator(existingValue, &lookupIter);

	args->maxDepth = INT32_MAX;
	bool fromSpecified = false;
	while (bson_iter_next(&lookupIter))
	{
		const char *key = bson_iter_key(&lookupIter);
		const bson_value_t *value = bson_iter_value(&lookupIter);
		if (strcmp(key, "as") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40103),
								errmsg(
									"graphlookup argument 'as' must be a string, is type %s",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"graphlookup argument 'as' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->asField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};

			if (args->asField.length > 0 && args->asField.string[0] == '$')
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION16410),
								errmsg(
									"'as' field name cannot begin with '$'"),
								errdetail_log(
									"'as' field name cannot begin with '$'")));
			}
		}
		else if (strcmp(key, "startWith") == 0)
		{
			args->inputExpression = *value;
		}
		else if (strcmp(key, "connectFromField") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40103),
								errmsg(
									"graphlookup argument 'connectFromField' must be a string, is type %s",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"graphlookup argument 'connectFromField' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->connectFromField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
			if (args->connectFromField.length > 0 && args->connectFromField.string[0] ==
				'$')
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION16410),
								errmsg(
									"FieldPath field names cannot begin with symbol such as '$'"),
								errdetail_log(
									"FieldPath field names cannot begin with symbol such as '$'")));
			}
		}
		else if (strcmp(key, "connectToField") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40103),
								errmsg(
									"graphlookup argument 'connectToField' must be a string, is type %s",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"graphlookup argument 'connectToField' must be a string, is type %s",
									BsonTypeName(value->value_type))));
			}

			args->connectToField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
			if (args->connectToField.length > 0 && args->connectToField.string[0] == '$')
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION16410),
								errmsg(
									"connectToField: FieldPath field names cannot begin with the symbol '$'"),
								errdetail_log(
									"connectToField: FieldPath field names cannot begin with the symbol '$'")));
			}
		}
		else if (strcmp(key, "from") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
								errmsg(
									"graphlookup 'from' parameter must be provided as a string, but a value of type %s was given instead.",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"graphlookup 'from' parameter must be provided as a string, but a value of type %s was given instead.",
									BsonTypeName(value->value_type))));
			}

			args->fromCollection = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
			fromSpecified = true;
		}
		else if (strcmp(key, "maxDepth") == 0)
		{
			if (!BsonValueIsNumber(value))
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40100),
								errmsg(
									"The 'maxDepth' argument in graphlookup must be specified as a numeric value, but a value of type %s was provided instead.",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"The 'maxDepth' argument in graphlookup must be specified as a numeric value, but a value of type %s was provided instead.",
									BsonTypeName(value->value_type))));
			}

			if (!IsBsonValueFixedInteger(value))
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40102),
								errmsg(
									"The value of graphlookup.maxDepth must always be a non‑negative integer number."),
								errdetail_log(
									"The value of graphlookup.maxDepth must always be a non‑negative integer number.")));
			}

			args->maxDepth = BsonValueAsInt32(value);

			if (args->maxDepth < 0)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40101),
								errmsg(
									"The value of graphlookup.maxDepth must always be a non‑negative integer number."),
								errdetail_log(
									"The value of graphlookup.maxDepth must always be a non‑negative integer number.")));
			}
		}
		else if (strcmp(key, "depthField") == 0)
		{
			if (value->value_type != BSON_TYPE_UTF8)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40103),
								errmsg(
									"The 'depthField' argument in graphlookup must be provided as a string, but a value of type %s was given instead.",
									BsonTypeName(value->value_type)),
								errdetail_log(
									"The 'depthField' argument in graphlookup must be provided as a string, but a value of type %s was given instead.",
									BsonTypeName(value->value_type))));
			}

			args->depthField = (StringView) {
				.length = value->value.v_utf8.len,
				.string = value->value.v_utf8.str
			};
			if (args->depthField.length > 0 && args->depthField.string[0] == '$')
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION16410),
								errmsg(
									"depthField:FieldPath field names cannot begin with the symbol '$'"),
								errdetail_log(
									"depthField:FieldPath field names cannot begin with the symbol '$'")));
			}
		}
		else if (strcmp(key, "restrictSearchWithMatch") == 0)
		{
			if (value->value_type != BSON_TYPE_DOCUMENT)
			{
				ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40185),
								errmsg(
									"The 'restrictSearchWithMatch' argument in graphlookup must be provided as a document, but a value of type %s was received instead.",
									BsonTypeName(value->value_type))));
			}

			args->restrictSearch = *value;
		}
		else
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40104),
							errmsg(
								"Unrecognized parameter supplied to stage $graphlookup: %s",
								key),
							errdetail_log(
								"Unrecognized parameter supplied to stage $graphlookup: %s",
								key)));
		}
	}

	if (args->asField.length == 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40105),
						errmsg("$graphLookup requires 'as' field to be specified")));
	}

	if (!fromSpecified)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_FAILEDTOPARSE),
						errmsg("$graphLookup requires 'from' field to be specified")));
	}
	if (args->fromCollection.length == 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_INVALIDNAMESPACE),
						errmsg("$graphLookup requires 'from' field to be specified")));
	}

	if (args->inputExpression.value_type == BSON_TYPE_EOD)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40105),
						errmsg(
							"You must provide a 'startWith' parameter when performing a $graphLookup operation")));
	}

	if (args->connectFromField.length == 0 || args->connectToField.length == 0)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION40105),
						errmsg(
							"Both 'connectFrom' and 'connectTo' operators must be provided for a $graphLookup operation.")));
	}

	StringInfo connectExpr = makeStringInfo();
	appendStringInfo(connectExpr, "$%.*s", args->connectFromField.length,
					 args->connectFromField.string);
	args->connectFromFieldExpression.value_type = BSON_TYPE_UTF8;
	args->connectFromFieldExpression.value.v_utf8.len = strlen(connectExpr->data);
	args->connectFromFieldExpression.value.v_utf8.str = connectExpr->data;
}


/*
 * Builds the graph lookup FuncExpr bson_expression_get(document, '{ "connectToField": { "$makeArray": "$inputExpression" } }' )
 * or bson_expression_get(document, '{ "connectToField": { "$makeArray": "$inputExpression" } }', collationString ),
 * if a valid collation string is provided.
 */
static FuncExpr *
BuildInputExpressionForQuery(Expr *origExpr, const StringView *connectToField, const
							 bson_value_t *inputExpression,
							 AggregationPipelineBuildContext *context)
{
	pgbson_writer expressionWriter;
	PgbsonWriterInit(&expressionWriter);

	pgbson_writer makeArrayWriter;
	PgbsonWriterStartDocument(&expressionWriter, connectToField->string,
							  connectToField->length,
							  &makeArrayWriter);

	/* { "$makeArray: $inputExpression } */
	PgbsonWriterAppendValue(&makeArrayWriter, "$makeArray", 10, inputExpression);
	PgbsonWriterEndDocument(&expressionWriter, &makeArrayWriter);

	pgbson *inputExpr = PgbsonWriterGetPgbson(&expressionWriter);
	Const *falseConst = (Const *) MakeBoolValueConst(false);
	List *inputExprArgs;
	Oid functionOid;

	Const *collationConst = IsCollationApplicable(context->collationString) ?
							MakeTextConst(context->collationString,
										  strlen(context->collationString)) : NULL;


	if (collationConst)
	{
		functionOid = BsonExpressionGetWithLetAndCollationFunctionOid();
		inputExprArgs = list_make5(origExpr,
								   MakeBsonConst(inputExpr),
								   falseConst,
								   context->variableSpec ? context->variableSpec :
								   (Expr *) MakeBsonConst(PgbsonInitEmpty()),
								   collationConst);
	}
	else if (context->variableSpec != NULL)
	{
		functionOid = BsonExpressionGetWithLetFunctionOid();
		inputExprArgs = list_make4(origExpr, MakeBsonConst(inputExpr),
								   falseConst, context->variableSpec);
	}
	else
	{
		functionOid = BsonExpressionGetFunctionOid();
		inputExprArgs = list_make3(origExpr, MakeBsonConst(inputExpr),
								   falseConst);
	}

	FuncExpr *inputFuncExpr = makeFuncExpr(
		functionOid, BsonTypeId(), inputExprArgs, InvalidOid,
		InvalidOid, COERCE_EXPLICIT_CALL);

	return inputFuncExpr;
}


/*
 * Adds input expression query to the input query projection list. This is the expression
 * for the inputExpression for the Graph lookup
 * bson_expression_get(document, '{ "connectToField": "$inputExpression" } ) AS "inputExpr"
 * or
 * bson_expression_get(document, '{ "connectToField": "$inputExpression" }', collationString )
 * AS "inputExpr",
 * if a collation string is provided.
 */
static AttrNumber
AddInputExpressionToQuery(Query *query, StringView *fieldName, const
						  bson_value_t *inputExpression,
						  AggregationPipelineBuildContext *context)
{
	/* Now, add the expression value projector to the left query */
	TargetEntry *origEntry = linitial(query->targetList);

	/*
	 * Adds the projector bson_expression_get(document, '{ "connectToField": { "$makeArray": "$inputExpression" } }' )
	 * AS "inputExpr" into the left query.
	 */
	FuncExpr *inputFuncExpr = BuildInputExpressionForQuery(origEntry->expr, fieldName,
														   inputExpression,
														   context);
	bool resjunk = false;
	AttrNumber expressionResultNumber = 2;
	TargetEntry *entry = makeTargetEntry((Expr *) inputFuncExpr, expressionResultNumber,
										 "inputExpr",
										 resjunk);
	query->targetList = lappend(query->targetList, entry);

	return expressionResultNumber;
}


/*
 * Core handling for a graph lookup query.
 * This forms the query as follows:
 *
 * WITH basecte AS (
 *  SELECT
 *      document,
 *      bson_expression_get(
 *          document,
 *          '{ "*connectToField*": { "$makeArray": "$*inputExpression*" } }',
 *          true
 *      ) AS initialexpr
 *  FROM inputCollection
 * ),
 * graphLookupStage AS (
 *  SELECT
 *      document,
 *      COALESCE(
 *          (
 *              WITH RECURSIVE graphLookup AS (
 *
 *          -- anchor member
 *                  SELECT
 *                      coll.document AS doc,
 *                      '{ "depth": 0 }' as depth,
 *                      bson_expression_get(coll.document, '{ "_id": "$_id" }') as baseDocId
 *                  FROM *fromCollection* coll
 *                  WHERE bson_dollar_in(coll.document, basecte.initialexpr)
 *
 *                  UNION ALL
 *
 *          -- recursive term
 *                  SELECT
 *                      document AS doc,
 *                      bson_expression_get(graphLookup.depth, '{ "depth": { "$add": [ "$depth", 1 ] }})') as depth,
 *                      bson_expression_get(coll.document, '{ "_id": "$_id" }') as baseDocId
 *                  FROM *fromCollection*, graphLookup   -- join the recursive cte with the target collection
 *                  WHERE bson_dollar_in(document, bson_expression_get(graphLookup.doc, '{ "*connectToField*": { "$makeArray": "$*connectFromField*" } }', true))
 *              ) CYCLE baseDocId SET is_cycle USING path
 *              SELECT bson_array_agg(doc, '*as*') FROM (SELECT DISTINCT ON (graphLookupStage.baseDocId) graphLookupStage.document FROM graphLookup ORDER BY depth)
 *          ),
 *          '{ "*as*": [] }'
 *      ) AS addFields
 *  FROM basecte
 * )
 *
 * SELECT bson_dollar_add_fields(document, addFields) FROM graphLookupStage;
 *
 */
static Query *
ProcessGraphLookupCore(Query *query, AggregationPipelineBuildContext *context,
					   GraphLookupArgs *lookupArgs)
{
	/* Similar to $lookup, if there's more than 1 projector push down */
	if (list_length(query->targetList) > 1)
	{
		query = MigrateQueryToSubQuery(query, context);
	}

	/* First add the input expression to the input query */
	AddInputExpressionToQuery(query, &lookupArgs->connectToField,
							  &lookupArgs->inputExpression,
							  context);

	/* Create the final query: Since the higher query is in a CTE - push this one down. */
	context->numNestedLevels++;
	Query *graphLookupQuery = makeNode(Query);
	graphLookupQuery->commandType = CMD_SELECT;
	graphLookupQuery->querySource = query->querySource;
	graphLookupQuery->canSetTag = true;

	/* Push the input query a base CTE list */
	StringInfo cteStr = makeStringInfo();
	appendStringInfo(cteStr, "graphLookupBase_%d", context->nestedPipelineLevel);
	CommonTableExpr *baseCteExpr = makeNode(CommonTableExpr);
	baseCteExpr->ctename = cteStr->data;
	baseCteExpr->ctequery = (Node *) query;
	graphLookupQuery->cteList = lappend(graphLookupQuery->cteList, baseCteExpr);

	/* Create a graphLookupStage CTE */
	StringInfo secondCteStr = makeStringInfo();
	appendStringInfo(secondCteStr, "graphLookupStage_%d", context->nestedPipelineLevel);
	CommonTableExpr *graphCteExpr = makeNode(CommonTableExpr);
	graphCteExpr->ctename = secondCteStr->data;
	graphCteExpr->ctequery = (Node *) BuildGraphLookupCteQuery(query->querySource,
															   baseCteExpr,
															   lookupArgs, context);
	graphLookupQuery->cteList = lappend(graphLookupQuery->cteList, graphCteExpr);

	/* Make the graphLookupStage the RTE of the final query */
	int stageNum = 1;
	RangeTblEntry *graphLookupStageRte = CreateCteRte(graphCteExpr, "graphLookup",
													  stageNum, 0);
	graphLookupQuery->rtable = list_make1(graphLookupStageRte);
	RangeTblRef *graphLookupRef = makeNode(RangeTblRef);
	graphLookupRef->rtindex = 1;
	graphLookupQuery->jointree = makeFromExpr(list_make1(graphLookupRef), NULL);

	/* Add the add_fields on the targetEntry
	 * SELECT bson_dollar_add_fields(document, addFields) FROM graphLookupStage;
	 */
	Var *documentVar = makeVar(graphLookupRef->rtindex, 1, BsonTypeId(), -1, InvalidOid,
							   0);
	Var *addFieldsVar = makeVar(graphLookupRef->rtindex, 2, BsonTypeId(), -1, InvalidOid,
								0);

	/* $graphlookup override nested array in merge projections */
	bool overrideArrayInProjection = true;
	FuncExpr *addFieldsExpr = makeFuncExpr(
		BsonDollaMergeDocumentsFunctionOid(), BsonTypeId(),
		list_make3(documentVar, addFieldsVar,
				   MakeBoolValueConst(overrideArrayInProjection)),
		InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL);
	TargetEntry *finalTargetEntry = makeTargetEntry((Expr *) addFieldsExpr, 1, "document",
													false);
	graphLookupQuery->targetList = list_make1(finalTargetEntry);

	return graphLookupQuery;
}


/*
 * This builds the the caller of the recursive CTE for a graphLookup
 * For the structure of this query, see ProcessGraphLookupCore
 */
static Query *
BuildGraphLookupCteQuery(QuerySource parentSource,
						 CommonTableExpr *baseCteExpr,
						 GraphLookupArgs *args,
						 AggregationPipelineBuildContext *parentContext)
{
	Query *graphLookupQuery = makeNode(Query);
	graphLookupQuery->commandType = CMD_SELECT;
	graphLookupQuery->querySource = parentSource;
	graphLookupQuery->canSetTag = true;

	/* The base RTE is the base query */
	/* This is now the SELECT ... FROM baseCte */
	int stageNum = 1;
	int levelsUp = 1;
	RangeTblEntry *graphLookupStageRte = CreateCteRte(baseCteExpr, "graphLookupBase",
													  stageNum, levelsUp);
	graphLookupQuery->rtable = list_make1(graphLookupStageRte);

	RangeTblRef *graphLookupRef = makeNode(RangeTblRef);
	graphLookupRef->rtindex = 1;
	graphLookupQuery->jointree = makeFromExpr(list_make1(graphLookupRef), NULL);

	/* The first projector is the document var from the CTE */
	Var *firstVar = makeVar(graphLookupRef->rtindex, 1, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *firstEntry = makeTargetEntry((Expr *) firstVar, 1, "document", false);

	/* The subquery for the recursive CTE goes here:
	 * The CTE goes 2 levels up since it has to go through this graphLookupQuery (1)
	 * to the parent query (2)
	 */
	int ctelevelsUp = 2;
	Query *recursiveSelectQuery = BuildRecursiveGraphLookupQuery(parentSource, args,
																 parentContext,
																 baseCteExpr,
																 ctelevelsUp);
	SubLink *subLink = makeNode(SubLink);
	subLink->subLinkType = EXPR_SUBLINK;
	subLink->subLinkId = 0;
	subLink->subselect = (Node *) recursiveSelectQuery;
	graphLookupQuery->hasSubLinks = true;

	/* Coalesce to handle NULL entries */
	/* COALESCE( recursiveQuery, '{ "*as*": [] }' ) */
	Expr *coalesceExpr = GetArrayAggCoalesce((Expr *) subLink, args->asField.string,
											 args->asField.length);
	TargetEntry *secondEntry = makeTargetEntry((Expr *) coalesceExpr, 2, "addFields",
											   false);

	graphLookupQuery->targetList = list_make2(firstEntry, secondEntry);
	return graphLookupQuery;
}


/*
 * Creates an expression for bson_expression_get(document, '{ "_id": "$_id"}', true)
 */
static Expr *
CreateIdProjectionExpr(Expr *baseExpr)
{
	pgbsonelement expressionElement = { 0 };
	expressionElement.path = "_id";
	expressionElement.pathLength = 3;
	expressionElement.bsonValue.value_type = BSON_TYPE_UTF8;
	expressionElement.bsonValue.value.v_utf8.len = 4;
	expressionElement.bsonValue.value.v_utf8.str = "$_id";

	return (Expr *) makeFuncExpr(BsonExpressionGetFunctionOid(), BsonTypeId(),
								 list_make2(baseExpr, MakeBsonConst(PgbsonElementToPgbson(
																		&expressionElement))),
								 InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL);
}


/*
 * This is the base case for the recursive CTE. This scans the from collection
 * with the equality match on the original table's rows.
 * SELECT * FROm from_collection WHERE document #= '{ "inputExpression" }
 */
static Query *
GenerateBaseCaseQuery(AggregationPipelineBuildContext *parentContext,
					  GraphLookupArgs *args, int baseCteLevelsUp)
{
	AggregationPipelineBuildContext subPipelineContext = { 0 };
	subPipelineContext.nestedPipelineLevel = parentContext->nestedPipelineLevel + 2;
	subPipelineContext.databaseNameDatum = parentContext->databaseNameDatum;
	subPipelineContext.variableSpec = parentContext->variableSpec;
	subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
	strncpy((char *) subPipelineContext.collationString, parentContext->collationString,
			MAX_ICU_COLLATION_LENGTH);
	pg_uuid_t *collectionUuid = NULL;
	bson_value_t *indexHint = NULL;
	Query *baseCaseQuery = GenerateBaseTableQuery(parentContext->databaseNameDatum,
												  &args->fromCollection, collectionUuid,
												  indexHint, &subPipelineContext);

	/* Citus doesn't suppor this scenario: ERROR:  recursive CTEs are not supported in distributed queries */
	if (subPipelineContext.mongoCollection != NULL &&
		subPipelineContext.mongoCollection->shardKey != NULL)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_COMMANDNOTSUPPORTED),
						errmsg(
							"$graphLookup using 'from' on a sharded collection is currently unsupported"),
						errdetail_log(
							"$graphLookup using 'from' on a sharded collection is currently unsupported")));
	}

	if (args->restrictSearch.value_type != BSON_TYPE_EOD)
	{
		baseCaseQuery = HandleMatch(&args->restrictSearch, baseCaseQuery,
									&subPipelineContext);
		if (baseCaseQuery->sortClause != NIL)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION5626500),
							errmsg(
								"$near, $nearSphere and $geoNear cannot be used here. Use $geoWithin instead.")));
		}
	}

	List *baseQuals = NIL;
	if (baseCaseQuery->jointree->quals != NULL)
	{
		baseQuals = make_ands_implicit((Expr *) baseCaseQuery->jointree->quals);
	}

	TargetEntry *firstEntry = linitial(baseCaseQuery->targetList);

	/* Match the Var of the top level input query: We're one level deeper (Since this is inside the SetOP) */
	Var *rightVar = makeVar(1, 2, BsonTypeId(), -1, InvalidOid, baseCteLevelsUp);
	Const *textConst = MakeTextConst(args->connectToField.string,
									 args->connectToField.length);

	FuncExpr *initialMatchFunc = makeFuncExpr(BsonDollarLookupJoinFilterFunctionOid(),
											  BOOLOID,
											  list_make3(firstEntry->expr, rightVar,
														 textConst),
											  InvalidOid, InvalidOid,
											  COERCE_EXPLICIT_CALL);

	baseQuals = lappend(baseQuals, initialMatchFunc);
	baseCaseQuery->jointree->quals = (Node *) make_ands_explicit(baseQuals);

	/* Add the depth field (base case is 0) */
	pgbsonelement element =
	{
		.path = "depth",
		.pathLength = 5,
		.bsonValue =
		{
			.value_type = BSON_TYPE_INT32,
			.value.v_int32 = 0,
			.padding = 0
		}
	};
	if (args->depthField.length > 0)
	{
		element.path = args->depthField.string;
		element.pathLength = args->depthField.length;
	}

	Const *depthConst = MakeBsonConst(PgbsonElementToPgbson(&element));
	baseCaseQuery->targetList = lappend(baseCaseQuery->targetList,
										makeTargetEntry((Expr *) depthConst, 2, "depth",
														false));

	baseCaseQuery->targetList = lappend(baseCaseQuery->targetList,
										makeTargetEntry(CreateIdProjectionExpr(
															firstEntry->expr), 3,
														"baseDocId",
														false));

	return baseCaseQuery;
}


/*
 * This is the recursive lookup case. This is equivalent to searching equality from the prior round
 * SELECT * FROm from_collection WHERE document #= '{ "previousFromExpr" }
 */
static Query *
GenerateRecursiveCaseQuery(AggregationPipelineBuildContext *parentContext,
						   CommonTableExpr *recursiveCte,
						   GraphLookupArgs *args, int levelsUp)
{
	AggregationPipelineBuildContext subPipelineContext = { 0 };
	subPipelineContext.nestedPipelineLevel = parentContext->nestedPipelineLevel + 2;
	subPipelineContext.databaseNameDatum = parentContext->databaseNameDatum;
	subPipelineContext.variableSpec = parentContext->variableSpec;
	subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
	strncpy((char *) subPipelineContext.collationString, parentContext->collationString,
			MAX_ICU_COLLATION_LENGTH);
	pg_uuid_t *collectionUuid = NULL;
	bson_value_t *indexHint = NULL;
	Query *recursiveQuery = GenerateBaseTableQuery(parentContext->databaseNameDatum,
												   &args->fromCollection, collectionUuid,
												   indexHint, &subPipelineContext);
	if (args->restrictSearch.value_type != BSON_TYPE_EOD)
	{
		recursiveQuery = HandleMatch(&args->restrictSearch, recursiveQuery,
									 &subPipelineContext);
		if (recursiveQuery->sortClause != NIL)
		{
			ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION5626500),
							errmsg(
								"$near, $nearSphere and $geoNear cannot be used here. Use $geoWithin instead.")));
		}
	}

	List *baseQuals = NIL;
	if (recursiveQuery->jointree->quals != NULL)
	{
		baseQuals = make_ands_implicit((Expr *) recursiveQuery->jointree->quals);
	}

	RangeTblEntry *entry = CreateCteRte(recursiveCte, "lookupRecursive", 1, levelsUp);
	entry->self_reference = true;
	recursiveQuery->rtable = lappend(recursiveQuery->rtable, entry);

	RangeTblRef *rangeTblRef = makeNode(RangeTblRef);
	rangeTblRef->rtindex = 2;
	recursiveQuery->jointree->fromlist = lappend(recursiveQuery->jointree->fromlist,
												 rangeTblRef);

	TargetEntry *firstEntry = linitial(recursiveQuery->targetList);

	/* Match the Var of the CTE level input query */
	Var *leftDocVar = makeVar(rangeTblRef->rtindex, 1, BsonTypeId(), -1, InvalidOid, 0);
	FuncExpr *inputExpr = BuildInputExpressionForQuery((Expr *) leftDocVar,
													   &args->connectToField,
													   &args->connectFromFieldExpression,
													   parentContext);

	Const *textConst = MakeTextConst(args->connectToField.string,
									 args->connectToField.length);

	FuncExpr *initialMatchFunc = makeFuncExpr(BsonDollarLookupJoinFilterFunctionOid(),
											  BOOLOID,
											  list_make3(firstEntry->expr, inputExpr,
														 textConst),
											  InvalidOid, InvalidOid,
											  COERCE_EXPLICIT_CALL);

	baseQuals = lappend(baseQuals, initialMatchFunc);

	StringView depthPath = { .length = 5, .string = "depth" };
	if (args->depthField.length > 0)
	{
		depthPath = args->depthField;
	}

	Var *priorDepthValue = makeVar(rangeTblRef->rtindex, 2, BsonTypeId(), -1, InvalidOid,
								   0);
	if (args->maxDepth >= 0 && args->maxDepth != INT32_MAX)
	{
		pgbsonelement depthQueryElement = { 0 };
		depthQueryElement.path = depthPath.string;
		depthQueryElement.pathLength = depthPath.length;
		depthQueryElement.bsonValue.value_type = BSON_TYPE_INT32;
		depthQueryElement.bsonValue.value.v_int32 = args->maxDepth;

		Const *depthConst = MakeBsonConst(PgbsonElementToPgbson(&depthQueryElement));
		depthConst->consttype = BsonQueryTypeId();
		OpExpr *depthMatchFunc = (OpExpr *) make_opclause(
			BsonLessThanMatchRuntimeOperatorId(),
			BOOLOID, false,
			(Expr *) priorDepthValue,
			(Expr *) depthConst, InvalidOid,
			InvalidOid);
		depthMatchFunc->opfuncid = BsonLessThanMatchRuntimeFunctionId();
		baseQuals = lappend(baseQuals, depthMatchFunc);
	}

	recursiveQuery->jointree->quals = (Node *) make_ands_explicit(baseQuals);

	/* Append the graph depth expression */
	pgbson_writer depthFuncWriter;
	PgbsonWriterInit(&depthFuncWriter);

	pgbson_writer childWriter;
	PgbsonWriterStartDocument(&depthFuncWriter, depthPath.string, depthPath.length,
							  &childWriter);

	pgbson_array_writer addWriter;
	PgbsonWriterStartArray(&childWriter, "$add", 4, &addWriter);
	PgbsonArrayWriterWriteUtf8(&addWriter, psprintf("$%.*s", depthPath.length,
													depthPath.string));

	bson_value_t number1Value = { 0 };
	number1Value.value_type = BSON_TYPE_INT32;
	number1Value.value.v_int32 = 1;
	PgbsonArrayWriterWriteValue(&addWriter, &number1Value);
	PgbsonWriterEndArray(&childWriter, &addWriter);
	PgbsonWriterEndDocument(&depthFuncWriter, &childWriter);

	Const *depthAddConst = MakeBsonConst(PgbsonWriterGetPgbson(&depthFuncWriter));
	Expr *newDepthFuncExpr = (Expr *) makeFuncExpr(BsonDollarAddFieldsFunctionOid(),
												   BsonTypeId(),
												   list_make2(priorDepthValue,
															  depthAddConst),
												   InvalidOid, InvalidOid,
												   COERCE_EXPLICIT_CALL);
	recursiveQuery->targetList = lappend(recursiveQuery->targetList,
										 makeTargetEntry(newDepthFuncExpr, 2, "depth",
														 false));

	recursiveQuery->targetList = lappend(recursiveQuery->targetList,
										 makeTargetEntry(CreateIdProjectionExpr(
															 firstEntry->expr), 3,
														 "baseDocId",
														 false));

	return recursiveQuery;
}


/*
 * Walker to replace the recursive graph CTE post cycle rewrite.
 */
static bool
RewriteGraphLookupRecursiveCteExprWalker(Node *node, CommonTableExpr *graphRecursiveCte)
{
	CHECK_FOR_INTERRUPTS();

	if (node == NULL)
	{
		return false;
	}

	if (IsA(node, Query))
	{
		return query_tree_walker((Query *) node, RewriteGraphLookupRecursiveCteExprWalker,
								 graphRecursiveCte, QTW_EXAMINE_RTES_BEFORE |
								 QTW_DONT_COPY_QUERY);
	}
	else if (IsA(node, RangeTblEntry))
	{
		RangeTblEntry *tblEntry = (RangeTblEntry *) node;
		if (tblEntry->rtekind == RTE_CTE && strcmp(tblEntry->ctename,
												   graphRecursiveCte->ctename) == 0)
		{
			UpdateCteRte(tblEntry, graphRecursiveCte);
			return true;
		}

		return false;
	}

	return false;
}


/*
 * This builds the core recursive CTE for a graphLookup
 * For the structure of this query, see ProcessGraphLookupCore
 */
static Query *
BuildRecursiveGraphLookupQuery(QuerySource parentSource, GraphLookupArgs *args,
							   AggregationPipelineBuildContext *parentContext,
							   CommonTableExpr *baseCteExpr, int baseCteLevelsUp)
{
	AggregationPipelineBuildContext subPipelineContext = { 0 };
	subPipelineContext.nestedPipelineLevel = parentContext->nestedPipelineLevel + 1;
	subPipelineContext.databaseNameDatum = parentContext->databaseNameDatum;
	subPipelineContext.variableSpec = parentContext->variableSpec;
	subPipelineContext.joinStatus = JoinStageStatus_HasJoinsOrUnions;
	strncpy((char *) subPipelineContext.collationString, parentContext->collationString,
			MAX_ICU_COLLATION_LENGTH);

	/* First build the recursive CTE object */
	CommonTableExpr *graphCteExpr = makeNode(CommonTableExpr);
	graphCteExpr->ctename = "graphLookupRecurseStage";
	graphCteExpr->cterecursive = true;

	/* Define the UNION ALL query step needed for the recursive CTE */
	Query *unionAllQuery = makeNode(Query);
	unionAllQuery->commandType = CMD_SELECT;
	unionAllQuery->querySource = parentSource;
	unionAllQuery->canSetTag = true;
	unionAllQuery->jointree = makeFromExpr(NIL, NULL);

	/* We need to build the output of the UNION ALL first (since this is recursive )*/
	/* The first var is the document */
	Var *documentVar = makeVar(1, 1, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *docEntry = makeTargetEntry((Expr *) documentVar,
											1, "document", false);

	/* Then comes the depth */
	Var *depthVar = makeVar(1, 2, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *depthEntry = makeTargetEntry((Expr *) depthVar,
											  2, "depth", false);

	Var *baseVar = makeVar(1, 3, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *baseDocEntry = makeTargetEntry((Expr *) baseVar,
												3, "baseDocId", false);
	List *unionTargetEntries = list_make3(docEntry, depthEntry, baseDocEntry);

	unionAllQuery->targetList = unionTargetEntries;
	graphCteExpr->ctequery = (Node *) unionAllQuery;

	CTECycleClause *cteCycleClause = makeNode(CTECycleClause);
	cteCycleClause->cycle_col_list = list_make1(makeString("baseDocId"));
	cteCycleClause->cycle_mark_collation = InvalidOid;
	cteCycleClause->cycle_mark_type = BOOLOID;
	cteCycleClause->cycle_mark_column = "is_cycle";
	cteCycleClause->cycle_mark_default = MakeBoolValueConst(false);
	cteCycleClause->cycle_mark_value = MakeBoolValueConst(true);
	cteCycleClause->cycle_path_column = "path";
	cteCycleClause->cycle_mark_typmod = -1;
	cteCycleClause->cycle_mark_neop = BooleanNotEqualOperator;
	graphCteExpr->cycle_clause = cteCycleClause;

	ParseState *parseState = make_parsestate(NULL);
	parseState->p_expr_kind = EXPR_KIND_SELECT_TARGET;
	parseState->p_next_resno = 1;

	/* Build the base case query (non recursive entry)*/
	/* CTE will move up 1 level because of the subquery after the group below. */
	baseCteLevelsUp += 2;
	Query *baseSubQuery = GenerateBaseCaseQuery(parentContext, args, baseCteLevelsUp);

	/* Build the recursive case query: Not ethe actual graph CTE is 2 levels up:
	 * One to get to the SetOpStatement query
	 * Two to get to the parent GraphLookup query.
	 */
	int graphCteLevelsUp = 2;
	Query *recursiveSubQuery = GenerateRecursiveCaseQuery(parentContext, graphCteExpr,
														  args, graphCteLevelsUp);

	/* To create a UNION ALL we need the left and right to be subquery RTEs */
	bool includeAllColumns = true;
	int depth = 2;
	RangeTblEntry *baseQuery = MakeSubQueryRte(baseSubQuery, 1, depth, "baseQuery",
											   includeAllColumns);
	baseQuery->inFromCl = false;
	RangeTblEntry *recursiveQuery = MakeSubQueryRte(recursiveSubQuery, 2, depth,
													"recursiveQuery", includeAllColumns);
	recursiveQuery->inFromCl = false;

	unionAllQuery->rtable = list_make2(baseQuery, recursiveQuery);
	RangeTblRef *baseReference = makeNode(RangeTblRef);
	baseReference->rtindex = 1;
	RangeTblRef *recursiveReference = makeNode(RangeTblRef);
	recursiveReference->rtindex = 2;

	/* For deduplication of _id, we use DISTINCT ON by _id.
	 * This is more efficient for PostgreSQL scenarios.
	 */
	SetOperationStmt *setOpStatement = makeNode(SetOperationStmt);
	setOpStatement->all = true;
	setOpStatement->op = SETOP_UNION;
	setOpStatement->larg = (Node *) baseReference;
	setOpStatement->rarg = (Node *) recursiveReference;

	setOpStatement->colCollations = list_make3_oid(InvalidOid, InvalidOid,
												   InvalidOid);
	setOpStatement->colTypes = list_make3_oid(BsonTypeId(), BsonTypeId(),
											  BsonTypeId());
	setOpStatement->colTypmods = list_make3_int(-1, -1, -1);

	graphCteExpr->ctecolnames = list_make3(makeString("document"), makeString("depth"),
										   makeString("baseDocId"));
	graphCteExpr->ctecoltypes = setOpStatement->colTypes;
	graphCteExpr->ctecoltypmods = setOpStatement->colTypmods;
	graphCteExpr->ctecolcollations = setOpStatement->colCollations;

	/* Update the query with the SetOp statement */
	unionAllQuery->setOperations = (Node *) setOpStatement;

	/* Now that the unionAllQuery is built, call the rewrite handler */
	graphCteExpr = rewriteSearchAndCycle(graphCteExpr);

	/* Reset cycle path after rewrite */
	graphCteExpr->cycle_clause = NULL;

	query_tree_walker((Query *) graphCteExpr->ctequery,
					  RewriteGraphLookupRecursiveCteExprWalker, graphCteExpr,
					  QTW_EXAMINE_RTES_BEFORE |
					  QTW_DONT_COPY_QUERY);

	/* Now form the top level Graph Lookup Recursive Query entry */
	Query *graphLookupQuery = makeNode(Query);
	graphLookupQuery->commandType = CMD_SELECT;
	graphLookupQuery->querySource = parentSource;
	graphLookupQuery->canSetTag = true;

	/* WITH RECURSIVE */
	graphLookupQuery->hasRecursive = true;

	graphLookupQuery->cteList = lappend(graphLookupQuery->cteList, graphCteExpr);

	/* Next build the RangeTables */
	RangeTblEntry *rte = CreateCteRte(graphCteExpr, "graphLookup", 2, 0);

	graphLookupQuery->rtable = list_make1(rte);

	/* Next build the FromList for the final query */
	RangeTblRef *singleRangeTableRef = makeNode(RangeTblRef);
	singleRangeTableRef->rtindex = 1;
	graphLookupQuery->jointree = makeFromExpr(list_make1(singleRangeTableRef), NULL);

	/* Build the final targetList: */
	/* SELECT bson_array_agg(doc, 'asField') FROM graphLookup */

	/* Build a base targetEntry that the arrayAgg will use */
	Var *simpleVar = makeVar(1, 1, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *simpleTargetEntry = makeTargetEntry((Expr *) simpleVar, 1, "document",
													 false);

	Var *distinctVar = makeVar(1, 3, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *distinctEntry = makeTargetEntry((Expr *) distinctVar, 2, "distinctOn",
												 true);
	distinctEntry->ressortgroupref = 1;

	Var *finalDepthVar = makeVar(1, 2, BsonTypeId(), -1, InvalidOid, 0);
	TargetEntry *finalDepthEntry = makeTargetEntry((Expr *) finalDepthVar, 3, "depthVar",
												   true);
	finalDepthEntry->ressortgroupref = 2;

	/* If a depthField is specified, merge the depth value into the document. */
	if (args->depthField.length > 0)
	{
		bool overrideArray = true;
		simpleTargetEntry->expr = (Expr *) makeFuncExpr(
			BsonDollaMergeDocumentsFunctionOid(),
			BsonTypeId(),
			list_make3(simpleVar,
					   finalDepthVar,
					   MakeBoolValueConst(overrideArray)),
			InvalidOid,
			InvalidOid, COERCE_EXPLICIT_CALL);
	}

	Assert(list_length(graphCteExpr->ctecoltypes) == 5);
	graphLookupQuery->targetList = list_make3(simpleTargetEntry, distinctEntry,
											  finalDepthEntry);

	/* Add Distinct ON */
	SortGroupClause *distinctonSortGroup = makeNode(SortGroupClause);
	distinctonSortGroup->eqop = BsonEqualOperatorId();
	distinctonSortGroup->sortop = BsonLessThanOperatorId();
	distinctonSortGroup->hashable = false;
	distinctonSortGroup->tleSortGroupRef = 1;
	graphLookupQuery->distinctClause = list_make1(distinctonSortGroup);
	graphLookupQuery->hasDistinctOn = true;

	/* Add a sort */
	SortGroupClause *sortOnDepthGroup = makeNode(SortGroupClause);
	sortOnDepthGroup->eqop = BsonEqualOperatorId();
	sortOnDepthGroup->sortop = BsonLessThanOperatorId();
	sortOnDepthGroup->hashable = false;
	sortOnDepthGroup->tleSortGroupRef = 2;
	graphLookupQuery->sortClause = list_make2(distinctonSortGroup, sortOnDepthGroup);

	/* Add the bson_array_agg */
	bool migrateToSubQuery = true;
	Aggref *arrayAggRef = NULL;
	graphLookupQuery = AddBsonArrayAggFunction(graphLookupQuery, &subPipelineContext,
											   parseState,
											   args->asField.string, args->asField.length,
											   migrateToSubQuery, &arrayAggRef);

	pfree(parseState);
	return graphLookupQuery;
}


static Query *
HandleLookupCore(const bson_value_t *existingValue, Query *query,
				 AggregationPipelineBuildContext *context,
				 LookupContext *lookupContext)
{
	LookupArgs lookupArgs;
	memset(&lookupArgs, 0, sizeof(LookupArgs));

	if (context->nestedPipelineLevel >= MaximumLookupPipelineDepth)
	{
		ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_MAXSUBPIPELINEDEPTHEXCEEDED),
						errmsg(
							"The allowed limit for nested sub-pipelines has been surpassed, exceeding the maximum of %d.",
							MaximumLookupPipelineDepth)));
	}

	ParseLookupStage(existingValue, &lookupArgs);

	/* Now build the base query for the lookup */
	return ProcessLookupCoreWithLet(query, context, &lookupArgs, lookupContext);
}


static Node *
ReplaceVariablesWithLevelsUpMutator(Node *node, LevelsUpQueryTreeWalkerState *state)
{
	CHECK_FOR_INTERRUPTS();

	if (node == NULL)
	{
		return NULL;
	}

	if (IsA(node, Query))
	{
		state->numLevels++;
		Query *result = query_tree_mutator((Query *) node,
										   ReplaceVariablesWithLevelsUpMutator,
										   state, QTW_EXAMINE_RTES_BEFORE |
										   QTW_DONT_COPY_QUERY | QTW_EXAMINE_SORTGROUP);
		state->numLevels--;
		return (Node *) result;
	}
	else if (IsA(node, Var))
	{
		Var *originalVar = (Var *) node;

		/*
		 * If the location of the VAR >= NESTED_PIPELINE_VAR_FLAG, then this means this VAR needs varlevelsup adjustment
		 */
		if (originalVar->location >= NESTED_PIPELINE_VAR_FLAG &&
			equal(node, state->originalVariable))
		{
			Var *copyVar = copyObject(originalVar);
			copyVar->varlevelsup = state->numLevels;
			copyVar->location = -1;
			return (Node *) copyVar;
		}
	}

	return expression_tree_mutator(node, ReplaceVariablesWithLevelsUpMutator, state);
}


static void
WalkQueryAndSetLevelsUp(Query *rightQuery, Var *varToCheck,
						int varLevelsUpBase)
{
	LevelsUpQueryTreeWalkerState state = { 0 };
	state.numLevels = varLevelsUpBase;
	state.originalVariable = varToCheck;

	query_tree_mutator((Query *) rightQuery,
					   ReplaceVariablesWithLevelsUpMutator, &state,
					   QTW_EXAMINE_RTES_BEFORE |
					   QTW_DONT_COPY_QUERY |
					   QTW_EXAMINE_SORTGROUP);
}


static bool
RangeTblEntryLevelsUpWalker(Node *expr, LevelsUpQueryTreeWalkerState *walkerState)
{
	if (expr == NULL)
	{
		return false;
	}

	if (IsA(expr, Query))
	{
		walkerState->numLevels++;
		query_tree_walker((Query *) expr,
						  RangeTblEntryLevelsUpWalker, walkerState,
						  QTW_EXAMINE_RTES_BEFORE |
						  QTW_DONT_COPY_QUERY);
		walkerState->numLevels--;
		return false;
	}
	else if (IsA(expr, RangeTblEntry))
	{
		RangeTblEntry *tblEntry = (RangeTblEntry *) expr;
		if (tblEntry->rtekind == RTE_CTE && strcmp(tblEntry->ctename,
												   walkerState->cteName) == 0)
		{
			tblEntry->ctelevelsup = walkerState->numLevels;
		}

		return false;
	}

	return expression_tree_walker(expr, RangeTblEntryLevelsUpWalker, walkerState);
}


static void
WalkQueryAndSetCteLevelsUp(Query *rightQuery, const char *cteName,
						   int varLevelsUpBase)
{
	LevelsUpQueryTreeWalkerState state = { 0 };
	state.numLevels = varLevelsUpBase;
	state.cteName = cteName;

	query_tree_walker((Query *) rightQuery,
					  RangeTblEntryLevelsUpWalker, &state,
					  QTW_EXAMINE_RTES_BEFORE |
					  QTW_DONT_COPY_QUERY);
}


/* Function that is passed down to the expression tree when a let expression is found under a lookup.
 * It validates to ensure no let variables are used to define other variables,
 * Operator variables (aliases) are allowed.
 * We only do this if it is the first level let on lookup.
 */
static void
ValidateVariableIsDefined(AggregationExpressionData *parsedExpression,
						  HTAB *operatorVariables)
{
	if (parsedExpression->kind != AggregationExpressionKind_Variable)
	{
		return;
	}

	/* Certain variables are disallowed in the variable spec, system variables are valid and should be considered at runtime
	 * if it is available or not, i.e SEARCH_META is only available if a $search stage was defined.
	 * Operator variables are also allowed. Example: $$this and 'as' alias in $map and $filter
	 * All others are disallowed */
	const bson_value_t parsedVarName = parsedExpression->value;
	const char *nameWithoutPrefix = parsedVarName.value.v_utf8.str + 2;

	if (EnableOperatorVariablesInLookup && operatorVariables)
	{
		/* Get the actual variable name and verify if it's an operator variable alias*/
		StringView variableName =
			CreateStringViewFromStringWithLength(nameWithoutPrefix,
												 parsedVarName.value.v_utf8.len - 2);
		StringView actualVariableName = StringViewFindPrefix(&variableName, '.');

		/* actualVariableName is empty when no '.' was found in the variable name */
		if (actualVariableName.length == 0)
		{
			actualVariableName = variableName;
		}

		bool found = false;
		hash_search(operatorVariables, &actualVariableName,
					HASH_FIND, &found);

		if (found)
		{
			return;
		}
	}

	ereport(ERROR, (errcode(ERRCODE_DOCUMENTDB_LOCATION17276),
					errmsg("Attempting to use an undefined variable: %s",
						   nameWithoutPrefix)));
}
