/*------------------------------------------------------------------------- * * llvmjit_expr.c * JIT compile expressions. * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/jit/llvm/llvmjit_expr.c * *------------------------------------------------------------------------- */ #include "postgres.h" #include #include #include "access/htup_details.h" #include "access/nbtree.h" #include "catalog/objectaccess.h" #include "catalog/pg_type.h" #include "executor/execExpr.h" #include "executor/execdebug.h" #include "executor/nodeAgg.h" #include "executor/nodeSubplan.h" #include "funcapi.h" #include "jit/llvmjit.h" #include "jit/llvmjit_emit.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_coerce.h" #include "parser/parsetree.h" #include "pgstat.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/date.h" #include "utils/fmgrtab.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/timestamp.h" #include "utils/typcache.h" #include "utils/xml.h" typedef struct CompiledExprState { LLVMJitContext *context; const char *funcname; } CompiledExprState; static Datum ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull); static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMModuleRef mod, FunctionCallInfo fcinfo, LLVMValueRef *v_fcinfo_isnull); static LLVMValueRef build_EvalXFuncInt(LLVMBuilderRef b, LLVMModuleRef mod, const char *funcname, LLVMValueRef v_state, ExprEvalStep *op, int natts, LLVMValueRef *v_args); static LLVMValueRef create_LifetimeEnd(LLVMModuleRef mod); /* macro making it easier to call ExecEval* functions */ #define build_EvalXFunc(b, mod, funcname, v_state, op, ...) \ build_EvalXFuncInt(b, mod, funcname, v_state, op, \ lengthof(((LLVMValueRef[]){__VA_ARGS__})), \ ((LLVMValueRef[]){__VA_ARGS__})) /* * JIT compile expression. */ bool llvm_compile_expr(ExprState *state) { PlanState *parent = state->parent; char *funcname; LLVMJitContext *context = NULL; LLVMBuilderRef b; LLVMModuleRef mod; LLVMContextRef lc; LLVMValueRef eval_fn; LLVMBasicBlockRef entry; LLVMBasicBlockRef *opblocks; /* state itself */ LLVMValueRef v_state; LLVMValueRef v_econtext; LLVMValueRef v_parent; /* returnvalue */ LLVMValueRef v_isnullp; /* tmp vars in state */ LLVMValueRef v_tmpvaluep; LLVMValueRef v_tmpisnullp; /* slots */ LLVMValueRef v_innerslot; LLVMValueRef v_outerslot; LLVMValueRef v_scanslot; LLVMValueRef v_oldslot; LLVMValueRef v_newslot; LLVMValueRef v_resultslot; /* nulls/values of slots */ LLVMValueRef v_innervalues; LLVMValueRef v_innernulls; LLVMValueRef v_outervalues; LLVMValueRef v_outernulls; LLVMValueRef v_scanvalues; LLVMValueRef v_scannulls; LLVMValueRef v_oldvalues; LLVMValueRef v_oldnulls; LLVMValueRef v_newvalues; LLVMValueRef v_newnulls; LLVMValueRef v_resultvalues; LLVMValueRef v_resultnulls; /* stuff in econtext */ LLVMValueRef v_aggvalues; LLVMValueRef v_aggnulls; instr_time starttime; instr_time deform_starttime; instr_time endtime; instr_time deform_endtime; llvm_enter_fatal_on_oom(); /* * Right now we don't support compiling expressions without a parent, as * we need access to the EState. */ Assert(parent); /* get or create JIT context */ if (parent->state->es_jit) context = (LLVMJitContext *) parent->state->es_jit; else { context = llvm_create_context(parent->state->es_jit_flags); parent->state->es_jit = &context->base; } INSTR_TIME_SET_CURRENT(starttime); mod = llvm_mutable_module(context); lc = LLVMGetModuleContext(mod); b = LLVMCreateBuilderInContext(lc); funcname = llvm_expand_funcname(context, "evalexpr"); /* create function */ eval_fn = LLVMAddFunction(mod, funcname, llvm_pg_var_func_type("ExecInterpExprStillValid")); LLVMSetLinkage(eval_fn, LLVMExternalLinkage); LLVMSetVisibility(eval_fn, LLVMDefaultVisibility); llvm_copy_attributes(AttributeTemplate, eval_fn); entry = LLVMAppendBasicBlockInContext(lc, eval_fn, "entry"); /* build state */ v_state = LLVMGetParam(eval_fn, 0); v_econtext = LLVMGetParam(eval_fn, 1); v_isnullp = LLVMGetParam(eval_fn, 2); LLVMPositionBuilderAtEnd(b, entry); v_tmpvaluep = l_struct_gep(b, StructExprState, v_state, FIELDNO_EXPRSTATE_RESVALUE, "v.state.resvalue"); v_tmpisnullp = l_struct_gep(b, StructExprState, v_state, FIELDNO_EXPRSTATE_RESNULL, "v.state.resnull"); v_parent = l_load_struct_gep(b, StructExprState, v_state, FIELDNO_EXPRSTATE_PARENT, "v.state.parent"); /* build global slots */ v_scanslot = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_SCANTUPLE, "v_scanslot"); v_innerslot = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_INNERTUPLE, "v_innerslot"); v_outerslot = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_OUTERTUPLE, "v_outerslot"); v_oldslot = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_OLDTUPLE, "v_oldslot"); v_newslot = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_NEWTUPLE, "v_newslot"); v_resultslot = l_load_struct_gep(b, StructExprState, v_state, FIELDNO_EXPRSTATE_RESULTSLOT, "v_resultslot"); /* build global values/isnull pointers */ v_scanvalues = l_load_struct_gep(b, StructTupleTableSlot, v_scanslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_scanvalues"); v_scannulls = l_load_struct_gep(b, StructTupleTableSlot, v_scanslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_scannulls"); v_innervalues = l_load_struct_gep(b, StructTupleTableSlot, v_innerslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_innervalues"); v_innernulls = l_load_struct_gep(b, StructTupleTableSlot, v_innerslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_innernulls"); v_outervalues = l_load_struct_gep(b, StructTupleTableSlot, v_outerslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_outervalues"); v_outernulls = l_load_struct_gep(b, StructTupleTableSlot, v_outerslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_outernulls"); v_oldvalues = l_load_struct_gep(b, StructTupleTableSlot, v_oldslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_oldvalues"); v_oldnulls = l_load_struct_gep(b, StructTupleTableSlot, v_oldslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_oldnulls"); v_newvalues = l_load_struct_gep(b, StructTupleTableSlot, v_newslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_newvalues"); v_newnulls = l_load_struct_gep(b, StructTupleTableSlot, v_newslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_newnulls"); v_resultvalues = l_load_struct_gep(b, StructTupleTableSlot, v_resultslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_resultvalues"); v_resultnulls = l_load_struct_gep(b, StructTupleTableSlot, v_resultslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_resultnulls"); /* aggvalues/aggnulls */ v_aggvalues = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_AGGVALUES, "v.econtext.aggvalues"); v_aggnulls = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_AGGNULLS, "v.econtext.aggnulls"); /* allocate blocks for each op upfront, so we can do jumps easily */ opblocks = palloc(sizeof(LLVMBasicBlockRef) * state->steps_len); for (int opno = 0; opno < state->steps_len; opno++) opblocks[opno] = l_bb_append_v(eval_fn, "b.op.%d.start", opno); /* jump from entry to first block */ LLVMBuildBr(b, opblocks[0]); for (int opno = 0; opno < state->steps_len; opno++) { ExprEvalStep *op; ExprEvalOp opcode; LLVMValueRef v_resvaluep; LLVMValueRef v_resnullp; LLVMPositionBuilderAtEnd(b, opblocks[opno]); op = &state->steps[opno]; opcode = ExecEvalStepOp(state, op); v_resvaluep = l_ptr_const(op->resvalue, l_ptr(TypeSizeT)); v_resnullp = l_ptr_const(op->resnull, l_ptr(TypeStorageBool)); switch (opcode) { case EEOP_DONE_RETURN: { LLVMValueRef v_tmpisnull; LLVMValueRef v_tmpvalue; v_tmpvalue = l_load(b, TypeSizeT, v_tmpvaluep, ""); v_tmpisnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); LLVMBuildStore(b, v_tmpisnull, v_isnullp); LLVMBuildRet(b, v_tmpvalue); break; } case EEOP_DONE_NO_RETURN: LLVMBuildRet(b, l_sizet_const(0)); break; case EEOP_INNER_FETCHSOME: case EEOP_OUTER_FETCHSOME: case EEOP_SCAN_FETCHSOME: case EEOP_OLD_FETCHSOME: case EEOP_NEW_FETCHSOME: { TupleDesc desc = NULL; LLVMValueRef v_slot; LLVMBasicBlockRef b_fetch; LLVMValueRef v_nvalid; LLVMValueRef l_jit_deform = NULL; const TupleTableSlotOps *tts_ops = NULL; b_fetch = l_bb_before_v(opblocks[opno + 1], "op.%d.fetch", opno); if (op->d.fetch.known_desc) desc = op->d.fetch.known_desc; if (op->d.fetch.fixed) tts_ops = op->d.fetch.kind; /* step should not have been generated */ Assert(tts_ops != &TTSOpsVirtual); if (opcode == EEOP_INNER_FETCHSOME) v_slot = v_innerslot; else if (opcode == EEOP_OUTER_FETCHSOME) v_slot = v_outerslot; else if (opcode == EEOP_SCAN_FETCHSOME) v_slot = v_scanslot; else if (opcode == EEOP_OLD_FETCHSOME) v_slot = v_oldslot; else v_slot = v_newslot; /* * Check if all required attributes are available, or * whether deforming is required. */ v_nvalid = l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntUGE, v_nvalid, l_int16_const(lc, op->d.fetch.last_var), ""), opblocks[opno + 1], b_fetch); LLVMPositionBuilderAtEnd(b, b_fetch); /* * If the tupledesc of the to-be-deformed tuple is known, * and JITing of deforming is enabled, build deform * function specific to tupledesc and the exact number of * to-be-extracted attributes. */ if (tts_ops && desc && (context->base.flags & PGJIT_DEFORM)) { INSTR_TIME_SET_CURRENT(deform_starttime); l_jit_deform = slot_compile_deform(context, desc, tts_ops, op->d.fetch.last_var); INSTR_TIME_SET_CURRENT(deform_endtime); INSTR_TIME_ACCUM_DIFF(context->base.instr.deform_counter, deform_endtime, deform_starttime); } if (l_jit_deform) { LLVMValueRef params[1]; params[0] = v_slot; l_call(b, LLVMGetFunctionType(l_jit_deform), l_jit_deform, params, lengthof(params), ""); } else { LLVMValueRef params[2]; params[0] = v_slot; params[1] = l_int32_const(lc, op->d.fetch.last_var); l_call(b, llvm_pg_var_func_type("slot_getsomeattrs_int"), llvm_pg_func(mod, "slot_getsomeattrs_int"), params, lengthof(params), ""); } LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_INNER_VAR: case EEOP_OUTER_VAR: case EEOP_SCAN_VAR: case EEOP_OLD_VAR: case EEOP_NEW_VAR: { LLVMValueRef value, isnull; LLVMValueRef v_attnum; LLVMValueRef v_values; LLVMValueRef v_nulls; if (opcode == EEOP_INNER_VAR) { v_values = v_innervalues; v_nulls = v_innernulls; } else if (opcode == EEOP_OUTER_VAR) { v_values = v_outervalues; v_nulls = v_outernulls; } else if (opcode == EEOP_SCAN_VAR) { v_values = v_scanvalues; v_nulls = v_scannulls; } else if (opcode == EEOP_OLD_VAR) { v_values = v_oldvalues; v_nulls = v_oldnulls; } else { v_values = v_newvalues; v_nulls = v_newnulls; } v_attnum = l_int32_const(lc, op->d.var.attnum); value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); LLVMBuildStore(b, value, v_resvaluep); LLVMBuildStore(b, isnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_INNER_SYSVAR: case EEOP_OUTER_SYSVAR: case EEOP_SCAN_SYSVAR: case EEOP_OLD_SYSVAR: case EEOP_NEW_SYSVAR: { LLVMValueRef v_slot; if (opcode == EEOP_INNER_SYSVAR) v_slot = v_innerslot; else if (opcode == EEOP_OUTER_SYSVAR) v_slot = v_outerslot; else if (opcode == EEOP_SCAN_SYSVAR) v_slot = v_scanslot; else if (opcode == EEOP_OLD_SYSVAR) v_slot = v_oldslot; else v_slot = v_newslot; build_EvalXFunc(b, mod, "ExecEvalSysVar", v_state, op, v_econtext, v_slot); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_WHOLEROW: build_EvalXFunc(b, mod, "ExecEvalWholeRowVar", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_ASSIGN_INNER_VAR: case EEOP_ASSIGN_OUTER_VAR: case EEOP_ASSIGN_SCAN_VAR: case EEOP_ASSIGN_OLD_VAR: case EEOP_ASSIGN_NEW_VAR: { LLVMValueRef v_value; LLVMValueRef v_isnull; LLVMValueRef v_rvaluep; LLVMValueRef v_risnullp; LLVMValueRef v_attnum; LLVMValueRef v_resultnum; LLVMValueRef v_values; LLVMValueRef v_nulls; if (opcode == EEOP_ASSIGN_INNER_VAR) { v_values = v_innervalues; v_nulls = v_innernulls; } else if (opcode == EEOP_ASSIGN_OUTER_VAR) { v_values = v_outervalues; v_nulls = v_outernulls; } else if (opcode == EEOP_ASSIGN_SCAN_VAR) { v_values = v_scanvalues; v_nulls = v_scannulls; } else if (opcode == EEOP_ASSIGN_OLD_VAR) { v_values = v_oldvalues; v_nulls = v_oldnulls; } else { v_values = v_newvalues; v_nulls = v_newnulls; } /* load data */ v_attnum = l_int32_const(lc, op->d.assign_var.attnum); v_value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); v_isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); /* compute addresses of targets */ v_resultnum = l_int32_const(lc, op->d.assign_var.resultnum); v_rvaluep = l_gep(b, TypeSizeT, v_resultvalues, &v_resultnum, 1, ""); v_risnullp = l_gep(b, TypeStorageBool, v_resultnulls, &v_resultnum, 1, ""); /* and store */ LLVMBuildStore(b, v_value, v_rvaluep); LLVMBuildStore(b, v_isnull, v_risnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_ASSIGN_TMP: case EEOP_ASSIGN_TMP_MAKE_RO: { LLVMValueRef v_value, v_isnull; LLVMValueRef v_rvaluep, v_risnullp; LLVMValueRef v_resultnum; size_t resultnum = op->d.assign_tmp.resultnum; /* load data */ v_value = l_load(b, TypeSizeT, v_tmpvaluep, ""); v_isnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); /* compute addresses of targets */ v_resultnum = l_int32_const(lc, resultnum); v_rvaluep = l_gep(b, TypeSizeT, v_resultvalues, &v_resultnum, 1, ""); v_risnullp = l_gep(b, TypeStorageBool, v_resultnulls, &v_resultnum, 1, ""); /* store nullness */ LLVMBuildStore(b, v_isnull, v_risnullp); /* make value readonly if necessary */ if (opcode == EEOP_ASSIGN_TMP_MAKE_RO) { LLVMBasicBlockRef b_notnull; LLVMValueRef v_params[1]; b_notnull = l_bb_before_v(opblocks[opno + 1], "op.%d.assign_tmp.notnull", opno); /* check if value is NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_isnull, l_sbool_const(0), ""), b_notnull, opblocks[opno + 1]); /* if value is not null, convert to RO datum */ LLVMPositionBuilderAtEnd(b, b_notnull); v_params[0] = v_value; v_value = l_call(b, llvm_pg_var_func_type("MakeExpandedObjectReadOnlyInternal"), llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), v_params, lengthof(v_params), ""); /* * Falling out of the if () with builder in b_notnull, * which is fine - the null is already stored above. */ } /* and finally store result */ LLVMBuildStore(b, v_value, v_rvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_CONST: { LLVMValueRef v_constvalue, v_constnull; v_constvalue = l_sizet_const(op->d.constval.value); v_constnull = l_sbool_const(op->d.constval.isnull); LLVMBuildStore(b, v_constvalue, v_resvaluep); LLVMBuildStore(b, v_constnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_FUNCEXPR: case EEOP_FUNCEXPR_STRICT: case EEOP_FUNCEXPR_STRICT_1: case EEOP_FUNCEXPR_STRICT_2: { FunctionCallInfo fcinfo = op->d.func.fcinfo_data; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_retval; if (opcode == EEOP_FUNCEXPR_STRICT || opcode == EEOP_FUNCEXPR_STRICT_1 || opcode == EEOP_FUNCEXPR_STRICT_2) { LLVMBasicBlockRef b_nonull; LLVMBasicBlockRef *b_checkargnulls; LLVMValueRef v_fcinfo; /* * Block for the actual function call, if args are * non-NULL. */ b_nonull = l_bb_before_v(opblocks[opno + 1], "b.%d.no-null-args", opno); /* should make sure they're optimized beforehand */ if (op->d.func.nargs == 0) elog(ERROR, "argumentless strict functions are pointless"); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); /* * set resnull to true, if the function is actually * called, it'll be reset */ LLVMBuildStore(b, l_sbool_const(1), v_resnullp); /* create blocks for checking args, one for each */ b_checkargnulls = palloc(sizeof(LLVMBasicBlockRef *) * op->d.func.nargs); for (int argno = 0; argno < op->d.func.nargs; argno++) b_checkargnulls[argno] = l_bb_before_v(b_nonull, "b.%d.isnull.%d", opno, argno); /* jump to check of first argument */ LLVMBuildBr(b, b_checkargnulls[0]); /* check each arg for NULLness */ for (int argno = 0; argno < op->d.func.nargs; argno++) { LLVMValueRef v_argisnull; LLVMBasicBlockRef b_argnotnull; LLVMPositionBuilderAtEnd(b, b_checkargnulls[argno]); /* * Compute block to jump to if argument is not * null. */ if (argno + 1 == op->d.func.nargs) b_argnotnull = b_nonull; else b_argnotnull = b_checkargnulls[argno + 1]; /* and finally load & check NULLness of arg */ v_argisnull = l_funcnull(b, v_fcinfo, argno); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argisnull, l_sbool_const(1), ""), opblocks[opno + 1], b_argnotnull); } LLVMPositionBuilderAtEnd(b, b_nonull); } v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); LLVMBuildStore(b, v_retval, v_resvaluep); LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_FUNCEXPR_FUSAGE: build_EvalXFunc(b, mod, "ExecEvalFuncExprFusage", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_FUNCEXPR_STRICT_FUSAGE: build_EvalXFunc(b, mod, "ExecEvalFuncExprStrictFusage", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; /* * Treat them the same for now, optimizer can remove * redundancy. Could be worthwhile to optimize during emission * though. */ case EEOP_BOOL_AND_STEP_FIRST: case EEOP_BOOL_AND_STEP: case EEOP_BOOL_AND_STEP_LAST: { LLVMValueRef v_boolvalue; LLVMValueRef v_boolnull; LLVMValueRef v_boolanynullp, v_boolanynull; LLVMBasicBlockRef b_boolisnull; LLVMBasicBlockRef b_boolcheckfalse; LLVMBasicBlockRef b_boolisfalse; LLVMBasicBlockRef b_boolcont; LLVMBasicBlockRef b_boolisanynull; b_boolisnull = l_bb_before_v(opblocks[opno + 1], "b.%d.boolisnull", opno); b_boolcheckfalse = l_bb_before_v(opblocks[opno + 1], "b.%d.boolcheckfalse", opno); b_boolisfalse = l_bb_before_v(opblocks[opno + 1], "b.%d.boolisfalse", opno); b_boolisanynull = l_bb_before_v(opblocks[opno + 1], "b.%d.boolisanynull", opno); b_boolcont = l_bb_before_v(opblocks[opno + 1], "b.%d.boolcont", opno); v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull, l_ptr(TypeStorageBool)); if (opcode == EEOP_BOOL_AND_STEP_FIRST) LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp); v_boolnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); /* check if current input is NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolnull, l_sbool_const(1), ""), b_boolisnull, b_boolcheckfalse); /* build block that sets anynull */ LLVMPositionBuilderAtEnd(b, b_boolisnull); /* set boolanynull to true */ LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp); /* and jump to next block */ LLVMBuildBr(b, b_boolcont); /* build block checking for false */ LLVMPositionBuilderAtEnd(b, b_boolcheckfalse); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue, l_sizet_const(0), ""), b_boolisfalse, b_boolcont); /* * Build block handling FALSE. Value is false, so short * circuit. */ LLVMPositionBuilderAtEnd(b, b_boolisfalse); /* result is already set to FALSE, need not change it */ /* and jump to the end of the AND expression */ LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]); /* Build block that continues if bool is TRUE. */ LLVMPositionBuilderAtEnd(b, b_boolcont); v_boolanynull = l_load(b, TypeStorageBool, v_boolanynullp, ""); /* set value to NULL if any previous values were NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull, l_sbool_const(0), ""), opblocks[opno + 1], b_boolisanynull); LLVMPositionBuilderAtEnd(b, b_boolisanynull); /* set resnull to true */ LLVMBuildStore(b, l_sbool_const(1), v_resnullp); /* reset resvalue */ LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } /* * Treat them the same for now, optimizer can remove * redundancy. Could be worthwhile to optimize during emission * though. */ case EEOP_BOOL_OR_STEP_FIRST: case EEOP_BOOL_OR_STEP: case EEOP_BOOL_OR_STEP_LAST: { LLVMValueRef v_boolvalue; LLVMValueRef v_boolnull; LLVMValueRef v_boolanynullp, v_boolanynull; LLVMBasicBlockRef b_boolisnull; LLVMBasicBlockRef b_boolchecktrue; LLVMBasicBlockRef b_boolistrue; LLVMBasicBlockRef b_boolcont; LLVMBasicBlockRef b_boolisanynull; b_boolisnull = l_bb_before_v(opblocks[opno + 1], "b.%d.boolisnull", opno); b_boolchecktrue = l_bb_before_v(opblocks[opno + 1], "b.%d.boolchecktrue", opno); b_boolistrue = l_bb_before_v(opblocks[opno + 1], "b.%d.boolistrue", opno); b_boolisanynull = l_bb_before_v(opblocks[opno + 1], "b.%d.boolisanynull", opno); b_boolcont = l_bb_before_v(opblocks[opno + 1], "b.%d.boolcont", opno); v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull, l_ptr(TypeStorageBool)); if (opcode == EEOP_BOOL_OR_STEP_FIRST) LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp); v_boolnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolnull, l_sbool_const(1), ""), b_boolisnull, b_boolchecktrue); /* build block that sets anynull */ LLVMPositionBuilderAtEnd(b, b_boolisnull); /* set boolanynull to true */ LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp); /* and jump to next block */ LLVMBuildBr(b, b_boolcont); /* build block checking for true */ LLVMPositionBuilderAtEnd(b, b_boolchecktrue); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue, l_sizet_const(1), ""), b_boolistrue, b_boolcont); /* * Build block handling True. Value is true, so short * circuit. */ LLVMPositionBuilderAtEnd(b, b_boolistrue); /* result is already set to TRUE, need not change it */ /* and jump to the end of the OR expression */ LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]); /* build block that continues if bool is FALSE */ LLVMPositionBuilderAtEnd(b, b_boolcont); v_boolanynull = l_load(b, TypeStorageBool, v_boolanynullp, ""); /* set value to NULL if any previous values were NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull, l_sbool_const(0), ""), opblocks[opno + 1], b_boolisanynull); LLVMPositionBuilderAtEnd(b, b_boolisanynull); /* set resnull to true */ LLVMBuildStore(b, l_sbool_const(1), v_resnullp); /* reset resvalue */ LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_BOOL_NOT_STEP: { LLVMValueRef v_boolvalue; LLVMValueRef v_negbool; /* compute !boolvalue */ v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); v_negbool = LLVMBuildZExt(b, LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue, l_sizet_const(0), ""), TypeSizeT, ""); /* * Store it back in resvalue. We can ignore resnull here; * if it was true, it stays true, and the value we store * in resvalue doesn't matter. */ LLVMBuildStore(b, v_negbool, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_QUAL: { LLVMValueRef v_resnull; LLVMValueRef v_resvalue; LLVMValueRef v_nullorfalse; LLVMBasicBlockRef b_qualfail; b_qualfail = l_bb_before_v(opblocks[opno + 1], "op.%d.qualfail", opno); v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_nullorfalse = LLVMBuildOr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), LLVMBuildICmp(b, LLVMIntEQ, v_resvalue, l_sizet_const(0), ""), ""); LLVMBuildCondBr(b, v_nullorfalse, b_qualfail, opblocks[opno + 1]); /* build block handling NULL or false */ LLVMPositionBuilderAtEnd(b, b_qualfail); /* set resnull to false */ LLVMBuildStore(b, l_sbool_const(0), v_resnullp); /* set resvalue to false */ LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); /* and jump out */ LLVMBuildBr(b, opblocks[op->d.qualexpr.jumpdone]); break; } case EEOP_JUMP: { LLVMBuildBr(b, opblocks[op->d.jump.jumpdone]); break; } case EEOP_JUMP_IF_NULL: { LLVMValueRef v_resnull; /* Transfer control if current result is null */ v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), opblocks[op->d.jump.jumpdone], opblocks[opno + 1]); break; } case EEOP_JUMP_IF_NOT_NULL: { LLVMValueRef v_resnull; /* Transfer control if current result is non-null */ v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(0), ""), opblocks[op->d.jump.jumpdone], opblocks[opno + 1]); break; } case EEOP_JUMP_IF_NOT_TRUE: { LLVMValueRef v_resnull; LLVMValueRef v_resvalue; LLVMValueRef v_nullorfalse; /* Transfer control if current result is null or false */ v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_nullorfalse = LLVMBuildOr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), LLVMBuildICmp(b, LLVMIntEQ, v_resvalue, l_sizet_const(0), ""), ""); LLVMBuildCondBr(b, v_nullorfalse, opblocks[op->d.jump.jumpdone], opblocks[opno + 1]); break; } case EEOP_NULLTEST_ISNULL: { LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMValueRef v_resvalue; v_resvalue = LLVMBuildSelect(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), l_sizet_const(1), l_sizet_const(0), ""); LLVMBuildStore(b, v_resvalue, v_resvaluep); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_NULLTEST_ISNOTNULL: { LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMValueRef v_resvalue; v_resvalue = LLVMBuildSelect(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), l_sizet_const(0), l_sizet_const(1), ""); LLVMBuildStore(b, v_resvalue, v_resvaluep); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_NULLTEST_ROWISNULL: build_EvalXFunc(b, mod, "ExecEvalRowNull", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_NULLTEST_ROWISNOTNULL: build_EvalXFunc(b, mod, "ExecEvalRowNotNull", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_BOOLTEST_IS_TRUE: case EEOP_BOOLTEST_IS_NOT_FALSE: case EEOP_BOOLTEST_IS_FALSE: case EEOP_BOOLTEST_IS_NOT_TRUE: { LLVMBasicBlockRef b_isnull, b_notnull; LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); b_isnull = l_bb_before_v(opblocks[opno + 1], "op.%d.isnull", opno); b_notnull = l_bb_before_v(opblocks[opno + 1], "op.%d.isnotnull", opno); /* check if value is NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), b_isnull, b_notnull); /* if value is NULL, return false */ LLVMPositionBuilderAtEnd(b, b_isnull); /* result is not null */ LLVMBuildStore(b, l_sbool_const(0), v_resnullp); if (opcode == EEOP_BOOLTEST_IS_TRUE || opcode == EEOP_BOOLTEST_IS_FALSE) { LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); } else { LLVMBuildStore(b, l_sizet_const(1), v_resvaluep); } LLVMBuildBr(b, opblocks[opno + 1]); LLVMPositionBuilderAtEnd(b, b_notnull); if (opcode == EEOP_BOOLTEST_IS_TRUE || opcode == EEOP_BOOLTEST_IS_NOT_FALSE) { /* * if value is not null NULL, return value (already * set) */ } else { LLVMValueRef v_value = l_load(b, TypeSizeT, v_resvaluep, ""); v_value = LLVMBuildZExt(b, LLVMBuildICmp(b, LLVMIntEQ, v_value, l_sizet_const(0), ""), TypeSizeT, ""); LLVMBuildStore(b, v_value, v_resvaluep); } LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_PARAM_EXEC: build_EvalXFunc(b, mod, "ExecEvalParamExec", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_PARAM_EXTERN: build_EvalXFunc(b, mod, "ExecEvalParamExtern", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_PARAM_CALLBACK: { LLVMValueRef v_func; LLVMValueRef v_params[3]; v_func = l_ptr_const(op->d.cparam.paramfunc, llvm_pg_var_type("TypeExecEvalSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; l_call(b, LLVMGetFunctionType(ExecEvalSubroutineTemplate), v_func, v_params, lengthof(v_params), ""); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_PARAM_SET: build_EvalXFunc(b, mod, "ExecEvalParamSet", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_SBSREF_SUBSCRIPTS: { int jumpdone = op->d.sbsref_subscript.jumpdone; LLVMValueRef v_func; LLVMValueRef v_params[3]; LLVMValueRef v_ret; v_func = l_ptr_const(op->d.sbsref_subscript.subscriptfunc, llvm_pg_var_type("TypeExecEvalBoolSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; v_ret = l_call(b, LLVMGetFunctionType(ExecEvalBoolSubroutineTemplate), v_func, v_params, lengthof(v_params), ""); v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_ret, l_sbool_const(1), ""), opblocks[opno + 1], opblocks[jumpdone]); break; } case EEOP_SBSREF_OLD: case EEOP_SBSREF_ASSIGN: case EEOP_SBSREF_FETCH: { LLVMValueRef v_func; LLVMValueRef v_params[3]; v_func = l_ptr_const(op->d.sbsref.subscriptfunc, llvm_pg_var_type("TypeExecEvalSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; l_call(b, LLVMGetFunctionType(ExecEvalSubroutineTemplate), v_func, v_params, lengthof(v_params), ""); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_CASE_TESTVAL: { LLVMValueRef v_casevaluep, v_casevalue; LLVMValueRef v_casenullp, v_casenull; v_casevaluep = l_ptr_const(op->d.casetest.value, l_ptr(TypeSizeT)); v_casenullp = l_ptr_const(op->d.casetest.isnull, l_ptr(TypeStorageBool)); v_casevalue = l_load(b, TypeSizeT, v_casevaluep, ""); v_casenull = l_load(b, TypeStorageBool, v_casenullp, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_CASE_TESTVAL_EXT: { LLVMValueRef v_casevalue; LLVMValueRef v_casenull; v_casevalue = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_CASEDATUM, ""); v_casenull = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_CASENULL, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_MAKE_READONLY: { LLVMBasicBlockRef b_notnull; LLVMValueRef v_params[1]; LLVMValueRef v_ret; LLVMValueRef v_nullp; LLVMValueRef v_valuep; LLVMValueRef v_null; LLVMValueRef v_value; b_notnull = l_bb_before_v(opblocks[opno + 1], "op.%d.readonly.notnull", opno); v_nullp = l_ptr_const(op->d.make_readonly.isnull, l_ptr(TypeStorageBool)); v_null = l_load(b, TypeStorageBool, v_nullp, ""); /* store null isnull value in result */ LLVMBuildStore(b, v_null, v_resnullp); /* check if value is NULL */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_null, l_sbool_const(1), ""), opblocks[opno + 1], b_notnull); /* if value is not null, convert to RO datum */ LLVMPositionBuilderAtEnd(b, b_notnull); v_valuep = l_ptr_const(op->d.make_readonly.value, l_ptr(TypeSizeT)); v_value = l_load(b, TypeSizeT, v_valuep, ""); v_params[0] = v_value; v_ret = l_call(b, llvm_pg_var_func_type("MakeExpandedObjectReadOnlyInternal"), llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), v_params, lengthof(v_params), ""); LLVMBuildStore(b, v_ret, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_IOCOERCE: { FunctionCallInfo fcinfo_out, fcinfo_in; LLVMValueRef v_fn_out, v_fn_in; LLVMValueRef v_fcinfo_out, v_fcinfo_in; LLVMValueRef v_fcinfo_in_isnullp; LLVMValueRef v_retval; LLVMValueRef v_resvalue; LLVMValueRef v_resnull; LLVMValueRef v_output_skip; LLVMValueRef v_output; LLVMBasicBlockRef b_skipoutput; LLVMBasicBlockRef b_calloutput; LLVMBasicBlockRef b_input; LLVMBasicBlockRef b_inputcall; fcinfo_out = op->d.iocoerce.fcinfo_data_out; fcinfo_in = op->d.iocoerce.fcinfo_data_in; b_skipoutput = l_bb_before_v(opblocks[opno + 1], "op.%d.skipoutputnull", opno); b_calloutput = l_bb_before_v(opblocks[opno + 1], "op.%d.calloutput", opno); b_input = l_bb_before_v(opblocks[opno + 1], "op.%d.input", opno); b_inputcall = l_bb_before_v(opblocks[opno + 1], "op.%d.inputcall", opno); v_fn_out = llvm_function_reference(context, b, mod, fcinfo_out); v_fn_in = llvm_function_reference(context, b, mod, fcinfo_in); v_fcinfo_out = l_ptr_const(fcinfo_out, l_ptr(StructFunctionCallInfoData)); v_fcinfo_in = l_ptr_const(fcinfo_in, l_ptr(StructFunctionCallInfoData)); v_fcinfo_in_isnullp = l_struct_gep(b, StructFunctionCallInfoData, v_fcinfo_in, FIELDNO_FUNCTIONCALLINFODATA_ISNULL, "v_fcinfo_in_isnull"); /* output functions are not called on nulls */ v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), b_skipoutput, b_calloutput); LLVMPositionBuilderAtEnd(b, b_skipoutput); v_output_skip = l_sizet_const(0); LLVMBuildBr(b, b_input); LLVMPositionBuilderAtEnd(b, b_calloutput); v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); /* set arg[0] */ LLVMBuildStore(b, v_resvalue, l_funcvaluep(b, v_fcinfo_out, 0)); LLVMBuildStore(b, l_sbool_const(0), l_funcnullp(b, v_fcinfo_out, 0)); /* and call output function (can never return NULL) */ v_output = l_call(b, LLVMGetFunctionType(v_fn_out), v_fn_out, &v_fcinfo_out, 1, "funccall_coerce_out"); LLVMBuildBr(b, b_input); /* build block handling input function call */ LLVMPositionBuilderAtEnd(b, b_input); /* phi between resnull and output function call branches */ { LLVMValueRef incoming_values[2]; LLVMBasicBlockRef incoming_blocks[2]; incoming_values[0] = v_output_skip; incoming_blocks[0] = b_skipoutput; incoming_values[1] = v_output; incoming_blocks[1] = b_calloutput; v_output = LLVMBuildPhi(b, TypeSizeT, "output"); LLVMAddIncoming(v_output, incoming_values, incoming_blocks, lengthof(incoming_blocks)); } /* * If input function is strict, skip if input string is * NULL. */ if (op->d.iocoerce.finfo_in->fn_strict) { LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_output, l_sizet_const(0), ""), opblocks[opno + 1], b_inputcall); } else { LLVMBuildBr(b, b_inputcall); } LLVMPositionBuilderAtEnd(b, b_inputcall); /* set arguments */ /* arg0: output */ LLVMBuildStore(b, v_output, l_funcvaluep(b, v_fcinfo_in, 0)); LLVMBuildStore(b, v_resnull, l_funcnullp(b, v_fcinfo_in, 0)); /* arg1: ioparam: preset in execExpr.c */ /* arg2: typmod: preset in execExpr.c */ /* reset fcinfo_in->isnull */ LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_in_isnullp); /* and call function */ v_retval = l_call(b, LLVMGetFunctionType(v_fn_in), v_fn_in, &v_fcinfo_in, 1, "funccall_iocoerce_in"); LLVMBuildStore(b, v_retval, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_IOCOERCE_SAFE: build_EvalXFunc(b, mod, "ExecEvalCoerceViaIOSafe", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_DISTINCT: case EEOP_NOT_DISTINCT: { FunctionCallInfo fcinfo = op->d.func.fcinfo_data; LLVMValueRef v_fcinfo; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_argnull0, v_argisnull0; LLVMValueRef v_argnull1, v_argisnull1; LLVMValueRef v_anyargisnull; LLVMValueRef v_bothargisnull; LLVMValueRef v_result; LLVMBasicBlockRef b_noargnull; LLVMBasicBlockRef b_checkbothargnull; LLVMBasicBlockRef b_bothargnull; LLVMBasicBlockRef b_anyargnull; b_noargnull = l_bb_before_v(opblocks[opno + 1], "op.%d.noargnull", opno); b_checkbothargnull = l_bb_before_v(opblocks[opno + 1], "op.%d.checkbothargnull", opno); b_bothargnull = l_bb_before_v(opblocks[opno + 1], "op.%d.bothargnull", opno); b_anyargnull = l_bb_before_v(opblocks[opno + 1], "op.%d.anyargnull", opno); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); /* load args[0|1].isnull for both arguments */ v_argnull0 = l_funcnull(b, v_fcinfo, 0); v_argisnull0 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull0, l_sbool_const(1), ""); v_argnull1 = l_funcnull(b, v_fcinfo, 1); v_argisnull1 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull1, l_sbool_const(1), ""); v_anyargisnull = LLVMBuildOr(b, v_argisnull0, v_argisnull1, ""); v_bothargisnull = LLVMBuildAnd(b, v_argisnull0, v_argisnull1, ""); /* * Check function arguments for NULLness: If either is * NULL, we check if both args are NULL. Otherwise call * comparator. */ LLVMBuildCondBr(b, v_anyargisnull, b_checkbothargnull, b_noargnull); /* * build block checking if any arg is null */ LLVMPositionBuilderAtEnd(b, b_checkbothargnull); LLVMBuildCondBr(b, v_bothargisnull, b_bothargnull, b_anyargnull); /* Both NULL? Then is not distinct... */ LLVMPositionBuilderAtEnd(b, b_bothargnull); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); if (opcode == EEOP_NOT_DISTINCT) LLVMBuildStore(b, l_sizet_const(1), v_resvaluep); else LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); /* Only one is NULL? Then is distinct... */ LLVMPositionBuilderAtEnd(b, b_anyargnull); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); if (opcode == EEOP_NOT_DISTINCT) LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); else LLVMBuildStore(b, l_sizet_const(1), v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); /* neither argument is null: compare */ LLVMPositionBuilderAtEnd(b, b_noargnull); v_result = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); if (opcode == EEOP_DISTINCT) { /* Must invert result of "=" */ v_result = LLVMBuildZExt(b, LLVMBuildICmp(b, LLVMIntEQ, v_result, l_sizet_const(0), ""), TypeSizeT, ""); } LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp); LLVMBuildStore(b, v_result, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_NULLIF: { FunctionCallInfo fcinfo = op->d.func.fcinfo_data; LLVMValueRef v_fcinfo; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_argnull0; LLVMValueRef v_argnull1; LLVMValueRef v_anyargisnull; LLVMValueRef v_arg0; LLVMBasicBlockRef b_hasnull; LLVMBasicBlockRef b_nonull; LLVMBasicBlockRef b_argsequal; LLVMValueRef v_retval; LLVMValueRef v_argsequal; b_hasnull = l_bb_before_v(opblocks[opno + 1], "b.%d.null-args", opno); b_nonull = l_bb_before_v(opblocks[opno + 1], "b.%d.no-null-args", opno); b_argsequal = l_bb_before_v(opblocks[opno + 1], "b.%d.argsequal", opno); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); /* save original arg[0] */ v_arg0 = l_funcvalue(b, v_fcinfo, 0); /* if either argument is NULL they can't be equal */ v_argnull0 = l_funcnull(b, v_fcinfo, 0); v_argnull1 = l_funcnull(b, v_fcinfo, 1); v_anyargisnull = LLVMBuildOr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argnull0, l_sbool_const(1), ""), LLVMBuildICmp(b, LLVMIntEQ, v_argnull1, l_sbool_const(1), ""), ""); LLVMBuildCondBr(b, v_anyargisnull, b_hasnull, b_nonull); /* one (or both) of the arguments are null, return arg[0] */ LLVMPositionBuilderAtEnd(b, b_hasnull); LLVMBuildStore(b, v_argnull0, v_resnullp); LLVMBuildStore(b, v_arg0, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); /* build block to invoke function and check result */ LLVMPositionBuilderAtEnd(b, b_nonull); /* * If first argument is of varlena type, it might be an * expanded datum. We need to ensure that the value * passed to the comparison function is a read-only * pointer. However, if we end by returning the first * argument, that will be the original read-write pointer * if it was read-write. */ if (op->d.func.make_ro) { LLVMValueRef v_params[1]; LLVMValueRef v_arg0_ro; v_params[0] = v_arg0; v_arg0_ro = l_call(b, llvm_pg_var_func_type("MakeExpandedObjectReadOnlyInternal"), llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), v_params, lengthof(v_params), ""); LLVMBuildStore(b, v_arg0_ro, l_funcvaluep(b, v_fcinfo, 0)); } v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); /* * If result not null and arguments are equal return null, * else return arg[0] (same result as if there'd been * NULLs, hence reuse b_hasnull). */ v_argsequal = LLVMBuildAnd(b, LLVMBuildICmp(b, LLVMIntEQ, v_fcinfo_isnull, l_sbool_const(0), ""), LLVMBuildICmp(b, LLVMIntEQ, v_retval, l_sizet_const(1), ""), ""); LLVMBuildCondBr(b, v_argsequal, b_argsequal, b_hasnull); /* build block setting result to NULL, if args are equal */ LLVMPositionBuilderAtEnd(b, b_argsequal); LLVMBuildStore(b, l_sbool_const(1), v_resnullp); LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_SQLVALUEFUNCTION: build_EvalXFunc(b, mod, "ExecEvalSQLValueFunction", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_CURRENTOFEXPR: build_EvalXFunc(b, mod, "ExecEvalCurrentOfExpr", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_NEXTVALUEEXPR: build_EvalXFunc(b, mod, "ExecEvalNextValueExpr", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_RETURNINGEXPR: { LLVMBasicBlockRef b_isnull; LLVMValueRef v_flagsp; LLVMValueRef v_flags; LLVMValueRef v_nullflag; b_isnull = l_bb_before_v(opblocks[opno + 1], "op.%d.row.isnull", opno); /* * The next op actually evaluates the expression. If the * OLD/NEW row doesn't exist, skip that and return NULL. */ v_flagsp = l_struct_gep(b, StructExprState, v_state, FIELDNO_EXPRSTATE_FLAGS, "v.state.flags"); v_flags = l_load(b, TypeStorageBool, v_flagsp, ""); v_nullflag = l_int8_const(lc, op->d.returningexpr.nullflag); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, LLVMBuildAnd(b, v_flags, v_nullflag, ""), l_sbool_const(0), ""), opblocks[opno + 1], b_isnull); LLVMPositionBuilderAtEnd(b, b_isnull); LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildStore(b, l_sbool_const(1), v_resnullp); LLVMBuildBr(b, opblocks[op->d.returningexpr.jumpdone]); break; } case EEOP_ARRAYEXPR: build_EvalXFunc(b, mod, "ExecEvalArrayExpr", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_ARRAYCOERCE: build_EvalXFunc(b, mod, "ExecEvalArrayCoerce", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_ROW: build_EvalXFunc(b, mod, "ExecEvalRow", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_ROWCOMPARE_STEP: { FunctionCallInfo fcinfo = op->d.rowcompare_step.fcinfo_data; LLVMValueRef v_fcinfo_isnull; LLVMBasicBlockRef b_null; LLVMBasicBlockRef b_compare; LLVMBasicBlockRef b_compare_result; LLVMValueRef v_retval; b_null = l_bb_before_v(opblocks[opno + 1], "op.%d.row-null", opno); b_compare = l_bb_before_v(opblocks[opno + 1], "op.%d.row-compare", opno); b_compare_result = l_bb_before_v(opblocks[opno + 1], "op.%d.row-compare-result", opno); /* * If function is strict, and either arg is null, we're * done. */ if (op->d.rowcompare_step.finfo->fn_strict) { LLVMValueRef v_fcinfo; LLVMValueRef v_argnull0; LLVMValueRef v_argnull1; LLVMValueRef v_anyargisnull; v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); v_argnull0 = l_funcnull(b, v_fcinfo, 0); v_argnull1 = l_funcnull(b, v_fcinfo, 1); v_anyargisnull = LLVMBuildOr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argnull0, l_sbool_const(1), ""), LLVMBuildICmp(b, LLVMIntEQ, v_argnull1, l_sbool_const(1), ""), ""); LLVMBuildCondBr(b, v_anyargisnull, b_null, b_compare); } else { LLVMBuildBr(b, b_compare); } /* build block invoking comparison function */ LLVMPositionBuilderAtEnd(b, b_compare); /* call function */ v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); LLVMBuildStore(b, v_retval, v_resvaluep); /* if result of function is NULL, force NULL result */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_fcinfo_isnull, l_sbool_const(0), ""), b_compare_result, b_null); /* build block analyzing the !NULL comparator result */ LLVMPositionBuilderAtEnd(b, b_compare_result); /* if results equal, compare next, otherwise done */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_retval, l_sizet_const(0), ""), opblocks[opno + 1], opblocks[op->d.rowcompare_step.jumpdone]); /* * Build block handling NULL input or NULL comparator * result. */ LLVMPositionBuilderAtEnd(b, b_null); LLVMBuildStore(b, l_sbool_const(1), v_resnullp); LLVMBuildBr(b, opblocks[op->d.rowcompare_step.jumpnull]); break; } case EEOP_ROWCOMPARE_FINAL: { CompareType cmptype = op->d.rowcompare_final.cmptype; LLVMValueRef v_cmpresult; LLVMValueRef v_result; LLVMIntPredicate predicate; /* * Btree comparators return 32 bit results, need to be * careful about sign (used as a 64 bit value it's * otherwise wrong). */ v_cmpresult = LLVMBuildTrunc(b, l_load(b, TypeSizeT, v_resvaluep, ""), LLVMInt32TypeInContext(lc), ""); switch (cmptype) { case COMPARE_LT: predicate = LLVMIntSLT; break; case COMPARE_LE: predicate = LLVMIntSLE; break; case COMPARE_GT: predicate = LLVMIntSGT; break; case COMPARE_GE: predicate = LLVMIntSGE; break; default: /* EQ and NE cases aren't allowed here */ Assert(false); predicate = 0; /* prevent compiler warning */ break; } v_result = LLVMBuildICmp(b, predicate, v_cmpresult, l_int32_const(lc, 0), ""); v_result = LLVMBuildZExt(b, v_result, TypeSizeT, ""); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); LLVMBuildStore(b, v_result, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_MINMAX: build_EvalXFunc(b, mod, "ExecEvalMinMax", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_FIELDSELECT: build_EvalXFunc(b, mod, "ExecEvalFieldSelect", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_FIELDSTORE_DEFORM: build_EvalXFunc(b, mod, "ExecEvalFieldStoreDeForm", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_FIELDSTORE_FORM: build_EvalXFunc(b, mod, "ExecEvalFieldStoreForm", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_DOMAIN_TESTVAL: { LLVMValueRef v_casevaluep, v_casevalue; LLVMValueRef v_casenullp, v_casenull; v_casevaluep = l_ptr_const(op->d.casetest.value, l_ptr(TypeSizeT)); v_casenullp = l_ptr_const(op->d.casetest.isnull, l_ptr(TypeStorageBool)); v_casevalue = l_load(b, TypeSizeT, v_casevaluep, ""); v_casenull = l_load(b, TypeStorageBool, v_casenullp, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_DOMAIN_TESTVAL_EXT: { LLVMValueRef v_casevalue; LLVMValueRef v_casenull; v_casevalue = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_DOMAINDATUM, ""); v_casenull = l_load_struct_gep(b, StructExprContext, v_econtext, FIELDNO_EXPRCONTEXT_DOMAINNULL, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_DOMAIN_NOTNULL: build_EvalXFunc(b, mod, "ExecEvalConstraintNotNull", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_DOMAIN_CHECK: build_EvalXFunc(b, mod, "ExecEvalConstraintCheck", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_HASHDATUM_SET_INITVAL: { LLVMValueRef v_initvalue; v_initvalue = l_sizet_const(op->d.hashdatum_initvalue.init_value); LLVMBuildStore(b, v_initvalue, v_resvaluep); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_HASHDATUM_FIRST: case EEOP_HASHDATUM_FIRST_STRICT: case EEOP_HASHDATUM_NEXT32: case EEOP_HASHDATUM_NEXT32_STRICT: { FunctionCallInfo fcinfo = op->d.hashdatum.fcinfo_data; LLVMValueRef v_fcinfo; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_retval; LLVMBasicBlockRef b_checkargnull; LLVMBasicBlockRef b_ifnotnull; LLVMBasicBlockRef b_ifnullblock; LLVMValueRef v_argisnull; LLVMValueRef v_prevhash = NULL; /* * When performing the next hash and not in strict mode we * perform a rotation of the previously stored hash value * before doing the NULL check. We want to do this even * when we receive a NULL Datum to hash. In strict mode, * we do this after the NULL check so as not to waste the * effort of rotating the bits when we're going to throw * away the hash value and return NULL. */ if (opcode == EEOP_HASHDATUM_NEXT32) { LLVMValueRef v_tmp1; LLVMValueRef v_tmp2; LLVMValueRef tmp; tmp = l_ptr_const(&op->d.hashdatum.iresult->value, l_ptr(TypeSizeT)); /* * Fetch the previously hashed value from where the * previous hash operation stored it. */ v_prevhash = l_load(b, TypeSizeT, tmp, "prevhash"); /* * Rotate bits left by 1 bit. Be careful not to * overflow uint32 when working with size_t. */ v_tmp1 = LLVMBuildShl(b, v_prevhash, l_sizet_const(1), ""); v_tmp1 = LLVMBuildAnd(b, v_tmp1, l_sizet_const(0xffffffff), ""); v_tmp2 = LLVMBuildLShr(b, v_prevhash, l_sizet_const(31), ""); v_prevhash = LLVMBuildOr(b, v_tmp1, v_tmp2, "rotatedhash"); } /* * Block for the actual function call, if args are * non-NULL. */ b_ifnotnull = l_bb_before_v(opblocks[opno + 1], "b.%d.ifnotnull", opno); /* we expect the hash function to have 1 argument */ if (fcinfo->nargs != 1) elog(ERROR, "incorrect number of function arguments"); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); b_checkargnull = l_bb_before_v(b_ifnotnull, "b.%d.isnull.0", opno); LLVMBuildBr(b, b_checkargnull); /* * Determine what to do if we find the argument to be * NULL. */ if (opcode == EEOP_HASHDATUM_FIRST_STRICT || opcode == EEOP_HASHDATUM_NEXT32_STRICT) { b_ifnullblock = l_bb_before_v(b_ifnotnull, "b.%d.strictnull", opno); LLVMPositionBuilderAtEnd(b, b_ifnullblock); /* * In strict node, NULL inputs result in NULL. Save * the NULL result and goto jumpdone. */ LLVMBuildStore(b, l_sbool_const(1), v_resnullp); LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); LLVMBuildBr(b, opblocks[op->d.hashdatum.jumpdone]); } else { b_ifnullblock = l_bb_before_v(b_ifnotnull, "b.%d.null", opno); LLVMPositionBuilderAtEnd(b, b_ifnullblock); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); if (opcode == EEOP_HASHDATUM_NEXT32) { Assert(v_prevhash != NULL); /* * Save the rotated hash value and skip to the * next op. */ LLVMBuildStore(b, v_prevhash, v_resvaluep); } else { Assert(opcode == EEOP_HASHDATUM_FIRST); /* * Store a zero Datum when the Datum to hash is * NULL */ LLVMBuildStore(b, l_sizet_const(0), v_resvaluep); } LLVMBuildBr(b, opblocks[opno + 1]); } LLVMPositionBuilderAtEnd(b, b_checkargnull); /* emit code to check if the input parameter is NULL */ v_argisnull = l_funcnull(b, v_fcinfo, 0); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argisnull, l_sbool_const(1), ""), b_ifnullblock, b_ifnotnull); LLVMPositionBuilderAtEnd(b, b_ifnotnull); /* * Rotate the previously stored hash value when performing * NEXT32 in strict mode. In non-strict mode we already * did this before checking for NULLs. */ if (opcode == EEOP_HASHDATUM_NEXT32_STRICT) { LLVMValueRef v_tmp1; LLVMValueRef v_tmp2; LLVMValueRef tmp; tmp = l_ptr_const(&op->d.hashdatum.iresult->value, l_ptr(TypeSizeT)); /* * Fetch the previously hashed value from where the * previous hash operation stored it. */ v_prevhash = l_load(b, TypeSizeT, tmp, "prevhash"); /* * Rotate bits left by 1 bit. Be careful not to * overflow uint32 when working with size_t. */ v_tmp1 = LLVMBuildShl(b, v_prevhash, l_sizet_const(1), ""); v_tmp1 = LLVMBuildAnd(b, v_tmp1, l_sizet_const(0xffffffff), ""); v_tmp2 = LLVMBuildLShr(b, v_prevhash, l_sizet_const(31), ""); v_prevhash = LLVMBuildOr(b, v_tmp1, v_tmp2, "rotatedhash"); } /* call the hash function */ v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); /* * For NEXT32 ops, XOR (^) the returned hash value with * the existing hash value. */ if (opcode == EEOP_HASHDATUM_NEXT32 || opcode == EEOP_HASHDATUM_NEXT32_STRICT) v_retval = LLVMBuildXor(b, v_prevhash, v_retval, "xorhash"); LLVMBuildStore(b, v_retval, v_resvaluep); LLVMBuildStore(b, l_sbool_const(0), v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_CONVERT_ROWTYPE: build_EvalXFunc(b, mod, "ExecEvalConvertRowtype", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_SCALARARRAYOP: build_EvalXFunc(b, mod, "ExecEvalScalarArrayOp", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_HASHED_SCALARARRAYOP: build_EvalXFunc(b, mod, "ExecEvalHashedScalarArrayOp", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_XMLEXPR: build_EvalXFunc(b, mod, "ExecEvalXmlExpr", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_JSON_CONSTRUCTOR: build_EvalXFunc(b, mod, "ExecEvalJsonConstructor", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_IS_JSON: build_EvalXFunc(b, mod, "ExecEvalJsonIsPredicate", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_JSONEXPR_PATH: { JsonExprState *jsestate = op->d.jsonexpr.jsestate; LLVMValueRef v_ret; /* * Call ExecEvalJsonExprPath(). It returns the address of * the step to perform next. */ v_ret = build_EvalXFunc(b, mod, "ExecEvalJsonExprPath", v_state, op, v_econtext); /* * Build a switch to map the return value (v_ret above), * which is a runtime value of the step address to perform * next, to either jump_empty, jump_error, * jump_eval_coercion, or jump_end. */ if (jsestate->jump_empty >= 0 || jsestate->jump_error >= 0 || jsestate->jump_eval_coercion >= 0) { LLVMValueRef v_jump_empty; LLVMValueRef v_jump_error; LLVMValueRef v_jump_coercion; LLVMValueRef v_switch; LLVMBasicBlockRef b_done, b_empty, b_error, b_coercion; b_empty = l_bb_before_v(opblocks[opno + 1], "op.%d.jsonexpr_empty", opno); b_error = l_bb_before_v(opblocks[opno + 1], "op.%d.jsonexpr_error", opno); b_coercion = l_bb_before_v(opblocks[opno + 1], "op.%d.jsonexpr_coercion", opno); b_done = l_bb_before_v(opblocks[opno + 1], "op.%d.jsonexpr_done", opno); v_switch = LLVMBuildSwitch(b, v_ret, b_done, 3); /* Returned jsestate->jump_empty? */ if (jsestate->jump_empty >= 0) { v_jump_empty = l_int32_const(lc, jsestate->jump_empty); LLVMAddCase(v_switch, v_jump_empty, b_empty); } /* ON EMPTY code */ LLVMPositionBuilderAtEnd(b, b_empty); if (jsestate->jump_empty >= 0) LLVMBuildBr(b, opblocks[jsestate->jump_empty]); else LLVMBuildUnreachable(b); /* Returned jsestate->jump_error? */ if (jsestate->jump_error >= 0) { v_jump_error = l_int32_const(lc, jsestate->jump_error); LLVMAddCase(v_switch, v_jump_error, b_error); } /* ON ERROR code */ LLVMPositionBuilderAtEnd(b, b_error); if (jsestate->jump_error >= 0) LLVMBuildBr(b, opblocks[jsestate->jump_error]); else LLVMBuildUnreachable(b); /* Returned jsestate->jump_eval_coercion? */ if (jsestate->jump_eval_coercion >= 0) { v_jump_coercion = l_int32_const(lc, jsestate->jump_eval_coercion); LLVMAddCase(v_switch, v_jump_coercion, b_coercion); } /* jump_eval_coercion code */ LLVMPositionBuilderAtEnd(b, b_coercion); if (jsestate->jump_eval_coercion >= 0) LLVMBuildBr(b, opblocks[jsestate->jump_eval_coercion]); else LLVMBuildUnreachable(b); LLVMPositionBuilderAtEnd(b, b_done); } LLVMBuildBr(b, opblocks[jsestate->jump_end]); break; } case EEOP_JSONEXPR_COERCION: build_EvalXFunc(b, mod, "ExecEvalJsonCoercion", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_JSONEXPR_COERCION_FINISH: build_EvalXFunc(b, mod, "ExecEvalJsonCoercionFinish", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_AGGREF: { LLVMValueRef v_aggno; LLVMValueRef value, isnull; v_aggno = l_int32_const(lc, op->d.aggref.aggno); /* load agg value / null */ value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_aggno, "aggvalue"); isnull = l_load_gep1(b, TypeStorageBool, v_aggnulls, v_aggno, "aggnull"); /* and store result */ LLVMBuildStore(b, value, v_resvaluep); LLVMBuildStore(b, isnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_GROUPING_FUNC: build_EvalXFunc(b, mod, "ExecEvalGroupingFunc", v_state, op); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_WINDOW_FUNC: { WindowFuncExprState *wfunc = op->d.window_func.wfstate; LLVMValueRef v_wfuncnop; LLVMValueRef v_wfuncno; LLVMValueRef value, isnull; /* * At this point aggref->wfuncno is not yet set (it's set * up in ExecInitWindowAgg() after initializing the * expression). So load it from memory each time round. */ v_wfuncnop = l_ptr_const(&wfunc->wfuncno, l_ptr(LLVMInt32TypeInContext(lc))); v_wfuncno = l_load(b, LLVMInt32TypeInContext(lc), v_wfuncnop, "v_wfuncno"); /* load window func value / null */ value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_wfuncno, "windowvalue"); isnull = l_load_gep1(b, TypeStorageBool, v_aggnulls, v_wfuncno, "windownull"); LLVMBuildStore(b, value, v_resvaluep); LLVMBuildStore(b, isnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_MERGE_SUPPORT_FUNC: build_EvalXFunc(b, mod, "ExecEvalMergeSupportFunc", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_SUBPLAN: build_EvalXFunc(b, mod, "ExecEvalSubPlan", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_AGG_STRICT_DESERIALIZE: case EEOP_AGG_DESERIALIZE: { AggState *aggstate; FunctionCallInfo fcinfo = op->d.agg_deserialize.fcinfo_data; LLVMValueRef v_retval; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_tmpcontext; LLVMValueRef v_oldcontext; if (opcode == EEOP_AGG_STRICT_DESERIALIZE) { LLVMValueRef v_fcinfo; LLVMValueRef v_argnull0; LLVMBasicBlockRef b_deserialize; b_deserialize = l_bb_before_v(opblocks[opno + 1], "op.%d.deserialize", opno); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); v_argnull0 = l_funcnull(b, v_fcinfo, 0); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argnull0, l_sbool_const(1), ""), opblocks[op->d.agg_deserialize.jumpnull], b_deserialize); LLVMPositionBuilderAtEnd(b, b_deserialize); } aggstate = castNode(AggState, state->parent); fcinfo = op->d.agg_deserialize.fcinfo_data; v_tmpcontext = l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory, l_ptr(StructMemoryContextData)); v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext); v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); l_mcxt_switch(mod, b, v_oldcontext); LLVMBuildStore(b, v_retval, v_resvaluep); LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_AGG_STRICT_INPUT_CHECK_ARGS: case EEOP_AGG_STRICT_INPUT_CHECK_ARGS_1: case EEOP_AGG_STRICT_INPUT_CHECK_NULLS: { int nargs = op->d.agg_strict_input_check.nargs; NullableDatum *args = op->d.agg_strict_input_check.args; bool *nulls = op->d.agg_strict_input_check.nulls; int jumpnull; LLVMValueRef v_argsp; LLVMValueRef v_nullsp; LLVMBasicBlockRef *b_checknulls; Assert(nargs > 0); jumpnull = op->d.agg_strict_input_check.jumpnull; v_argsp = l_ptr_const(args, l_ptr(StructNullableDatum)); v_nullsp = l_ptr_const(nulls, l_ptr(TypeStorageBool)); /* create blocks for checking args */ b_checknulls = palloc(sizeof(LLVMBasicBlockRef *) * nargs); for (int argno = 0; argno < nargs; argno++) { b_checknulls[argno] = l_bb_before_v(opblocks[opno + 1], "op.%d.check-null.%d", opno, argno); } LLVMBuildBr(b, b_checknulls[0]); /* strict function, check for NULL args */ for (int argno = 0; argno < nargs; argno++) { LLVMValueRef v_argno = l_int32_const(lc, argno); LLVMValueRef v_argisnull; LLVMBasicBlockRef b_argnotnull; LLVMPositionBuilderAtEnd(b, b_checknulls[argno]); if (argno + 1 == nargs) b_argnotnull = opblocks[opno + 1]; else b_argnotnull = b_checknulls[argno + 1]; if (opcode == EEOP_AGG_STRICT_INPUT_CHECK_NULLS) v_argisnull = l_load_gep1(b, TypeStorageBool, v_nullsp, v_argno, ""); else { LLVMValueRef v_argn; v_argn = l_gep(b, StructNullableDatum, v_argsp, &v_argno, 1, ""); v_argisnull = l_load_struct_gep(b, StructNullableDatum, v_argn, FIELDNO_NULLABLE_DATUM_ISNULL, ""); } LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_argisnull, l_sbool_const(1), ""), opblocks[jumpnull], b_argnotnull); } break; } case EEOP_AGG_PLAIN_PERGROUP_NULLCHECK: { int jumpnull; LLVMValueRef v_aggstatep; LLVMValueRef v_allpergroupsp; LLVMValueRef v_pergroup_allaggs; LLVMValueRef v_setoff; jumpnull = op->d.agg_plain_pergroup_nullcheck.jumpnull; /* * pergroup_allaggs = aggstate->all_pergroups * [op->d.agg_plain_pergroup_nullcheck.setoff]; */ v_aggstatep = LLVMBuildBitCast(b, v_parent, l_ptr(StructAggState), ""); v_allpergroupsp = l_load_struct_gep(b, StructAggState, v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); v_setoff = l_int32_const(lc, op->d.agg_plain_pergroup_nullcheck.setoff); v_pergroup_allaggs = l_load_gep1(b, l_ptr(StructAggStatePerGroupData), v_allpergroupsp, v_setoff, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, LLVMBuildPtrToInt(b, v_pergroup_allaggs, TypeSizeT, ""), l_sizet_const(0), ""), opblocks[jumpnull], opblocks[opno + 1]); break; } case EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYVAL: case EEOP_AGG_PLAIN_TRANS_STRICT_BYVAL: case EEOP_AGG_PLAIN_TRANS_BYVAL: case EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYREF: case EEOP_AGG_PLAIN_TRANS_STRICT_BYREF: case EEOP_AGG_PLAIN_TRANS_BYREF: { AggState *aggstate; AggStatePerTrans pertrans; FunctionCallInfo fcinfo; LLVMValueRef v_aggstatep; LLVMValueRef v_fcinfo; LLVMValueRef v_fcinfo_isnull; LLVMValueRef v_transvaluep; LLVMValueRef v_transnullp; LLVMValueRef v_setoff; LLVMValueRef v_transno; LLVMValueRef v_aggcontext; LLVMValueRef v_allpergroupsp; LLVMValueRef v_current_setp; LLVMValueRef v_current_pertransp; LLVMValueRef v_curaggcontext; LLVMValueRef v_pertransp; LLVMValueRef v_pergroupp; LLVMValueRef v_retval; LLVMValueRef v_tmpcontext; LLVMValueRef v_oldcontext; aggstate = castNode(AggState, state->parent); pertrans = op->d.agg_trans.pertrans; fcinfo = pertrans->transfn_fcinfo; v_aggstatep = LLVMBuildBitCast(b, v_parent, l_ptr(StructAggState), ""); v_pertransp = l_ptr_const(pertrans, l_ptr(StructAggStatePerTransData)); /* * pergroup = &aggstate->all_pergroups * [op->d.agg_trans.setoff] [op->d.agg_trans.transno]; */ v_allpergroupsp = l_load_struct_gep(b, StructAggState, v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); v_setoff = l_int32_const(lc, op->d.agg_trans.setoff); v_transno = l_int32_const(lc, op->d.agg_trans.transno); v_pergroupp = l_gep(b, StructAggStatePerGroupData, l_load_gep1(b, l_ptr(StructAggStatePerGroupData), v_allpergroupsp, v_setoff, ""), &v_transno, 1, ""); if (opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYVAL || opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYREF) { LLVMValueRef v_notransvalue; LLVMBasicBlockRef b_init; LLVMBasicBlockRef b_no_init; v_notransvalue = l_load_struct_gep(b, StructAggStatePerGroupData, v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_NOTRANSVALUE, "notransvalue"); b_init = l_bb_before_v(opblocks[opno + 1], "op.%d.inittrans", opno); b_no_init = l_bb_before_v(opblocks[opno + 1], "op.%d.no_inittrans", opno); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_notransvalue, l_sbool_const(1), ""), b_init, b_no_init); /* block to init the transition value if necessary */ { LLVMValueRef params[4]; LLVMPositionBuilderAtEnd(b, b_init); v_aggcontext = l_ptr_const(op->d.agg_trans.aggcontext, l_ptr(StructExprContext)); params[0] = v_aggstatep; params[1] = v_pertransp; params[2] = v_pergroupp; params[3] = v_aggcontext; l_call(b, llvm_pg_var_func_type("ExecAggInitGroup"), llvm_pg_func(mod, "ExecAggInitGroup"), params, lengthof(params), ""); LLVMBuildBr(b, opblocks[opno + 1]); } LLVMPositionBuilderAtEnd(b, b_no_init); } if (opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYVAL || opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYREF || opcode == EEOP_AGG_PLAIN_TRANS_STRICT_BYVAL || opcode == EEOP_AGG_PLAIN_TRANS_STRICT_BYREF) { LLVMValueRef v_transnull; LLVMBasicBlockRef b_strictpass; b_strictpass = l_bb_before_v(opblocks[opno + 1], "op.%d.strictpass", opno); v_transnull = l_load_struct_gep(b, StructAggStatePerGroupData, v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL, "transnull"); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_transnull, l_sbool_const(1), ""), opblocks[opno + 1], b_strictpass); LLVMPositionBuilderAtEnd(b, b_strictpass); } v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); v_aggcontext = l_ptr_const(op->d.agg_trans.aggcontext, l_ptr(StructExprContext)); v_current_setp = l_struct_gep(b, StructAggState, v_aggstatep, FIELDNO_AGGSTATE_CURRENT_SET, "aggstate.current_set"); v_curaggcontext = l_struct_gep(b, StructAggState, v_aggstatep, FIELDNO_AGGSTATE_CURAGGCONTEXT, "aggstate.curaggcontext"); v_current_pertransp = l_struct_gep(b, StructAggState, v_aggstatep, FIELDNO_AGGSTATE_CURPERTRANS, "aggstate.curpertrans"); /* set aggstate globals */ LLVMBuildStore(b, v_aggcontext, v_curaggcontext); LLVMBuildStore(b, l_int32_const(lc, op->d.agg_trans.setno), v_current_setp); LLVMBuildStore(b, v_pertransp, v_current_pertransp); /* invoke transition function in per-tuple context */ v_tmpcontext = l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory, l_ptr(StructMemoryContextData)); v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext); /* store transvalue in fcinfo->args[0] */ v_transvaluep = l_struct_gep(b, StructAggStatePerGroupData, v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE, "transvalue"); v_transnullp = l_struct_gep(b, StructAggStatePerGroupData, v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL, "transnullp"); LLVMBuildStore(b, l_load(b, TypeSizeT, v_transvaluep, "transvalue"), l_funcvaluep(b, v_fcinfo, 0)); LLVMBuildStore(b, l_load(b, TypeStorageBool, v_transnullp, "transnull"), l_funcnullp(b, v_fcinfo, 0)); /* and invoke transition function */ v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull); /* * For pass-by-ref datatype, must copy the new value into * aggcontext and free the prior transValue. But if * transfn returned a pointer to its first input, we don't * need to do anything. Also, if transfn returned a * pointer to a R/W expanded object that is already a * child of the aggcontext, assume we can adopt that value * without copying it. */ if (opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYREF || opcode == EEOP_AGG_PLAIN_TRANS_STRICT_BYREF || opcode == EEOP_AGG_PLAIN_TRANS_BYREF) { LLVMBasicBlockRef b_call; LLVMBasicBlockRef b_nocall; LLVMValueRef v_fn; LLVMValueRef v_transvalue; LLVMValueRef v_transnull; LLVMValueRef v_newval; LLVMValueRef params[6]; b_call = l_bb_before_v(opblocks[opno + 1], "op.%d.transcall", opno); b_nocall = l_bb_before_v(opblocks[opno + 1], "op.%d.transnocall", opno); v_transvalue = l_load(b, TypeSizeT, v_transvaluep, ""); v_transnull = l_load(b, TypeStorageBool, v_transnullp, ""); /* * DatumGetPointer(newVal) != * DatumGetPointer(pergroup->transValue)) */ LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_transvalue, v_retval, ""), b_nocall, b_call); /* returned datum not passed datum, reparent */ LLVMPositionBuilderAtEnd(b, b_call); params[0] = v_aggstatep; params[1] = v_pertransp; params[2] = v_retval; params[3] = LLVMBuildTrunc(b, v_fcinfo_isnull, TypeParamBool, ""); params[4] = v_transvalue; params[5] = LLVMBuildTrunc(b, v_transnull, TypeParamBool, ""); v_fn = llvm_pg_func(mod, "ExecAggCopyTransValue"); v_newval = l_call(b, LLVMGetFunctionType(v_fn), v_fn, params, lengthof(params), ""); /* store trans value */ LLVMBuildStore(b, v_newval, v_transvaluep); LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp); l_mcxt_switch(mod, b, v_oldcontext); LLVMBuildBr(b, opblocks[opno + 1]); /* returned datum passed datum, no need to reparent */ LLVMPositionBuilderAtEnd(b, b_nocall); } /* store trans value */ LLVMBuildStore(b, v_retval, v_transvaluep); LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp); l_mcxt_switch(mod, b, v_oldcontext); LLVMBuildBr(b, opblocks[opno + 1]); break; } case EEOP_AGG_PRESORTED_DISTINCT_SINGLE: { AggState *aggstate = castNode(AggState, state->parent); AggStatePerTrans pertrans = op->d.agg_presorted_distinctcheck.pertrans; int jumpdistinct = op->d.agg_presorted_distinctcheck.jumpdistinct; LLVMValueRef v_fn = llvm_pg_func(mod, "ExecEvalPreOrderedDistinctSingle"); LLVMValueRef v_args[2]; LLVMValueRef v_ret; v_args[0] = l_ptr_const(aggstate, l_ptr(StructAggState)); v_args[1] = l_ptr_const(pertrans, l_ptr(StructAggStatePerTransData)); v_ret = l_call(b, LLVMGetFunctionType(v_fn), v_fn, v_args, 2, ""); v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_ret, l_sbool_const(1), ""), opblocks[opno + 1], opblocks[jumpdistinct]); break; } case EEOP_AGG_PRESORTED_DISTINCT_MULTI: { AggState *aggstate = castNode(AggState, state->parent); AggStatePerTrans pertrans = op->d.agg_presorted_distinctcheck.pertrans; int jumpdistinct = op->d.agg_presorted_distinctcheck.jumpdistinct; LLVMValueRef v_fn = llvm_pg_func(mod, "ExecEvalPreOrderedDistinctMulti"); LLVMValueRef v_args[2]; LLVMValueRef v_ret; v_args[0] = l_ptr_const(aggstate, l_ptr(StructAggState)); v_args[1] = l_ptr_const(pertrans, l_ptr(StructAggStatePerTransData)); v_ret = l_call(b, LLVMGetFunctionType(v_fn), v_fn, v_args, 2, ""); v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_ret, l_sbool_const(1), ""), opblocks[opno + 1], opblocks[jumpdistinct]); break; } case EEOP_AGG_ORDERED_TRANS_DATUM: build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransDatum", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_AGG_ORDERED_TRANS_TUPLE: build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransTuple", v_state, op, v_econtext); LLVMBuildBr(b, opblocks[opno + 1]); break; case EEOP_LAST: Assert(false); break; } } LLVMDisposeBuilder(b); /* * Don't immediately emit function, instead do so the first time the * expression is actually evaluated. That allows to emit a lot of * functions together, avoiding a lot of repeated llvm and memory * remapping overhead. */ { CompiledExprState *cstate = palloc0(sizeof(CompiledExprState)); cstate->context = context; cstate->funcname = funcname; state->evalfunc = ExecRunCompiledExpr; state->evalfunc_private = cstate; } llvm_leave_fatal_on_oom(); INSTR_TIME_SET_CURRENT(endtime); INSTR_TIME_ACCUM_DIFF(context->base.instr.generation_counter, endtime, starttime); return true; } /* * Run compiled expression. * * This will only be called the first time a JITed expression is called. We * first make sure the expression is still up-to-date, and then get a pointer to * the emitted function. The latter can be the first thing that triggers * optimizing and emitting all the generated functions. */ static Datum ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull) { CompiledExprState *cstate = state->evalfunc_private; ExprStateEvalFunc func; CheckExprStillValid(state, econtext); llvm_enter_fatal_on_oom(); func = (ExprStateEvalFunc) llvm_get_function(cstate->context, cstate->funcname); llvm_leave_fatal_on_oom(); Assert(func); /* remove indirection via this function for future calls */ state->evalfunc = func; return func(state, econtext, isNull); } static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, LLVMModuleRef mod, FunctionCallInfo fcinfo, LLVMValueRef *v_fcinfo_isnull) { LLVMContextRef lc; LLVMValueRef v_fn; LLVMValueRef v_fcinfo_isnullp; LLVMValueRef v_retval; LLVMValueRef v_fcinfo; lc = LLVMGetModuleContext(mod); v_fn = llvm_function_reference(context, b, mod, fcinfo); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); v_fcinfo_isnullp = l_struct_gep(b, StructFunctionCallInfoData, v_fcinfo, FIELDNO_FUNCTIONCALLINFODATA_ISNULL, "v_fcinfo_isnull"); LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_isnullp); v_retval = l_call(b, LLVMGetFunctionType(AttributeTemplate), v_fn, &v_fcinfo, 1, "funccall"); if (v_fcinfo_isnull) *v_fcinfo_isnull = l_load(b, TypeStorageBool, v_fcinfo_isnullp, ""); /* * Add lifetime-end annotation, signaling that writes to memory don't have * to be retained (important for inlining potential). */ { LLVMValueRef v_lifetime = create_LifetimeEnd(mod); LLVMValueRef params[2]; params[0] = l_int64_const(lc, sizeof(NullableDatum) * fcinfo->nargs); params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8TypeInContext(lc))); l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); params[0] = l_int64_const(lc, sizeof(fcinfo->isnull)); params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8TypeInContext(lc))); l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); } return v_retval; } /* * Implement an expression step by calling the function funcname. */ static LLVMValueRef build_EvalXFuncInt(LLVMBuilderRef b, LLVMModuleRef mod, const char *funcname, LLVMValueRef v_state, ExprEvalStep *op, int nargs, LLVMValueRef *v_args) { LLVMValueRef v_fn = llvm_pg_func(mod, funcname); LLVMValueRef *params; int argno = 0; LLVMValueRef v_ret; /* cheap pre-check as llvm just asserts out */ if (LLVMCountParams(v_fn) != (nargs + 2)) elog(ERROR, "parameter mismatch: %s expects %d passed %d", funcname, LLVMCountParams(v_fn), nargs + 2); params = palloc(sizeof(LLVMValueRef) * (2 + nargs)); params[argno++] = v_state; params[argno++] = l_ptr_const(op, l_ptr(StructExprEvalStep)); for (int i = 0; i < nargs; i++) params[argno++] = v_args[i]; v_ret = l_call(b, LLVMGetFunctionType(v_fn), v_fn, params, argno, ""); pfree(params); return v_ret; } static LLVMValueRef create_LifetimeEnd(LLVMModuleRef mod) { LLVMTypeRef sig; LLVMValueRef fn; LLVMTypeRef param_types[2]; LLVMContextRef lc; /* variadic pointer argument */ const char *nm = "llvm.lifetime.end.p0"; fn = LLVMGetNamedFunction(mod, nm); if (fn) return fn; lc = LLVMGetModuleContext(mod); param_types[0] = LLVMInt64TypeInContext(lc); param_types[1] = l_ptr(LLVMInt8TypeInContext(lc)); sig = LLVMFunctionType(LLVMVoidTypeInContext(lc), param_types, lengthof(param_types), false); fn = LLVMAddFunction(mod, nm, sig); LLVMSetFunctionCallConv(fn, LLVMCCallConv); Assert(LLVMGetIntrinsicID(fn)); return fn; }