Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ext/opcache/jit/ir/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,6 @@ tests/**/*.log

win32/vcpkg
win32/build_*

fuzz/build/
fuzz/corpus/
66 changes: 61 additions & 5 deletions ext/opcache/jit/ir/ir.c
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted
case IR_CHAR:
if (insn->val.c == '\\') {
fprintf(f, "'\\\\'");
} else if (insn->val.c == '\'') {
fprintf(f, "'\\\''");
} else if (insn->val.c >= ' ') {
fprintf(f, "'%c'", insn->val.c);
} else if (insn->val.c == '\t') {
Expand Down Expand Up @@ -283,6 +285,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted
#define ir_op_kind_src IR_OPND_CONTROL
#define ir_op_kind_reg IR_OPND_CONTROL_DEP
#define ir_op_kind_ret IR_OPND_CONTROL_REF
#define ir_op_kind_grd IR_OPND_CONTROL_GUARD
#define ir_op_kind_str IR_OPND_STR
#define ir_op_kind_num IR_OPND_NUM
#define ir_op_kind_fld IR_OPND_STR
Expand Down Expand Up @@ -1843,7 +1846,7 @@ int ir_mem_unprotect(void *ptr, size_t size)

int ir_mem_flush(void *ptr, size_t size)
{
return 1;
return FlushInstructionCache(GetCurrentProcess(), ptr, size) == TRUE ? 1 : 0;
}
#else

Expand Down Expand Up @@ -2168,7 +2171,10 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(const ir_ctx *ctx, ir_ref ref, i
if (!(proto->flags & (IR_CONST_FUNC|IR_PURE_FUNC))) {
break;
}
} else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_VSTORE) {
} else if (insn->op == IR_MERGE
|| insn->op == IR_LOOP_BEGIN
|| insn->op == IR_VSTORE
|| (insn->op == IR_BEGIN && insn->op2)) {
return IR_UNUSED;
}
ref = insn->op1;
Expand Down Expand Up @@ -2233,7 +2239,10 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(const ir_ctx *ctx, ir_ref ref,
if (!(proto->flags & (IR_CONST_FUNC|IR_PURE_FUNC))) {
break;
}
} else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_STORE) {
} else if (insn->op == IR_MERGE
|| insn->op == IR_LOOP_BEGIN
|| insn->op == IR_STORE
|| (insn->op == IR_BEGIN && insn->op2)) {
break;
}
ref = insn->op1;
Expand Down Expand Up @@ -2326,7 +2335,15 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_store_i(ir_ctx *ctx, ir_ref ref, ir_ref
}
} else if (insn->op == IR_GUARD || insn->op == IR_GUARD_NOT) {
guarded = 1;
} else if (insn->op >= IR_START || insn->op == IR_CALL) {
} else if (insn->op >= IR_START) {
if (insn->op == IR_BEGIN && insn->op1 && !insn->op2) {
/* skip END */
ref = insn->op1;
insn = &ctx->ir_base[ref];
} else {
break;
}
} else if (insn->op == IR_CALL) {
break;
}
next = ref;
Expand Down Expand Up @@ -2407,7 +2424,15 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vstore_i(ir_ctx *ctx, ir_ref ref, ir_re
}
} else if (insn->op == IR_GUARD || insn->op == IR_GUARD_NOT) {
guarded = 1;
} else if (insn->op >= IR_START || insn->op == IR_CALL || insn->op == IR_LOAD || insn->op == IR_STORE) {
} else if (insn->op >= IR_START) {
if (insn->op == IR_BEGIN && insn->op1 && !insn->op2) {
/* skip END */
ref = insn->op1;
insn = &ctx->ir_base[ref];
} else {
break;
}
} else if (insn->op == IR_CALL || insn->op == IR_LOAD || insn->op == IR_STORE) {
break;
}
next = ref;
Expand All @@ -2422,6 +2447,37 @@ ir_ref ir_find_aliasing_vstore(ir_ctx *ctx, ir_ref ref, ir_ref var, ir_ref val)
}

/* IR Construction API */
static ir_ref ir_last_guard(ir_ctx *ctx)
{
ir_ref ref;
ir_insn *insn;

IR_ASSERT(ctx->control);
ref = ctx->control;
while (1) {
insn = &ctx->ir_base[ref];
if (IR_IS_BB_START(insn->op) || insn->op == IR_GUARD || insn->op == IR_GUARD_NOT) {
if (insn->op == IR_START) ref = IR_UNUSED;
break;
}
ref = insn->op1;
}
return ref;
}

ir_ref _ir_DIV(ir_ctx *ctx, ir_type type, ir_ref op1, ir_ref op2)
{
ir_ref guard = (IR_IS_TYPE_FP(type) || (IR_IS_CONST_REF(op2) && ctx->ir_base[op2].val.u64 != 0)) ?
IR_UNUSED : ir_last_guard(ctx);
return ir_fold3(ctx, IR_OPT(IR_DIV, type), op1, op2, guard);
}

ir_ref _ir_MOD(ir_ctx *ctx, ir_type type, ir_ref op1, ir_ref op2)
{
ir_ref guard = (IR_IS_CONST_REF(op2) && ctx->ir_base[op2].val.u64 != 0) ?
IR_UNUSED : ir_last_guard(ctx);
return ir_fold3(ctx, IR_OPT(IR_MOD, type), op1, op2, guard);
}

ir_ref _ir_PARAM(ir_ctx *ctx, ir_type type, const char* name, ir_ref num)
{
Expand Down
13 changes: 11 additions & 2 deletions ext/opcache/jit/ir/ir.h
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ typedef enum _ir_type {
* arg - argument reference CALL/TAILCALL/CARG->CARG
* src - reference to a previous control region (IF, IF_TRUE, IF_FALSE, MERGE, LOOP_BEGIN, LOOP_END, RETURN)
* reg - data-control dependency on region (PHI, VAR, PARAM)
* grd - optional data-control dependency guard (DIV, MOD)
* ret - reference to a previous RETURN instruction (RETURN)
* str - string: variable/argument name (VAR, PARAM, CALL, TAILCALL)
* num - number: argument number (PARAM)
Expand Down Expand Up @@ -265,8 +266,8 @@ typedef enum _ir_type {
_(ADD, d2C, def, def, ___) /* addition */ \
_(SUB, d2, def, def, ___) /* subtraction (must be ADD+1) */ \
_(MUL, d2C, def, def, ___) /* multiplication */ \
_(DIV, d2, def, def, ___) /* division */ \
_(MOD, d2, def, def, ___) /* modulo */ \
_(DIV, d3, def, def, grd) /* division */ \
_(MOD, d3, def, def, grd) /* modulo */ \
_(NEG, d1, def, ___, ___) /* change sign */ \
_(ABS, d1, def, ___, ___) /* absolute value */ \
/* (LDEXP, MIN, MAX, FPMATH) */ \
Expand Down Expand Up @@ -383,6 +384,14 @@ typedef enum _ir_type {
_(RETURN, T2X1, src, def, ret) /* function return */ \
_(UNREACHABLE, T1X2, src, ___, ret) /* unreachable (tailcall, etc) */ \
\
/* inline assembler */ \
_(ASM, xN, src, def, def) /* GCC inline assembler */ \
/* op2 - asm template string */ \
/* op3 - asm constraint string */ \
/* opN - asm input argument */ \
_(ASM_OUT, x1, src, ___, ___) /* ASM data output projection */ \
_(ASM_GOTO, E1, src, ___, ___) /* ASM goto (bb end after ASM) */ \
\
/* deoptimization helper */ \
_(EXITCALL, x2, src, def, ___) /* save CPU regs and call op2 */ \

Expand Down
30 changes: 20 additions & 10 deletions ext/opcache/jit/ir/ir_aarch64.dasc
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
int flags = IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_MUST_BE_IN_REG;
const ir_proto_t *proto;
const ir_call_conv_dsc *cc;
ir_ref next;

constraints->def_reg = IR_REG_NONE;
constraints->hints_count = 0;
Expand Down Expand Up @@ -562,11 +563,13 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co
constraints->tmp_regs[0] = IR_TMP_REG(1, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n = 1;
}
if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) {
insn = &ctx->ir_base[insn->op2];
if (IR_IS_SYM_CONST(insn->op) || !aarch64_may_encode_imm12(insn->val.u64)) {
constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n++;
if (IR_IS_CONST_REF(insn->op2)) {
if (insn->op1 != insn->op2) {
insn = &ctx->ir_base[insn->op2];
if (IR_IS_SYM_CONST(insn->op) || !aarch64_may_encode_imm12(insn->val.u64)) {
constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
n++;
}
}
} else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
Expand Down Expand Up @@ -751,6 +754,10 @@ get_arg_hints:
break;
case IR_SNAPSHOT:
flags = 0;
next = ir_next_control(ctx, ref);
if (ctx->ir_base[next].op == IR_GUARD || ctx->ir_base[next].op == IR_GUARD_NOT) {
flags = IR_EXTEND_INPUTS_TO_NEXT;
}
break;
case IR_VA_START:
flags = IR_OP2_MUST_BE_IN_REG;
Expand Down Expand Up @@ -1199,10 +1206,6 @@ binop_fp:
if (!IR_IS_CONST_REF(insn->op2) && (ctx->use_lists[insn->op2].count == 1 || all_usages_are_fusable(ctx, insn->op2))) {
op2_insn = &ctx->ir_base[insn->op2];
if (op2_insn->op >= IR_EQ && op2_insn->op <= IR_UNORDERED) {
// TODO: register allocator may clobber operands of CMP before they are used in the GUARD_CMP
//??? && (insn->op2 == ref - 1 ||
//??? (insn->op2 == ctx->prev_ref[ref] - 1
//??? && ctx->ir_base[ctx->prev_ref[ref]].op == IR_SNAPSHOT))) {
if (IR_IS_TYPE_INT(ctx->ir_base[op2_insn->op1].type)) {
ctx->rules[insn->op2] = IR_FUSED | IR_CMP_INT;
return IR_GUARD_CMP_INT;
Expand Down Expand Up @@ -1265,6 +1268,12 @@ binop_fp:
return IR_FUSED | IR_ARGVAL;
case IR_NOP:
return IR_SKIPPED | IR_NOP;
case IR_ASM:
case IR_ASM_OUT:
case IR_ASM_GOTO:
fprintf(stderr, "ERROR: IR_ASM is not implemented yet\n");
exit(1);
return IR_SKIPPED | IR_NOP;
default:
break;
}
Expand Down Expand Up @@ -4996,7 +5005,8 @@ static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]);

| .addr &addr
if (ctx->ir_base[bb->start].op != IR_CASE_DEFAULT) {
if (ctx->ir_base[bb->start].op1 == def
&& ctx->ir_base[bb->start].op != IR_CASE_DEFAULT) {
bb->flags |= IR_BB_EMPTY;
}
continue;
Expand Down
46 changes: 24 additions & 22 deletions ext/opcache/jit/ir/ir_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -118,31 +118,31 @@ extern "C" {
#define ir_MUL_D(_op1, _op2) ir_BINARY_OP_D(IR_MUL, (_op1), (_op2))
#define ir_MUL_F(_op1, _op2) ir_BINARY_OP_F(IR_MUL, (_op1), (_op2))

#define ir_DIV(_type, _op1, _op2) ir_BINARY_OP(IR_DIV, (_type), (_op1), (_op2))
#define ir_DIV_U8(_op1, _op2) ir_BINARY_OP_U8(IR_DIV, (_op1), (_op2))
#define ir_DIV_U16(_op1, _op2) ir_BINARY_OP_U16(IR_DIV, (_op1), (_op2))
#define ir_DIV_U32(_op1, _op2) ir_BINARY_OP_U32(IR_DIV, (_op1), (_op2))
#define ir_DIV_U64(_op1, _op2) ir_BINARY_OP_U64(IR_DIV, (_op1), (_op2))
#define ir_DIV_A(_op1, _op2) ir_BINARY_OP_A(IR_DIV, (_op1), (_op2))
#define ir_DIV_C(_op1, _op2) ir_BINARY_OP_C(IR_DIV, (_op1), (_op2))
#define ir_DIV_I8(_op1, _op2) ir_BINARY_OP_I8(IR_DIV, (_op1), (_op2))
#define ir_DIV_I16(_op1, _op2) ir_BINARY_OP_I16(IR_DIV, (_op1), (_op2))
#define ir_DIV_I32(_op1, _op2) ir_BINARY_OP_I32(IR_DIV, (_op1), (_op2))
#define ir_DIV_I64(_op1, _op2) ir_BINARY_OP_I64(IR_DIV, (_op1), (_op2))
#define ir_DIV(_type, _op1, _op2) _ir_DIV(_ir_CTX, (_type), (_op1), (_op2))
#define ir_DIV_U8(_op1, _op2) ir_DIV(IR_U8, (_op1), (_op2))
#define ir_DIV_U16(_op1, _op2) ir_DIV(IR_U16, (_op1), (_op2))
#define ir_DIV_U32(_op1, _op2) ir_DIV(IR_U32, (_op1), (_op2))
#define ir_DIV_U64(_op1, _op2) ir_DIV(IR_U64, (_op1), (_op2))
#define ir_DIV_A(_op1, _op2) ir_DIV(IR_ADDR, (_op1), (_op2))
#define ir_DIV_C(_op1, _op2) ir_DIV(IR_CHAR, (_op1), (_op2))
#define ir_DIV_I8(_op1, _op2) ir_DIV(IR_I8, (_op1), (_op2))
#define ir_DIV_I16(_op1, _op2) ir_DIV(IR_I16, (_op1), (_op2))
#define ir_DIV_I32(_op1, _op2) ir_DIV(IR_I32, (_op1), (_op2))
#define ir_DIV_I64(_op1, _op2) ir_DIV(IR_I64, (_op1), (_op2))
#define ir_DIV_D(_op1, _op2) ir_BINARY_OP_D(IR_DIV, (_op1), (_op2))
#define ir_DIV_F(_op1, _op2) ir_BINARY_OP_F(IR_DIV, (_op1), (_op2))

#define ir_MOD(_type, _op1, _op2) ir_BINARY_OP(IR_MOD, (_type), (_op1), (_op2))
#define ir_MOD_U8(_op1, _op2) ir_BINARY_OP_U8(IR_MOD, (_op1), (_op2))
#define ir_MOD_U16(_op1, _op2) ir_BINARY_OP_U16(IR_MOD, (_op1), (_op2))
#define ir_MOD_U32(_op1, _op2) ir_BINARY_OP_U32(IR_MOD, (_op1), (_op2))
#define ir_MOD_U64(_op1, _op2) ir_BINARY_OP_U64(IR_MOD, (_op1), (_op2))
#define ir_MOD_A(_op1, _op2) ir_BINARY_OP_A(IR_MOD, (_op1), (_op2))
#define ir_MOD_C(_op1, _op2) ir_BINARY_OP_C(IR_MOD, (_op1), (_op2))
#define ir_MOD_I8(_op1, _op2) ir_BINARY_OP_I8(IR_MOD, (_op1), (_op2))
#define ir_MOD_I16(_op1, _op2) ir_BINARY_OP_I16(IR_MOD, (_op1), (_op2))
#define ir_MOD_I32(_op1, _op2) ir_BINARY_OP_I32(IR_MOD, (_op1), (_op2))
#define ir_MOD_I64(_op1, _op2) ir_BINARY_OP_I64(IR_MOD, (_op1), (_op2))
#define ir_MOD(_type, _op1, _op2) _ir_MOD(_ir_CTX, (_type), (_op1), (_op2))
#define ir_MOD_U8(_op1, _op2) ir_MOD(IR_U8, (_op1), (_op2))
#define ir_MOD_U16(_op1, _op2) ir_MOD(IR_U16, (_op1), (_op2))
#define ir_MOD_U32(_op1, _op2) ir_MOD(IR_U32, (_op1), (_op2))
#define ir_MOD_U64(_op1, _op2) ir_MOD(IR_U64, (_op1), (_op2))
#define ir_MOD_A(_op1, _op2) ir_MOD(IR_ADDR, (_op1), (_op2))
#define ir_MOD_C(_op1, _op2) ir_MOD(IR_CHAR, (_op1), (_op2))
#define ir_MOD_I8(_op1, _op2) ir_MOD(IR_I8, (_op1), (_op2))
#define ir_MOD_I16(_op1, _op2) ir_MOD(IR_I16, (_op1), (_op2))
#define ir_MOD_I32(_op1, _op2) ir_MOD(IR_I32, (_op1), (_op2))
#define ir_MOD_I64(_op1, _op2) ir_MOD(IR_I64, (_op1), (_op2))

#define ir_NEG(_type, _op1) ir_UNARY_OP(IR_NEG, (_type), (_op1))
#define ir_NEG_C(_op1) ir_UNARY_OP_C(IR_NEG, (_op1))
Expand Down Expand Up @@ -633,6 +633,8 @@ extern "C" {
#define ir_MERGE_WITH_EMPTY_TRUE(_if) do {ir_ref end = ir_END(); ir_IF_TRUE(_if); ir_MERGE_2(end, ir_END());} while (0)
#define ir_MERGE_WITH_EMPTY_FALSE(_if) do {ir_ref end = ir_END(); ir_IF_FALSE(_if); ir_MERGE_2(end, ir_END());} while (0)

ir_ref _ir_DIV(ir_ctx *ctx, ir_type type, ir_ref op1, ir_ref op2);
ir_ref _ir_MOD(ir_ctx *ctx, ir_type type, ir_ref op1, ir_ref op2);
ir_ref _ir_ADD_OFFSET(ir_ctx *ctx, ir_ref addr, uintptr_t offset);
ir_ref _ir_PHI_2(ir_ctx *ctx, ir_type type, ir_ref src1, ir_ref src2);
ir_ref _ir_PHI_N(ir_ctx *ctx, ir_type type, ir_ref n, ir_ref *inputs);
Expand Down
19 changes: 18 additions & 1 deletion ext/opcache/jit/ir/ir_cfg.c
Original file line number Diff line number Diff line change
Expand Up @@ -1502,6 +1502,23 @@ static bool ir_is_merged_loop_back_edge(ir_ctx *ctx, uint32_t hdr, uint32_t b)
}
#endif

static bool ir_should_align_loop(ir_ctx *ctx, ir_chain *chains, uint32_t b, ir_block *bb)
{
uint32_t n = bb->predecessors_count;
uint32_t *p = ctx->cfg_edges + bb->predecessors;

for (; n > 0; p++, n--) {
uint32_t pred = *p;
if (chains[pred].head) {
if (ir_chain_head(chains, pred) == b) return 1;
} else {
if (ir_should_align_loop(ctx, chains, b, &ctx->cfg_blocks[pred])) return 1;
}
}

return 0;
}

static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
{
uint32_t max_edges_count = ctx->cfg_edges_count / 2;
Expand Down Expand Up @@ -1862,7 +1879,7 @@ static int ir_schedule_blocks_bottom_up(ir_ctx *ctx)
if (chains[b].head == b) {
bb = &ctx->cfg_blocks[b];
if (bb->loop_depth) {
if ((bb->flags & IR_BB_LOOP_HEADER) || ir_chain_head(chains, bb->loop_header) == b) {
if (ir_should_align_loop(ctx, chains, b, bb)) {
bb->flags |= IR_BB_ALIGN_LOOP;
}
}
Expand Down
21 changes: 21 additions & 0 deletions ext/opcache/jit/ir/ir_check.c
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,12 @@ bool ir_check(const ir_ctx *ctx)
bool ok = 1;
ir_check_ctx check_ctx;

if (ctx->insns_count < 1 || ctx->ir_base[1].op != IR_START) {
fprintf(stderr, "ir_base[1].op invalid opcode (%d)\n",
(ctx->insns_count < 1) ? IR_NOP : ctx->ir_base[0].op);
ok = 0;
}

check_ctx.arena = NULL;
check_ctx.use_set = NULL;
check_ctx.input_set = NULL;
Expand Down Expand Up @@ -297,6 +303,14 @@ bool ir_check(const ir_ctx *ctx)
ok = 0;
}
break;
case IR_OPND_CONTROL_GUARD:
if (!(ir_op_flags[use_insn->op] & IR_OP_FLAG_BB_START)
&& use_insn->op != IR_GUARD
&& use_insn->op != IR_GUARD_NOT) {
fprintf(stderr, "ir_base[%d].ops[%d] reference (%d) must be BB_START or GUARD\n", i, j, use);
ok = 0;
}
break;
default:
fprintf(stderr, "ir_base[%d].ops[%d] reference (%d) of unsupported kind\n", i, j, use);
ok = 0;
Expand All @@ -306,6 +320,8 @@ bool ir_check(const ir_ctx *ctx)
/* pass (function returns void) */
} else if (insn->op == IR_BEGIN && j == 1) {
/* pass (start of unreachable basic block) */
} else if (IR_OPND_KIND(flags, j) == IR_OPND_CONTROL_GUARD) {
/* reference to control guard is optional */
} else if (IR_OPND_KIND(flags, j) != IR_OPND_CONTROL_REF
&& (insn->op != IR_SNAPSHOT || j == 1)) {
fprintf(stderr, "ir_base[%d].ops[%d] missing reference (%d)\n", i, j, use);
Expand Down Expand Up @@ -413,6 +429,7 @@ bool ir_check(const ir_ctx *ctx)
}
break;
case IR_IGOTO:
case IR_ASM_GOTO:
break;
default:
/* skip data references */
Expand Down Expand Up @@ -464,6 +481,10 @@ bool ir_check(const ir_ctx *ctx)
// if (!ok) {
// ir_dump_codegen(ctx, stderr);
// }

#ifndef IR_CHECK_NO_ABORT
IR_ASSERT(ok);
#endif

return ok;
}
Loading