From ccc49ead681add7bdf6a021c7a3420aff543bc1e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 20 Mar 2020 10:54:45 +0300 Subject: [PATCH] Change jit_trace_stack_frame layout --- ext/opcache/jit/zend_jit_internal.h | 67 +++++++++++++++++++++++++---- ext/opcache/jit/zend_jit_trace.c | 65 ++++++++++------------------ ext/opcache/jit/zend_jit_x86.dasc | 28 +++++++----- 3 files changed, 97 insertions(+), 63 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 922f4b25ac1..05183c87046 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -331,18 +331,67 @@ struct _zend_jit_trace_stack_frame { zend_jit_trace_stack_frame *call; zend_jit_trace_stack_frame *prev; const zend_function *func; - union { - struct { - int8_t return_value_used; - int8_t nested; - int8_t num_args; - int8_t last_send_by_ref; - }; - int return_ssa_var; - }; + uint32_t _info; zend_jit_trace_stack stack[1]; }; +#define TRACE_FRAME_SHIFT_NUM_ARGS 16 +#define TRACE_FRAME_MAX_NUM_ARGS 32767 + +#define TRACE_FRAME_MASK_NUM_ARGS 0xffff0000 +#define TRACE_FRAME_MASK_NESTED 0x00000001 +#define TRACE_FRAME_MASK_LAST_SEND_BY_REF 0x00000002 +#define TRACE_FRAME_MASK_LAST_SEND_BY_VAL 0x00000004 +#define TRACE_FRAME_MASK_RETURN_VALUE_USED 0x00000008 +#define TRACE_FRAME_MASK_RETURN_VALUE_UNUSED 0x00000010 + + +#define TRACE_FRAME_INIT(frame, _func, nested, num_args) do { \ + zend_jit_trace_stack_frame *_frame = (frame); \ + _frame->call = NULL; \ + _frame->prev = NULL; \ + _frame->func = (const zend_function*)_func; \ + _frame->_info = (uint32_t)((((int)(num_args)) << TRACE_FRAME_SHIFT_NUM_ARGS) & TRACE_FRAME_MASK_NUM_ARGS); \ + if (nested) { \ + _frame->_info |= TRACE_FRAME_MASK_NESTED; \ + }; \ + } while (0) + +#define TRACE_FRAME_RETURN_SSA_VAR(frame) \ + ((int)(frame)->_info) +#define TRACE_FRAME_NUM_ARGS(frame) \ + ((int)((frame)->_info) >> TRACE_FRAME_SHIFT_NUM_ARGS) +#define TRACE_FRAME_IS_NESTED(frame) \ + ((frame)->_info & TRACE_FRAME_MASK_NESTED) +#define TRACE_FRAME_IS_LAST_SEND_BY_REF(frame) \ + ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_REF) +#define TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame) \ + ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_VAL) +#define TRACE_FRAME_IS_RETURN_VALUE_USED(frame) \ + ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_USED) +#define TRACE_FRAME_IS_RETURN_VALUE_UNUSED(frame) \ + ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_UNUSED) + +#define TRACE_FRAME_SET_RETURN_SSA_VAR(frame, var) do { \ + (frame)->_info = var; \ + } while (0) +#define TRACE_FRAME_SET_LAST_SEND_BY_REF(frame) do { \ + (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_REF; \ + (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \ + } while (0) +#define TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame) do { \ + (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \ + (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \ + } while (0) +#define TRACE_FRAME_SET_RETURN_VALUE_USED(frame) do { \ + (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_USED; \ + (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \ + } while (0) +#define TRACE_FRAME_SET_RETURN_VALUE_UNUSED(frame) do { \ + (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \ + (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_USED; \ + } while (0) + typedef struct _zend_jit_globals { zend_jit_trace_stack_frame *current_frame; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index c0b32b2a7b6..3a42bc06cfb 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -734,7 +734,7 @@ static int find_call_num_args(zend_jit_trace_rec *p) || p->opline->opcode == ZEND_NEW || p->opline->opcode == ZEND_INIT_METHOD_CALL || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) { - if (p->opline->extended_value <= 127) { + if (p->opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) { return p->opline->extended_value; } else { return -1; @@ -1147,10 +1147,8 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin frame = JIT_G(current_frame); top = zend_jit_trace_call_frame(frame, op_array); - frame->call = NULL; - frame->prev = NULL; - frame->func = (const zend_function*)op_array; - frame->return_ssa_var = -1; + TRACE_FRAME_INIT(frame, op_array, 0, 0); + TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); for (i = 0; i < op_array->last_var + op_array->T; i++) { frame->stack[i] = -1; } @@ -1444,9 +1442,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if (!call) { /* Trace missed INIT_FCALL opcode */ call = top; - call->call = NULL; - call->prev = NULL; - call->func = (const zend_function*)op_array; + TRACE_FRAME_INIT(call, op_array, 0, 0); top = zend_jit_trace_call_frame(top, op_array); for (i = 0; i < op_array->last_var + op_array->T; i++) { call->stack[i] = -1; @@ -1456,7 +1452,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } frame->call = call->prev; call->prev = frame; - call->return_ssa_var = find_return_ssa_var(p - 1, ssa_ops + (idx - 1)); + TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1))); frame = call; level++; @@ -1534,8 +1530,8 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if ((p+1)->op == ZEND_JIT_TRACE_VM) { const zend_op *opline = (p+1)->opline - 1; if (opline->result_type != IS_UNUSED) { - if (frame->return_ssa_var >= 0) { - ssa_var_info[frame->return_ssa_var] = return_value_info; + if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) { + ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info; } } } @@ -1549,10 +1545,8 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ZEND_ASSERT(&frame->func->op_array == op_array); } else { frame = zend_jit_trace_ret_frame(frame, op_array); - frame->call = NULL; - frame->prev = NULL; - frame->func = (const zend_function*)op_array; - frame->return_ssa_var = -1; + TRACE_FRAME_INIT(frame, op_array, 0, 0); + TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); for (i = 0; i < op_array->last_var + op_array->T; i++) { frame->stack[i] = -1; } @@ -1560,9 +1554,8 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call = top; - call->call = NULL; + TRACE_FRAME_INIT(call, p->func, 0, 0); call->prev = frame->call; - call->func = p->func; frame->call = call; top = zend_jit_trace_call_frame(top, p->op_array); if (p->func->type == ZEND_USER_FUNCTION) { @@ -1662,13 +1655,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op_array = p->op_array; frame = JIT_G(current_frame); top = zend_jit_trace_call_frame(frame, op_array); - frame->call = NULL; - frame->prev = NULL; - frame->func = (const zend_function*)op_array; - frame->return_value_used = -1; - frame->nested = 0; - frame->num_args = -1; - frame->last_send_by_ref = -1; + TRACE_FRAME_INIT(frame, op_array, 0, -1); stack = frame->stack; if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { @@ -2885,12 +2872,7 @@ done: if (!call) { /* Trace missed INIT_FCALL opcode */ call = top; - call->call = NULL; - call->prev = NULL; - call->func = (const zend_function*)op_array; - call->nested = 0; - call->num_args = -1; // TODO: should be possible to get the real number ??? - call->last_send_by_ref = -1; + TRACE_FRAME_INIT(call, op_array, 0, -1); // TODO: should be possible to get the real number af arguments ??? top = zend_jit_trace_call_frame(top, op_array); i = 0; while (i < p->op_array->num_args) { @@ -2916,7 +2898,11 @@ done: } frame->call = call->prev; call->prev = frame; - call->return_value_used = p->return_value_used; + if (p->return_value_used) { + TRACE_FRAME_SET_RETURN_VALUE_USED(call); + } else { + TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call); + } JIT_G(current_frame) = frame = call; stack = frame->stack; zend_jit_set_opline(&dasm_state, (p+1)->opline); @@ -2932,13 +2918,7 @@ done: ZEND_ASSERT(&frame->func->op_array == op_array); } else { frame = zend_jit_trace_ret_frame(frame, op_array); - frame->call = NULL; - frame->prev = NULL; - frame->func = (const zend_function*)op_array; - frame->return_value_used = -1; - frame->nested = 0; - frame->num_args = -1; - frame->last_send_by_ref = -1; + TRACE_FRAME_INIT(frame, op_array, 0, -1); stack = frame->stack; for (i = 0; i < op_array->last_var + op_array->T; i++) { /* Initialize abstract stack using SSA */ @@ -2963,12 +2943,11 @@ done: break; } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call = top; - call->call = NULL; + TRACE_FRAME_INIT(call, p->func, 1, find_call_num_args(p-1)); call->prev = frame->call; - call->func = p->func; - call->nested = 1; - call->num_args = find_call_num_args(p-1); - call->last_send_by_ref = p->fake ? -1 : 0; + if (!p->fake) { + TRACE_FRAME_SET_LAST_SEND_BY_VAL(call); + } frame->call = call; top = zend_jit_trace_call_frame(top, p->op_array); if (p->func->type == ZEND_USER_FUNCTION) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4ac2b08d111..4a9306aff29 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -7881,8 +7881,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && - JIT_G(current_frame)->call->num_args >= 0) { - call_num_args = JIT_G(current_frame)->call->num_args; + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } @@ -7893,8 +7893,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && - JIT_G(current_frame)->call->num_args >= 0) { - call_num_args = JIT_G(current_frame)->call->num_args; + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } @@ -8319,7 +8319,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !JIT_G(current_frame)->call || - !JIT_G(current_frame)->call->nested) { + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call)) { | // zend_vm_stack_free_call_frame(call); | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) @@ -8759,8 +8759,8 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline, cons && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { - if (JIT_G(current_frame)->call->last_send_by_ref != 1) { - JIT_G(current_frame)->call->last_send_by_ref = 1; + if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF @@ -8770,8 +8770,8 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline, cons || } } } else { - if (JIT_G(current_frame)->call->last_send_by_ref != 0) { - JIT_G(current_frame)->call->last_send_by_ref = 0; + if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF @@ -9269,7 +9269,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || - !JIT_G(current_frame)->nested) { + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame))) { // TODO: try to avoid this check ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | cmp IP, zend_jit_halt_op @@ -9411,7 +9411,13 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { - return_value_used = JIT_G(current_frame)->return_value_used; + if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { + return_value_used = 1; + } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { + return_value_used = 0; + } else { + return_value_used = -1; + } } else { return_value_used = -1; }