Last active
May 18, 2021 08:56
-
-
Save dstogov/6760c1158e919f0ca185ddf63601b8c7 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c | |
index 151eb4ecc5..c8bd6936be 100644 | |
--- a/Zend/zend_execute.c | |
+++ b/Zend/zend_execute.c | |
@@ -4833,10 +4833,8 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint | |
} \ | |
} while (0) | |
-#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
/* This callback disables optimization of "vm_stack_data" variable in VM */ | |
-void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL; | |
-#endif | |
+ZEND_API void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data) = NULL; | |
#include "zend_vm_execute.h" | |
diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h | |
index 209c77650c..b61509c36a 100644 | |
--- a/Zend/zend_vm_execute.h | |
+++ b/Zend/zend_vm_execute.h | |
@@ -50963,12 +50963,6 @@ ZEND_API void execute_ex(zend_execute_data *ex) | |
#else | |
zend_execute_data *execute_data = ex; | |
#endif | |
-#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
- memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE); | |
- if (zend_touch_vm_stack_data) { | |
- zend_touch_vm_stack_data(&vm_stack_data); | |
- } | |
-#endif | |
#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) | |
if (UNEXPECTED(execute_data == NULL)) { | |
@@ -54431,6 +54425,12 @@ ZEND_API void execute_ex(zend_execute_data *ex) | |
zend_handlers_count = sizeof(labels) / sizeof(void*); | |
memset(&hybrid_halt_op, 0, sizeof(hybrid_halt_op)); | |
hybrid_halt_op.handler = (void*)&&HYBRID_HALT_LABEL; | |
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
+ memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE); | |
+#endif | |
+ if (zend_touch_vm_stack_data) { | |
+ zend_touch_vm_stack_data(&vm_stack_data); | |
+ } | |
goto HYBRID_HALT_LABEL; | |
} | |
#endif | |
diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php | |
index 973ae380ef..578237a0e4 100755 | |
--- a/Zend/zend_vm_gen.php | |
+++ b/Zend/zend_vm_gen.php | |
@@ -2089,12 +2089,6 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) | |
out($f,"#else\n"); | |
out($f,$m[1]."zend_execute_data *execute_data = ex;\n"); | |
out($f,"#endif\n"); | |
- out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n"); | |
- out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n"); | |
- out($f,$m[1]."if (zend_touch_vm_stack_data) {\n"); | |
- out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n"); | |
- out($f,$m[1]."}\n"); | |
- out($f,"#endif\n"); | |
} | |
break; | |
case "INTERNAL_LABELS": | |
@@ -2114,6 +2108,12 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) | |
if ($kind == ZEND_VM_KIND_HYBRID) { | |
out($f,$prolog."\tmemset(&hybrid_halt_op, 0, sizeof(hybrid_halt_op));\n"); | |
out($f,$prolog."\thybrid_halt_op.handler = (void*)&&HYBRID_HALT_LABEL;\n"); | |
+ out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n"); | |
+ out($f,$prolog."\tmemset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n"); | |
+ out($f,"#endif\n"); | |
+ out($f,$prolog."\tif (zend_touch_vm_stack_data) {\n"); | |
+ out($f,$prolog."\t\tzend_touch_vm_stack_data(&vm_stack_data);\n"); | |
+ out($f,$prolog."\t}\n"); | |
out($f,$prolog."\tgoto HYBRID_HALT_LABEL;\n"); | |
} else { | |
out($f,$prolog."\treturn;\n"); | |
diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c | |
index 8cd14fc987..9a88c8886b 100644 | |
--- a/ext/opcache/jit/zend_jit.c | |
+++ b/ext/opcache/jit/zend_jit.c | |
@@ -94,10 +94,12 @@ zend_jit_globals jit_globals; | |
typedef struct _zend_jit_stub { | |
const char *name; | |
int (*stub)(dasm_State **Dst); | |
+ uint32_t offset; | |
+ uint32_t adjustment; | |
} zend_jit_stub; | |
-#define JIT_STUB(name) \ | |
- {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} | |
+#define JIT_STUB(name, offset, adjustment) \ | |
+ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment} | |
zend_ulong zend_jit_profile_counter = 0; | |
int zend_jit_profile_counter_rid = -1; | |
@@ -348,7 +350,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, | |
const zend_op *rt_opline, | |
zend_lifetime_interval **ra, | |
const char *name, | |
- uint32_t trace_num) | |
+ uint32_t trace_num, | |
+ uint32_t sp_offset, | |
+ uint32_t sp_adjustment) | |
{ | |
size_t size; | |
int ret; | |
@@ -507,7 +511,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, | |
name, | |
op_array, | |
entry, | |
- size); | |
+ size, | |
+ sp_adj[sp_offset], | |
+ sp_adj[sp_adjustment]); | |
} | |
} | |
#endif | |
@@ -3547,7 +3553,8 @@ done: | |
} | |
} | |
- handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0); | |
+ handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0, | |
+ (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT); | |
if (!handler) { | |
goto jit_failure; | |
} | |
@@ -4103,7 +4110,8 @@ static int zend_jit_make_stubs(void) | |
if (!zend_jit_stubs[i].stub(&dasm_state)) { | |
return 0; | |
} | |
- if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0)) { | |
+ if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0, | |
+ zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) { | |
return 0; | |
} | |
} | |
diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc | |
index fea32f9c0a..b5238992b6 100644 | |
--- a/ext/opcache/jit/zend_jit_arm64.dasc | |
+++ b/ext/opcache/jit/zend_jit_arm64.dasc | |
@@ -83,11 +83,22 @@ | |
|.define ZREG_TMP3, ZREG_X17 | |
|.define ZREG_FPTMP, ZREG_V16 | |
-|.define HYBRID_SPAD, #32 // padding for stack alignment | |
+|.define HYBRID_SPAD, 32 // padding for stack alignment | |
#define TMP_ZVAL_OFFSET 16 | |
#define DASM_ALIGNMENT 16 | |
+typedef enum _sp_adj_kind { | |
+ SP_ADJ_NONE, | |
+ SP_ADJ_RET, | |
+ SP_ADJ_VM, | |
+ SP_ADJ_JIT, | |
+ SP_ADJ_ASSIGN, | |
+ SP_ADJ_LAST | |
+} sp_adj_kind; | |
+ | |
+static int sp_adj[SP_ADJ_LAST]; | |
+ | |
/* Encoding of immediate. TODO: shift mode may be supported in the near future. */ | |
#define MAX_IMM12 0xfff // maximum value for imm12 | |
#define MAX_IMM16 0xffff // maximum value for imm16 | |
@@ -196,13 +207,13 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | |
|.macro ADD_HYBRID_SPAD | |
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
-| add sp, sp, HYBRID_SPAD | |
+| add sp, sp, # HYBRID_SPAD | |
||#endif | |
|.endmacro | |
|.macro SUB_HYBRID_SPAD | |
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
-| sub sp, sp, HYBRID_SPAD | |
+| sub sp, sp, # HYBRID_SPAD | |
||#endif | |
|.endmacro | |
@@ -2592,42 +2603,42 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) | |
} | |
static const zend_jit_stub zend_jit_stubs[] = { | |
- JIT_STUB(interrupt_handler), | |
- JIT_STUB(exception_handler), | |
- JIT_STUB(exception_handler_undef), | |
- JIT_STUB(leave_function), | |
- JIT_STUB(leave_throw), | |
- JIT_STUB(icall_throw), | |
- JIT_STUB(throw_cannot_pass_by_ref), | |
- JIT_STUB(undefined_offset), | |
- JIT_STUB(undefined_index), | |
- JIT_STUB(cannot_add_element), | |
- JIT_STUB(undefined_offset_ex), | |
- JIT_STUB(undefined_index_ex), | |
- JIT_STUB(cannot_add_element_ex), | |
- JIT_STUB(undefined_function), | |
- JIT_STUB(negative_shift), | |
- JIT_STUB(mod_by_zero), | |
- JIT_STUB(invalid_this), | |
- JIT_STUB(trace_halt), | |
- JIT_STUB(trace_exit), | |
- JIT_STUB(trace_escape), | |
- JIT_STUB(hybrid_runtime_jit), | |
- JIT_STUB(hybrid_profile_jit), | |
- JIT_STUB(hybrid_hot_code), | |
- JIT_STUB(hybrid_func_hot_counter), | |
- JIT_STUB(hybrid_loop_hot_counter), | |
- JIT_STUB(hybrid_hot_trace), | |
- JIT_STUB(hybrid_func_trace_counter), | |
- JIT_STUB(hybrid_ret_trace_counter), | |
- JIT_STUB(hybrid_loop_trace_counter), | |
- JIT_STUB(assign_const), | |
- JIT_STUB(assign_tmp), | |
- JIT_STUB(assign_var), | |
- JIT_STUB(assign_cv_noref), | |
- JIT_STUB(assign_cv), | |
+ JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
#ifdef CONTEXT_THREADED_JIT | |
- JIT_STUB(context_threaded_call), | |
+ JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), | |
#endif | |
}; | |
@@ -2637,6 +2648,66 @@ extern char *_tls_start; | |
extern char *_tls_end; | |
#endif | |
+#ifdef HAVE_GDB | |
+# if 0 | |
+typedef struct _Unwind_Context _Unwind_Context; | |
+typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); | |
+extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); | |
+extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); | |
+ | |
+typedef struct _zend_jit_unwind_arg { | |
+ int cnt; | |
+ uintptr_t cfa[3]; | |
+} zend_jit_unwind_arg; | |
+ | |
+static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) | |
+{ | |
+ zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; | |
+ arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); | |
+ arg->cnt++; | |
+ if (arg->cnt == 3) { | |
+ return 5; // _URC_END_OF_STACK | |
+ } | |
+ return 0; // _URC_NO_REASON; | |
+} | |
+ | |
+static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) | |
+{ | |
+ zend_jit_unwind_arg arg; | |
+ | |
+ memset(&arg, 0, sizeof(arg)); | |
+ _Unwind_Backtrace(zend_jit_unwind_cb, &arg); | |
+ if (arg.cnt == 3) { | |
+ sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; | |
+ } | |
+} | |
+# else | |
+static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) | |
+{ | |
+ uintptr_t ret; | |
+ | |
+ __asm__ ( | |
+ "ldr %0, [x29]\n\t" | |
+ "sub %0 ,%0, x29" | |
+ : "=r"(ret)); | |
+ | |
+ sp_adj[SP_ADJ_VM] = ret; | |
+} | |
+# endif | |
+ | |
+extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); | |
+ | |
+static zend_never_inline void zend_jit_set_sp_adj_vm(void) | |
+{ | |
+ void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); | |
+ | |
+ orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; | |
+ zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; | |
+ execute_ex(NULL); // set sp_adj[SP_ADJ_VM] | |
+ zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; | |
+} | |
+#endif | |
+ | |
static int zend_jit_setup(void) | |
{ | |
allowed_opt_flags = 0; | |
@@ -2753,6 +2824,24 @@ static int zend_jit_setup(void) | |
# endif | |
#endif | |
+ memset(sp_adj, 0, sizeof(sp_adj)); | |
+#ifdef HAVE_GDB | |
+ sp_adj[SP_ADJ_RET] = 0; | |
+ sp_adj[SP_ADJ_ASSIGN] = 32; | |
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | |
+ zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] | |
+#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD | |
+#else | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; | |
+#endif | |
+ } else if (GCC_GLOBAL_REGS) { | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD | |
+ } else { | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD | |
+ } | |
+#endif | |
+ | |
return SUCCESS; | |
} | |
@@ -2780,8 +2869,10 @@ static int zend_jit_prologue(dasm_State **Dst) | |
| SUB_HYBRID_SPAD | |
} else if (GCC_GLOBAL_REGS) { | |
| stp x29, x30, [sp, # -SPAD]! // stack alignment | |
+ |// mov x29, sp | |
} else { | |
| stp x29, x30, [sp, # -NR_SPAD]! // stack alignment | |
+ |// mov x29, sp | |
| stp FP, RX, T2 // save FP and IP | |
| mov FP, FCARG1x | |
} | |
diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c | |
index be26583277..0cb171b5ba 100644 | |
--- a/ext/opcache/jit/zend_jit_gdb.c | |
+++ b/ext/opcache/jit/zend_jit_gdb.c | |
@@ -96,6 +96,7 @@ enum { | |
#elif defined(__aarch64__) | |
DW_REG_SP = 31, | |
DW_REG_RA = 30, | |
+ DW_REG_X29 = 29, | |
#else | |
#error "Unsupported target architecture" | |
#endif | |
@@ -305,7 +306,7 @@ typedef ZEND_SET_ALIGNED(1, uintptr_t unaligned_uintptr_t); | |
{ unaligned_uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ | |
*szp_##name = (uint32_t)((p-(uint8_t *)szp_##name)-4); } | |
-static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) | |
+static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx, uint32_t sp_offset, uint32_t sp_adjustment) | |
{ | |
uint8_t *p = ctx->p; | |
uint8_t *framep = p; | |
@@ -319,8 +320,12 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) | |
DSV(-(int32_t)sizeof(uintptr_t)); /* Data alignment factor. */ | |
DB(DW_REG_RA); /* Return address register. */ | |
DB(1); DB(DW_EH_PE_textrel|DW_EH_PE_udata4); /* Augmentation data. */ | |
+#if defined(__x86_64__) || defined(i386) | |
DB(DW_CFA_def_cfa); DUV(DW_REG_SP); DUV(sizeof(uintptr_t)); | |
DB(DW_CFA_offset|DW_REG_RA); DUV(1); | |
+#elif defined(__aarch64__) | |
+ DB(DW_CFA_def_cfa); DUV(DW_REG_SP); DUV(0); | |
+#endif | |
DALIGNNOP(sizeof(uintptr_t)); | |
) | |
@@ -330,19 +335,22 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) | |
DU32(0); /* Machine code offset relative to .text. */ | |
DU32(ctx->szmcode); /* Machine code length. */ | |
DB(0); /* Augmentation data. */ | |
- DB(DW_CFA_def_cfa_offset); DUV(sizeof(uintptr_t)); | |
-#if defined(__i386__) | |
- DB(DW_CFA_advance_loc|3); /* sub $0xc,%esp */ | |
- DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ | |
-#elif defined(__x86_64__) | |
- DB(DW_CFA_advance_loc|4); /* sub $0x8,%rsp */ | |
- DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ | |
-#elif defined(__aarch64__) | |
- DB(DW_CFA_advance_loc|1); /* Only an approximation. */ | |
- DB(DW_CFA_def_cfa_offset); DUV(32); /* Aligned stack frame size. */ | |
-#else | |
-# error "Unsupported target architecture" | |
+ DB(DW_CFA_def_cfa_offset); DUV(sp_offset); | |
+#if defined(__aarch64__) | |
+ if (sp_offset) { | |
+ DB(DW_CFA_offset|DW_REG_X29); DUV(sp_offset / sizeof(uintptr_t)); | |
+ DB(DW_CFA_offset|DW_REG_RA); DUV((sp_offset / sizeof(uintptr_t)) - 1); | |
+ } | |
+#endif | |
+ if (sp_adjustment && sp_offset != sp_adjustment) { | |
+ DB(DW_CFA_advance_loc|1); DB(DW_CFA_def_cfa_offset); DUV(sp_adjustment); | |
+#if defined(__aarch64__) | |
+ if (!sp_offset) { | |
+ DB(DW_CFA_offset|DW_REG_X29); DUV(sp_adjustment / sizeof(uintptr_t)); | |
+ DB(DW_CFA_offset|DW_REG_RA); DUV((sp_adjustment / sizeof(uintptr_t)) - 1); | |
+ } | |
#endif | |
+ } | |
DALIGNNOP(sizeof(uintptr_t)); | |
) | |
@@ -439,15 +447,18 @@ static void zend_gdbjit_debugline(zend_gdbjit_ctx *ctx) | |
typedef void (*zend_gdbjit_initf) (zend_gdbjit_ctx *ctx); | |
-static void zend_gdbjit_initsect(zend_gdbjit_ctx *ctx, int sect, zend_gdbjit_initf initf) | |
+static void zend_gdbjit_initsect(zend_gdbjit_ctx *ctx, int sect) | |
{ | |
ctx->startp = ctx->p; | |
ctx->obj.sect[sect].ofs = (uintptr_t)((char *)ctx->p - (char *)&ctx->obj); | |
- initf(ctx); | |
+} | |
+ | |
+static void zend_gdbjit_initsect_done(zend_gdbjit_ctx *ctx, int sect) | |
+{ | |
ctx->obj.sect[sect].size = (uintptr_t)(ctx->p - ctx->startp); | |
} | |
-static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) | |
+static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx, uint32_t sp_offset, uint32_t sp_adjustment) | |
{ | |
zend_gdbjit_obj *obj = &ctx->obj; | |
@@ -458,13 +469,13 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) | |
/* Initialize sections. */ | |
ctx->p = obj->space; | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_shstrtab, zend_gdbjit_secthdr); | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_strtab, zend_gdbjit_symtab); | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_info, zend_gdbjit_debuginfo); | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_abbrev, zend_gdbjit_debugabbrev); | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_line, zend_gdbjit_debugline); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_shstrtab); zend_gdbjit_secthdr(ctx); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_shstrtab); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_strtab); zend_gdbjit_symtab(ctx); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_strtab); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_info); zend_gdbjit_debuginfo(ctx); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_debug_info); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_abbrev); zend_gdbjit_debugabbrev(ctx); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_debug_abbrev); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_debug_line); zend_gdbjit_debugline(ctx); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_debug_line); | |
SECTALIGN(ctx->p, sizeof(uintptr_t)); | |
- zend_gdbjit_initsect(ctx, GDBJIT_SECT_eh_frame, zend_gdbjit_ehframe); | |
+ zend_gdbjit_initsect(ctx, GDBJIT_SECT_eh_frame); zend_gdbjit_ehframe(ctx, sp_offset, sp_adjustment); zend_gdbjit_initsect_done(ctx, GDBJIT_SECT_eh_frame); | |
ctx->objsize = (size_t)((char *)ctx->p - (char *)obj); | |
ZEND_ASSERT(ctx->objsize < sizeof(zend_gdbjit_obj)); | |
@@ -473,7 +484,9 @@ static void zend_gdbjit_buildobj(zend_gdbjit_ctx *ctx) | |
static int zend_jit_gdb_register(const char *name, | |
const zend_op_array *op_array, | |
const void *start, | |
- size_t size) | |
+ size_t size, | |
+ uint32_t sp_offset, | |
+ uint32_t sp_adjustment) | |
{ | |
zend_gdbjit_ctx ctx; | |
@@ -483,7 +496,7 @@ static int zend_jit_gdb_register(const char *name, | |
ctx.filename = op_array ? ZSTR_VAL(op_array->filename) : "unknown"; | |
ctx.lineno = op_array ? op_array->line_start : 0; | |
- zend_gdbjit_buildobj(&ctx); | |
+ zend_gdbjit_buildobj(&ctx, sp_offset, sp_adjustment); | |
return zend_gdb_register_code(&ctx.obj, ctx.objsize); | |
} | |
diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c | |
index 2b379e2429..6e72b50ef0 100644 | |
--- a/ext/opcache/jit/zend_jit_trace.c | |
+++ b/ext/opcache/jit/zend_jit_trace.c | |
@@ -86,7 +86,7 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) | |
zend_jit_trace_exit_group_stub(&dasm_state, n); | |
sprintf(name, "jit$$trace_exit_%d", n); | |
- entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0); | |
+ entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); | |
dasm_free(&dasm_state); | |
#ifdef HAVE_DISASM | |
@@ -6386,7 +6386,9 @@ done: | |
goto jit_failure; | |
} | |
- handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM); | |
+ handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM, | |
+ parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET), | |
+ parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT); | |
if (handler) { | |
if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) { | |
@@ -6518,7 +6520,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n | |
zend_jit_trace_return(&dasm_state, original_handler); | |
- handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM); | |
+ handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE); | |
jit_failure: | |
dasm_free(&dasm_state); | |
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc | |
index 31a19dcc1f..d369cdb125 100644 | |
--- a/ext/opcache/jit/zend_jit_x86.dasc | |
+++ b/ext/opcache/jit/zend_jit_x86.dasc | |
@@ -103,6 +103,17 @@ | |
#define DASM_ALIGNMENT 16 | |
+typedef enum _sp_adj_kind { | |
+ SP_ADJ_NONE, | |
+ SP_ADJ_RET, | |
+ SP_ADJ_VM, | |
+ SP_ADJ_JIT, | |
+ SP_ADJ_ASSIGN, | |
+ SP_ADJ_LAST | |
+} sp_adj_kind; | |
+ | |
+static int sp_adj[SP_ADJ_LAST]; | |
+ | |
/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to | |
* guarantee proper alignment of 128-bit SSE data allocated on stack. | |
* With broken alignment any execution of SSE code, including calls to | |
@@ -2781,43 +2792,43 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) | |
} | |
static const zend_jit_stub zend_jit_stubs[] = { | |
- JIT_STUB(interrupt_handler), | |
- JIT_STUB(exception_handler), | |
- JIT_STUB(exception_handler_undef), | |
- JIT_STUB(leave_function), | |
- JIT_STUB(leave_throw), | |
- JIT_STUB(icall_throw), | |
- JIT_STUB(throw_cannot_pass_by_ref), | |
- JIT_STUB(undefined_offset), | |
- JIT_STUB(undefined_index), | |
- JIT_STUB(cannot_add_element), | |
- JIT_STUB(undefined_offset_ex), | |
- JIT_STUB(undefined_index_ex), | |
- JIT_STUB(cannot_add_element_ex), | |
- JIT_STUB(undefined_function), | |
- JIT_STUB(negative_shift), | |
- JIT_STUB(mod_by_zero), | |
- JIT_STUB(invalid_this), | |
- JIT_STUB(trace_halt), | |
- JIT_STUB(trace_exit), | |
- JIT_STUB(trace_escape), | |
- JIT_STUB(hybrid_runtime_jit), | |
- JIT_STUB(hybrid_profile_jit), | |
- JIT_STUB(hybrid_hot_code), | |
- JIT_STUB(hybrid_func_hot_counter), | |
- JIT_STUB(hybrid_loop_hot_counter), | |
- JIT_STUB(hybrid_hot_trace), | |
- JIT_STUB(hybrid_func_trace_counter), | |
- JIT_STUB(hybrid_ret_trace_counter), | |
- JIT_STUB(hybrid_loop_trace_counter), | |
- JIT_STUB(assign_const), | |
- JIT_STUB(assign_tmp), | |
- JIT_STUB(assign_var), | |
- JIT_STUB(assign_cv_noref), | |
- JIT_STUB(assign_cv), | |
- JIT_STUB(double_one), | |
+ JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), | |
+ JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), | |
+ JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE), | |
#ifdef CONTEXT_THREADED_JIT | |
- JIT_STUB(context_threaded_call), | |
+ JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE), | |
#endif | |
}; | |
@@ -2827,6 +2838,52 @@ extern char *_tls_start; | |
extern char *_tls_end; | |
#endif | |
+#ifdef HAVE_GDB | |
+typedef struct _Unwind_Context _Unwind_Context; | |
+typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); | |
+extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); | |
+extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); | |
+ | |
+typedef struct _zend_jit_unwind_arg { | |
+ int cnt; | |
+ uintptr_t cfa[3]; | |
+} zend_jit_unwind_arg; | |
+ | |
+static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) | |
+{ | |
+ zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; | |
+ arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); | |
+ arg->cnt++; | |
+ if (arg->cnt == 3) { | |
+ return 5; // _URC_END_OF_STACK | |
+ } | |
+ return 0; // _URC_NO_REASON; | |
+} | |
+ | |
+static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) | |
+{ | |
+ zend_jit_unwind_arg arg; | |
+ | |
+ memset(&arg, 0, sizeof(arg)); | |
+ _Unwind_Backtrace(zend_jit_unwind_cb, &arg); | |
+ if (arg.cnt == 3) { | |
+ sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; | |
+ } | |
+} | |
+ | |
+extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); | |
+ | |
+static zend_never_inline void zend_jit_set_sp_adj_vm(void) | |
+{ | |
+ void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); | |
+ | |
+ orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; | |
+ zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; | |
+ execute_ex(NULL); // set sp_adj[SP_ADJ_VM] | |
+ zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; | |
+} | |
+#endif | |
+ | |
static int zend_jit_setup(void) | |
{ | |
if (!zend_cpu_supports_sse2()) { | |
@@ -2946,6 +3003,30 @@ static int zend_jit_setup(void) | |
# endif | |
#endif | |
+ memset(sp_adj, 0, sizeof(sp_adj)); | |
+#ifdef HAVE_GDB | |
+ sp_adj[SP_ADJ_RET] = sizeof(void*); | |
+ |.if X64WIN | |
+ || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28 | |
+ |.elif X64 | |
+ || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8 | |
+ |.else | |
+ || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12 | |
+ |.endif | |
+ if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | |
+ zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] | |
+#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD | |
+#else | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; | |
+#endif | |
+ } else if (GCC_GLOBAL_REGS) { | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD | |
+ } else { | |
+ || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD | |
+ } | |
+#endif | |
+ | |
return SUCCESS; | |
} | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment