From e771fd001005373cf1b62a0fd3473480fefe31fe Mon Sep 17 00:00:00 2001 From: cocolato Date: Sun, 11 Jan 2026 16:15:46 +0800 Subject: [PATCH 1/9] use add_op to save optimized uops --- Include/internal/pycore_optimizer_types.h | 4 + Python/optimizer.c | 1 + Python/optimizer_analysis.c | 42 +++++- Python/optimizer_bytecodes.c | 118 +++++++-------- Python/optimizer_cases.c.h | 144 +++++++++---------- Tools/cases_generator/optimizer_generator.py | 2 +- 6 files changed, 172 insertions(+), 139 deletions(-) diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 0a193268c4d618..57928cb3a09490 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -128,6 +128,10 @@ typedef struct _JitOptContext { JitOptRef *n_consumed; JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + + // buffer to save optimized result + int out_len; + _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; } JitOptContext; diff --git a/Python/optimizer.c b/Python/optimizer.c index 79ac179d0b710a..74ad47c40cd6e9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1509,6 +1509,7 @@ uop_optimize( if (length <= 0) { return length; } + buffer = _tstate->jit_tracer_state->opt_context.out_buffer; } assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e855df4977acf8..2b5cd37309a347 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -152,6 +152,16 @@ incorrect_keys(PyObject *obj, uint32_t version) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; +#define ADD_OP(OP, ARG, OPERAND) ( \ + ctx->out_buffer[ctx->out_len].opcode = (OP), \ + ctx->out_buffer[ctx->out_len].format = this_instr->format, \ + ctx->out_buffer[ctx->out_len].oparg = (ARG), \ + ctx->out_buffer[ctx->out_len].target = this_instr->target, \ + ctx->out_buffer[ctx->out_len].operand0 = (OPERAND), \ + ctx->out_buffer[ctx->out_len].operand1 = this_instr->operand1, \ + ctx->out_len++ \ +) + /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const @@ -219,7 +229,7 @@ optimize_to_bool( bool insert_mode) { if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); *result_ptr = value; return 1; } @@ -229,7 +239,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)load); + ADD_OP(opcode, 0, (uintptr_t)load); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -237,9 +247,9 @@ optimize_to_bool( } static void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) { - REPLACE_OP(this_instr, _POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); if (exit) { REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; @@ -256,7 +266,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); + ADD_OP(opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); @@ -364,6 +374,7 @@ optimize_uops( frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; + ctx->out_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -387,6 +398,7 @@ optimize_uops( } #endif + int out_len_before = ctx->out_len; switch (opcode) { #include "optimizer_cases.c.h" @@ -395,6 +407,10 @@ optimize_uops( DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } + // If no ADD_OP was called during this iteration, copy the original instruction + if (ctx->out_len == out_len_before) { + ctx->out_buffer[ctx->out_len++] = *this_instr; + } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); @@ -423,7 +439,18 @@ optimize_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - return trace_len; + // Check that the trace ends with a proper terminator + if (ctx->out_len > 0) { + int last_opcode = ctx->out_buffer[ctx->out_len - 1].opcode; + if (last_opcode != _EXIT_TRACE && + last_opcode != _JUMP_TO_TOP && + last_opcode != _DYNAMIC_EXIT && + last_opcode != _DEOPT) { + return 0; + } + } + + return ctx->out_len; error: DPRINTF(3, "\n"); @@ -584,6 +611,7 @@ _Py_uop_analyze_and_optimize( ) { OPT_STAT_INC(optimizer_attempts); + JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; length = optimize_uops( tstate, buffer, @@ -595,7 +623,7 @@ _Py_uop_analyze_and_optimize( assert(length > 0); - length = remove_unneeded_uops(buffer, length); + length = remove_unneeded_uops(ctx->out_buffer, length); assert(length > 0); OPT_STAT_INC(optimizer_successes); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d8868dc020787d..626b478c840c3a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -48,7 +48,7 @@ optimize_to_bool( bool insert_mode); extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -139,11 +139,11 @@ dummy_func(void) { op(_GUARD_TOS_INT, (value -- value)) { if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -151,11 +151,11 @@ dummy_func(void) { op(_GUARD_NOS_INT, (left, unused -- left, unused)) { if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -165,7 +165,7 @@ dummy_func(void) { PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -176,7 +176,7 @@ dummy_func(void) { op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { // add watcher so that whenever the type changes we invalidate this PyTypeObject *type = _PyType_LookupByVersion(type_version); @@ -198,14 +198,14 @@ dummy_func(void) { op(_GUARD_TOS_FLOAT, (value -- value)) { if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); } @@ -352,7 +352,7 @@ dummy_func(void) { assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -419,7 +419,7 @@ dummy_func(void) { op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); } @@ -430,7 +430,7 @@ dummy_func(void) { op(_GUARD_TOS_UNICODE, (value -- value)) { if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); } @@ -533,7 +533,7 @@ dummy_func(void) { op(_LOAD_CONST, (-- value)) { PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -541,7 +541,7 @@ dummy_func(void) { PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); } @@ -579,34 +579,34 @@ dummy_func(void) { if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } } op(_POP_TOP_INT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_FLOAT, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } op(_POP_TOP_UNICODE, (value --)) { if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } } @@ -659,11 +659,11 @@ dummy_func(void) { op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) { if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } @@ -758,8 +758,8 @@ dummy_func(void) { op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -768,8 +768,8 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -781,7 +781,7 @@ dummy_func(void) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -1044,7 +1044,7 @@ dummy_func(void) { op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); } @@ -1057,7 +1057,7 @@ dummy_func(void) { PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)type); } else { @@ -1095,7 +1095,7 @@ dummy_func(void) { out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } } @@ -1103,7 +1103,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); } @@ -1148,7 +1148,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); } @@ -1157,11 +1157,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); } @@ -1170,11 +1170,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } } @@ -1182,7 +1182,7 @@ dummy_func(void) { /* Setting the eval frame function invalidates * all executors, so no need to check dynamically */ if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } @@ -1210,7 +1210,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res, v)) { - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; } @@ -1265,42 +1265,42 @@ dummy_func(void) { op(_GUARD_TOS_LIST, (tos -- tos)) { if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); } op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); } op(_GUARD_TOS_TUPLE, (tos -- tos)) { if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); } op(_GUARD_TOS_DICT, (tos -- tos)) { if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); } op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); } @@ -1309,48 +1309,48 @@ dummy_func(void) { if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } op(_GUARD_NOS_NULL, (null, unused -- null, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); } op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); } op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); } op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); } op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); } @@ -1364,7 +1364,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); @@ -1386,7 +1386,7 @@ dummy_func(void) { goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); Py_DECREF(temp); @@ -1396,7 +1396,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); } @@ -1404,7 +1404,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); } @@ -1412,7 +1412,7 @@ dummy_func(void) { op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); } @@ -1448,7 +1448,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1509,7 +1509,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 845d2f235aeaf8..eb6b53196c6fef 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -76,7 +76,7 @@ JitOptRef value; PyCodeObject *co = get_current_code_object(ctx); PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -90,7 +90,7 @@ PyObject *val = PyLong_FromLong(oparg); assert(val); assert(_Py_IsImmortal(val)); - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; @@ -117,16 +117,16 @@ if (PyJitRef_IsBorrowed(value) || sym_is_immortal(PyJitRef_Unwrap(value)) || sym_is_null(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } else if (typ == &PyLong_Type) { - REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + ADD_OP(_POP_TOP_INT, 0, 0); } else if (typ == &PyFloat_Type) { - REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + ADD_OP(_POP_TOP_FLOAT, 0, 0); } else if (typ == &PyUnicode_Type) { - REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + ADD_OP(_POP_TOP_UNICODE, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -145,7 +145,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -157,7 +157,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -169,7 +169,7 @@ JitOptRef value; value = stack_pointer[-1]; if (PyJitRef_IsBorrowed(value)) { - REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + ADD_OP(_POP_TOP_NOP, 0, 0); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; @@ -241,7 +241,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -283,7 +283,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -335,7 +335,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyList_Type); break; @@ -345,7 +345,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyList_Type); break; @@ -391,7 +391,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyUnicode_Type); break; @@ -401,7 +401,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyUnicode_Type); break; @@ -430,7 +430,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); v = value; CHECK_STACK_BOUNDS(1); @@ -465,7 +465,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } stack_pointer[-1] = res; @@ -486,11 +486,11 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_is_compact_int(left)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(left) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0); } sym_set_compact_int(left); } @@ -501,11 +501,11 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_is_compact_int(value)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { if (sym_get_type(value) == &PyLong_Type) { - REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0); } sym_set_compact_int(value); } @@ -564,7 +564,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -628,7 +628,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -692,7 +692,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -716,7 +716,7 @@ JitOptRef left; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(left, &PyFloat_Type); break; @@ -726,7 +726,7 @@ JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyFloat_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(value, &PyFloat_Type); break; @@ -957,7 +957,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyTuple_Type); break; @@ -967,7 +967,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyTuple_Type); break; @@ -985,7 +985,7 @@ assert(index >= 0); int tuple_length = sym_tuple_length(tuple_st); if (tuple_length != -1 && index < tuple_length) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } break; @@ -1031,7 +1031,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(nos, &PyDict_Type); break; @@ -1041,7 +1041,7 @@ JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(tos, &PyDict_Type); break; @@ -1438,11 +1438,11 @@ JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { - REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); + ADD_OP(_PUSH_NULL, 0, 0); null[0] = sym_new_null(ctx); } else { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); @@ -1467,7 +1467,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version == version) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } } } @@ -1496,7 +1496,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -1762,7 +1762,7 @@ uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); if (sym_matches_type_version(owner, type_version)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { PyTypeObject *type = _PyType_LookupByVersion(type_version); if (type) { @@ -1876,7 +1876,7 @@ PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); if (type) { if (type == sym_get_const(ctx, owner)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } else { sym_set_const(owner, type); @@ -2005,7 +2005,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -2067,7 +2067,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2131,7 +2131,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2193,7 +2193,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -2262,7 +2262,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -2285,7 +2285,7 @@ if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } break; } @@ -2375,7 +2375,7 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(1); @@ -2500,7 +2500,7 @@ JitOptRef iter; iter = stack_pointer[-2]; if (sym_matches_type(iter, &PyTuple_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_type(iter, &PyTuple_Type); break; @@ -2754,8 +2754,8 @@ uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2772,8 +2772,8 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); - REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)method->im_func; + ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; @@ -2821,7 +2821,7 @@ case _CHECK_PEP_523: { if (_PyInterpreterState_GET()->eval_frame == NULL) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } break; } @@ -2837,7 +2837,7 @@ PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { - REPLACE_OP(this_instr, _NOP, 0 ,0); + ADD_OP(_NOP, 0 ,0); } } } @@ -2915,7 +2915,7 @@ JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -2925,7 +2925,7 @@ JitOptRef nos; nos = stack_pointer[-2]; if (sym_is_not_null(nos)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_non_null(nos); break; @@ -2935,7 +2935,7 @@ JitOptRef null; null = stack_pointer[-3]; if (sym_is_null(null)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_null(null); break; @@ -2945,7 +2945,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyType_Type); break; @@ -2959,8 +2959,8 @@ PyObject* type = (PyObject *)sym_get_type(arg); if (type) { res = sym_new_const(ctx, type); - REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, - (uintptr_t)type); + ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -2978,7 +2978,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyUnicode_Type); break; @@ -3008,7 +3008,7 @@ JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, (PyObject *)&PyTuple_Type); break; @@ -3150,7 +3150,7 @@ callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, len); break; @@ -3172,8 +3172,8 @@ goto error; } if (_Py_IsImmortal(temp)) { - REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - 0, (uintptr_t)temp); + ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); CHECK_STACK_BOUNDS(-2); @@ -3196,7 +3196,7 @@ callable = stack_pointer[-4]; PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; if (sym_get_const(ctx, callable) == isinstance) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, isinstance); break; @@ -3217,7 +3217,7 @@ out = Py_True; } sym_set_const(res, out); - REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; @@ -3231,7 +3231,7 @@ callable = stack_pointer[-3]; PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; if (sym_get_const(ctx, callable) == list_append) { - REPLACE_OP(this_instr, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); } sym_set_const(callable, list_append); break; @@ -3525,7 +3525,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(-1); @@ -3612,7 +3612,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); CHECK_STACK_BOUNDS(-1); @@ -3627,7 +3627,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); CHECK_STACK_BOUNDS(-1); @@ -3642,11 +3642,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); CHECK_STACK_BOUNDS(-1); @@ -3661,11 +3661,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, false); + eliminate_pop_guard(this_instr, ctx, false); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 922ed803c93fb5..56560ba88e47b4 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -270,7 +270,7 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"PyObject *result = sym_get_const(ctx, {output_identifier.text});\n") emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") - emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit(f"ADD_OP({replacement_uop}, 0, (uintptr_t)result);\n") emitter.emit("}\n") emitter.emit("}\n") From 4a0db151c0bffe8d077dbfffef5d283b14c19b88 Mon Sep 17 00:00:00 2001 From: cocolato Date: Mon, 12 Jan 2026 01:41:15 +0800 Subject: [PATCH 2/9] optimize code struct --- Include/internal/pycore_optimizer_types.h | 4 +- Include/internal/pycore_tstate.h | 2 + Python/optimizer.c | 2 +- Python/optimizer_analysis.c | 59 ++++++++++++++--------- Python/optimizer_bytecodes.c | 6 +-- Python/optimizer_cases.c.h | 6 +-- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 57928cb3a09490..1062c4253feba1 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -129,9 +129,7 @@ typedef struct _JitOptContext { JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; - // buffer to save optimized result - int out_len; - _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; + struct _PyJitTracerState *tracer; } JitOptContext; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 518fd94a31ae5e..2b41d6ad848a16 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -59,6 +59,8 @@ typedef struct _PyJitTracerState { _PyJitTracerTranslatorState translator_state; JitOptContext opt_context; _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH]; + _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; + int out_len; } _PyJitTracerState; #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index 74ad47c40cd6e9..0d7dd0446fe255 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1509,7 +1509,7 @@ uop_optimize( if (length <= 0) { return length; } - buffer = _tstate->jit_tracer_state->opt_context.out_buffer; + buffer = _tstate->jit_tracer_state->out_buffer; } assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 2b5cd37309a347..38392386cd85d0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -143,6 +143,18 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +static inline int +is_terminator_uop(const _PyUOpInstruction *uop) +{ + int opcode = uop->opcode; + return ( + opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT || + opcode == _DEOPT + ); +} + #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -152,15 +164,21 @@ incorrect_keys(PyObject *obj, uint32_t version) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; -#define ADD_OP(OP, ARG, OPERAND) ( \ - ctx->out_buffer[ctx->out_len].opcode = (OP), \ - ctx->out_buffer[ctx->out_len].format = this_instr->format, \ - ctx->out_buffer[ctx->out_len].oparg = (ARG), \ - ctx->out_buffer[ctx->out_len].target = this_instr->target, \ - ctx->out_buffer[ctx->out_len].operand0 = (OPERAND), \ - ctx->out_buffer[ctx->out_len].operand1 = this_instr->operand1, \ - ctx->out_len++ \ -) +#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND)) + +static inline void +add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, + uint16_t opcode, uint16_t oparg, uintptr_t operand0) +{ + _PyUOpInstruction *out = &ctx->tracer->out_buffer[ctx->tracer->out_len]; + out->opcode = (opcode); + out->format = this_instr->format; + out->oparg = (oparg); + out->target = this_instr->target; + out->operand0 = (operand0); + out->operand1 = this_instr->operand1; + ctx->tracer->out_len++; +} /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -359,6 +377,8 @@ optimize_uops( JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; uint32_t opcode = UINT16_MAX; + ctx->tracer = tstate->jit_tracer_state; + // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { @@ -374,7 +394,7 @@ optimize_uops( frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; - ctx->out_len = 0; + ctx->tracer->out_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -398,7 +418,7 @@ optimize_uops( } #endif - int out_len_before = ctx->out_len; + int out_len_before = ctx->tracer->out_len; switch (opcode) { #include "optimizer_cases.c.h" @@ -408,8 +428,8 @@ optimize_uops( Py_UNREACHABLE(); } // If no ADD_OP was called during this iteration, copy the original instruction - if (ctx->out_len == out_len_before) { - ctx->out_buffer[ctx->out_len++] = *this_instr; + if (ctx->tracer->out_len == out_len_before) { + ctx->tracer->out_buffer[ctx->tracer->out_len++] = *this_instr; } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -440,17 +460,13 @@ optimize_uops( * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); // Check that the trace ends with a proper terminator - if (ctx->out_len > 0) { - int last_opcode = ctx->out_buffer[ctx->out_len - 1].opcode; - if (last_opcode != _EXIT_TRACE && - last_opcode != _JUMP_TO_TOP && - last_opcode != _DYNAMIC_EXIT && - last_opcode != _DEOPT) { + if (ctx->tracer->out_len > 0) { + if (!is_terminator_uop(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1])) { return 0; } } - return ctx->out_len; + return ctx->tracer->out_len; error: DPRINTF(3, "\n"); @@ -611,7 +627,6 @@ _Py_uop_analyze_and_optimize( ) { OPT_STAT_INC(optimizer_attempts); - JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; length = optimize_uops( tstate, buffer, @@ -623,7 +638,7 @@ _Py_uop_analyze_and_optimize( assert(length > 0); - length = remove_unneeded_uops(ctx->out_buffer, length); + length = remove_unneeded_uops(tstate->jit_tracer_state->out_buffer, length); assert(length > 0); OPT_STAT_INC(optimizer_successes); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 626b478c840c3a..75e28e3a2d000a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -759,7 +759,7 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -769,7 +769,7 @@ dummy_func(void) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -1509,7 +1509,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index eb6b53196c6fef..8f721507b464de 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1496,7 +1496,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -2755,7 +2755,7 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2773,7 +2773,7 @@ PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; From 25797cf308d8d5497f61bd45099bdc34ff9e3e88 Mon Sep 17 00:00:00 2001 From: cocolato Date: Mon, 12 Jan 2026 20:59:51 +0800 Subject: [PATCH 3/9] optimize code struct --- Python/optimizer_analysis.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 38392386cd85d0..230c06afc4b0b3 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -418,7 +418,6 @@ optimize_uops( } #endif - int out_len_before = ctx->tracer->out_len; switch (opcode) { #include "optimizer_cases.c.h" @@ -428,7 +427,7 @@ optimize_uops( Py_UNREACHABLE(); } // If no ADD_OP was called during this iteration, copy the original instruction - if (ctx->tracer->out_len == out_len_before) { + if (ctx->tracer->out_len == i) { ctx->tracer->out_buffer[ctx->tracer->out_len++] = *this_instr; } assert(ctx->frame != NULL); @@ -461,8 +460,15 @@ optimize_uops( _Py_uop_abstractcontext_fini(ctx); // Check that the trace ends with a proper terminator if (ctx->tracer->out_len > 0) { - if (!is_terminator_uop(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1])) { - return 0; + _PyUOpInstruction *last_uop = &ctx->tracer->out_buffer[ctx->tracer->out_len - 1]; + if (!is_terminator_uop(last_uop)) { + // Copy remaining uops from original trace until we find a terminator + for (int i = ctx->tracer->out_len; i < trace_len; i++) { + ctx->tracer->out_buffer[ctx->tracer->out_len++] = trace[i]; + if (is_terminator_uop(&trace[i])) { + break; + } + } } } From 57a22e946314fb1635b3f1753e8b912fa2492a0e Mon Sep 17 00:00:00 2001 From: cocolato Date: Mon, 12 Jan 2026 21:34:51 +0800 Subject: [PATCH 4/9] regen cases --- Python/optimizer_cases.c.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4c1c934da10b22..203070878767d7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -245,7 +245,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); @@ -481,7 +481,7 @@ PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { // Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); From 031d6e3e842e62fd38bde29aaced21118d66f8b5 Mon Sep 17 00:00:00 2001 From: cocolato Date: Tue, 13 Jan 2026 00:18:09 +0800 Subject: [PATCH 5/9] optimize code --- Include/internal/pycore_optimizer_types.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 1062c4253feba1..77fbb930d22b04 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -10,6 +10,8 @@ extern "C" { #include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH +typedef struct _PyJitTracerState _PyJitTracerState; + // Holds locals, stack, locals, stack ... (in that order) #define MAX_ABSTRACT_INTERP_SIZE 512 @@ -128,8 +130,7 @@ typedef struct _JitOptContext { JitOptRef *n_consumed; JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; - - struct _PyJitTracerState *tracer; + _PyJitTracerState *tracer; } JitOptContext; From 45f8ce0cbd0b9e3ad2194817a3b0ccfe4bdb3eef Mon Sep 17 00:00:00 2001 From: cocolato Date: Tue, 13 Jan 2026 01:00:49 +0800 Subject: [PATCH 6/9] remove out_len --- Include/internal/pycore_optimizer_types.h | 4 +- Include/internal/pycore_tstate.h | 1 - Python/optimizer_analysis.c | 39 +++++++++---------- Python/optimizer_bytecodes.c | 46 ++++++++++++----------- Python/optimizer_cases.c.h | 42 ++++++++++----------- 5 files changed, 66 insertions(+), 66 deletions(-) diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 77fbb930d22b04..7a429442602bfc 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -10,8 +10,6 @@ extern "C" { #include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH -typedef struct _PyJitTracerState _PyJitTracerState; - // Holds locals, stack, locals, stack ... (in that order) #define MAX_ABSTRACT_INTERP_SIZE 512 @@ -130,7 +128,7 @@ typedef struct _JitOptContext { JitOptRef *n_consumed; JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; - _PyJitTracerState *tracer; + _PyUOpInstruction *out_buffer; } JitOptContext; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 2b41d6ad848a16..c278e290076fe6 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -60,7 +60,6 @@ typedef struct _PyJitTracerState { JitOptContext opt_context; _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH]; _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH]; - int out_len; } _PyJitTracerState; #endif diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 230c06afc4b0b3..97c066042e0a08 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -164,20 +164,20 @@ is_terminator_uop(const _PyUOpInstruction *uop) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; -#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND)) +#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, &out_len, (OP), (ARG), (OPERAND)) static inline void -add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, +add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, int *out_len, uint16_t opcode, uint16_t oparg, uintptr_t operand0) { - _PyUOpInstruction *out = &ctx->tracer->out_buffer[ctx->tracer->out_len]; + _PyUOpInstruction *out = &ctx->out_buffer[*out_len]; out->opcode = (opcode); out->format = this_instr->format; out->oparg = (oparg); out->target = this_instr->target; out->operand0 = (operand0); out->operand1 = this_instr->operand1; - ctx->tracer->out_len++; + (*out_len)++; } /* Shortened forms for convenience, used in optimizer_bytecodes.c */ @@ -242,12 +242,13 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, + int *out_len, JitOptRef value, JitOptRef *result_ptr, bool insert_mode) { if (sym_matches_type(value, &PyBool_Type)) { - ADD_OP(_NOP, 0, 0); + add_op(ctx, this_instr, out_len, _NOP, 0, 0); *result_ptr = value; return 1; } @@ -257,7 +258,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - ADD_OP(opcode, 0, (uintptr_t)load); + add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)load); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -265,9 +266,9 @@ optimize_to_bool( } static void -eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit) { - ADD_OP(_POP_TOP, 0, 0); + add_op(ctx, this_instr, out_len, _POP_TOP, 0, 0); if (exit) { REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; @@ -276,7 +277,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit static JitOptRef lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction *this_instr, - PyTypeObject *type, PyObject *name, uint16_t immortal, + int *out_len, PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) { // The cached value may be dead, so we need to do the lookup again... :( @@ -284,7 +285,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; - ADD_OP(opcode, 0, (uintptr_t)lookup); + add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); @@ -377,7 +378,7 @@ optimize_uops( JitOptContext *ctx = &tstate->jit_tracer_state->opt_context; uint32_t opcode = UINT16_MAX; - ctx->tracer = tstate->jit_tracer_state; + ctx->out_buffer = tstate->jit_tracer_state->out_buffer; // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -394,7 +395,7 @@ optimize_uops( frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; - ctx->tracer->out_len = 0; + int out_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -427,8 +428,8 @@ optimize_uops( Py_UNREACHABLE(); } // If no ADD_OP was called during this iteration, copy the original instruction - if (ctx->tracer->out_len == i) { - ctx->tracer->out_buffer[ctx->tracer->out_len++] = *this_instr; + if (out_len == i) { + ctx->out_buffer[out_len++] = *this_instr; } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -459,12 +460,12 @@ optimize_uops( * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); // Check that the trace ends with a proper terminator - if (ctx->tracer->out_len > 0) { - _PyUOpInstruction *last_uop = &ctx->tracer->out_buffer[ctx->tracer->out_len - 1]; + if (out_len > 0) { + _PyUOpInstruction *last_uop = &ctx->out_buffer[out_len - 1]; if (!is_terminator_uop(last_uop)) { // Copy remaining uops from original trace until we find a terminator - for (int i = ctx->tracer->out_len; i < trace_len; i++) { - ctx->tracer->out_buffer[ctx->tracer->out_len++] = trace[i]; + for (int i = out_len; i < trace_len; i++) { + ctx->out_buffer[out_len++] = trace[i]; if (is_terminator_uop(&trace[i])) { break; } @@ -472,7 +473,7 @@ optimize_uops( } } - return ctx->tracer->out_len; + return out_len; error: DPRINTF(3, "\n"); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a5df5b7b51c0eb..6e0d54515ae03e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -43,12 +43,13 @@ extern int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, + int *out_len, JitOptSymbol *value, JitOptSymbol **result_ptr, bool insert_mode); extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit); extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -57,6 +58,7 @@ dummy_func(void) { PyCodeObject *co; int oparg; + int out_len; JitOptSymbol *flag; JitOptSymbol *left; JitOptSymbol *right; @@ -381,21 +383,21 @@ dummy_func(void) { } op(_TO_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &value, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &value, false); if (!already_bool) { sym_set_type(value, &PyBool_Type); } } op(_TO_BOOL_INT, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { sym_set_type(value, &PyLong_Type); res = sym_new_truthiness(ctx, value, true); @@ -403,14 +405,14 @@ dummy_func(void) { } op(_TO_BOOL_LIST, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { sym_set_const(value, Py_None); res = sym_new_const(ctx, Py_False); @@ -436,7 +438,7 @@ dummy_func(void) { } op(_TO_BOOL_STR, (value -- res, v)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, true); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, true); v = value; if (!already_bool) { res = sym_new_truthiness(ctx, value, true); @@ -693,7 +695,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -702,7 +704,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -711,7 +713,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -720,7 +722,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -730,7 +732,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -740,7 +742,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -761,7 +763,7 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -771,7 +773,7 @@ dummy_func(void) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -1105,7 +1107,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_True); + eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_True); } sym_set_const(flag, Py_True); } @@ -1150,7 +1152,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_False); + eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_False); } sym_set_const(flag, Py_False); } @@ -1159,11 +1161,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, &out_len, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, true); + eliminate_pop_guard(this_instr, ctx, &out_len, true); } sym_set_const(val, Py_None); } @@ -1172,11 +1174,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, &out_len, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, false); + eliminate_pop_guard(this_instr, ctx, &out_len, false); } } @@ -1511,7 +1513,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 203070878767d7..8dfd348a67fa93 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -311,7 +311,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { res = sym_new_truthiness(ctx, value, true); } @@ -322,7 +322,7 @@ case _TO_BOOL_BOOL: { JitOptRef value; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &value, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &value, false); if (!already_bool) { sym_set_type(value, &PyBool_Type); } @@ -334,7 +334,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { sym_set_type(value, &PyLong_Type); res = sym_new_truthiness(ctx, value, true); @@ -371,7 +371,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { res = sym_new_type(ctx, &PyBool_Type); } @@ -383,7 +383,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); if (!already_bool) { sym_set_const(value, Py_None); res = sym_new_const(ctx, Py_False); @@ -424,7 +424,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res, true); + int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, true); v = value; if (!already_bool) { res = sym_new_truthiness(ctx, value, true); @@ -1520,7 +1520,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->tracer->out_buffer[ctx->tracer->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -1917,7 +1917,7 @@ (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2649,7 +2649,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2670,7 +2670,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2690,7 +2690,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2705,7 +2705,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2725,7 +2725,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2779,7 +2779,7 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2797,7 +2797,7 @@ PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->tracer->out_buffer[ctx->tracer->out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; @@ -3636,7 +3636,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_True); + eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_True); } sym_set_const(flag, Py_True); CHECK_STACK_BOUNDS(-1); @@ -3651,7 +3651,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, value != Py_False); + eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_False); } sym_set_const(flag, Py_False); CHECK_STACK_BOUNDS(-1); @@ -3666,11 +3666,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, &out_len, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, true); + eliminate_pop_guard(this_instr, ctx, &out_len, true); } sym_set_const(val, Py_None); CHECK_STACK_BOUNDS(-1); @@ -3685,11 +3685,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, &out_len, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, false); + eliminate_pop_guard(this_instr, ctx, &out_len, false); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; From 0d07c5b5281afe0118e46ed1b3598629a104598c Mon Sep 17 00:00:00 2001 From: cocolato Date: Tue, 13 Jan 2026 01:44:32 +0800 Subject: [PATCH 7/9] regen cases --- Python/optimizer_cases.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bab64a7850e870..f1f5f20fe934d4 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2307,7 +2307,7 @@ PyObject *result = sym_get_const(ctx, b); if (_Py_IsImmortal(result)) { // Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } CHECK_STACK_BOUNDS(1); From 15f32b9385e7471d4abcf0d741e4e4d7085a8d4d Mon Sep 17 00:00:00 2001 From: cocolato Date: Tue, 13 Jan 2026 02:17:19 +0800 Subject: [PATCH 8/9] move out_len to JitOptContext --- Include/internal/pycore_optimizer_types.h | 1 + Python/optimizer_analysis.c | 39 +++++++++---------- Python/optimizer_bytecodes.c | 46 +++++++++++------------ Python/optimizer_cases.c.h | 42 ++++++++++----------- 4 files changed, 64 insertions(+), 64 deletions(-) diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 7a429442602bfc..f2459a36921125 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -129,6 +129,7 @@ typedef struct _JitOptContext { JitOptRef *limit; JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; _PyUOpInstruction *out_buffer; + int out_len; } JitOptContext; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 97c066042e0a08..7697312a492c96 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -164,20 +164,20 @@ is_terminator_uop(const _PyUOpInstruction *uop) (INST)->oparg = ARG; \ (INST)->operand0 = OPERAND; -#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, &out_len, (OP), (ARG), (OPERAND)) +#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG), (OPERAND)) static inline void -add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, int *out_len, +add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, uint16_t opcode, uint16_t oparg, uintptr_t operand0) { - _PyUOpInstruction *out = &ctx->out_buffer[*out_len]; + _PyUOpInstruction *out = &ctx->out_buffer[ctx->out_len]; out->opcode = (opcode); out->format = this_instr->format; out->oparg = (oparg); out->target = this_instr->target; out->operand0 = (operand0); out->operand1 = this_instr->operand1; - (*out_len)++; + ctx->out_len++; } /* Shortened forms for convenience, used in optimizer_bytecodes.c */ @@ -242,13 +242,12 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - int *out_len, JitOptRef value, JitOptRef *result_ptr, bool insert_mode) { if (sym_matches_type(value, &PyBool_Type)) { - add_op(ctx, this_instr, out_len, _NOP, 0, 0); + ADD_OP(_NOP, 0, 0); *result_ptr = value; return 1; } @@ -258,7 +257,7 @@ optimize_to_bool( int opcode = insert_mode ? _INSERT_1_LOAD_CONST_INLINE_BORROW : _POP_TOP_LOAD_CONST_INLINE_BORROW; - add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)load); + ADD_OP(opcode, 0, (uintptr_t)load); *result_ptr = sym_new_const(ctx, load); return 1; } @@ -266,9 +265,9 @@ optimize_to_bool( } static void -eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit) +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit) { - add_op(ctx, this_instr, out_len, _POP_TOP, 0, 0); + ADD_OP(_POP_TOP, 0, 0); if (exit) { REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; @@ -277,7 +276,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_ static JitOptRef lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction *this_instr, - int *out_len, PyTypeObject *type, PyObject *name, uint16_t immortal, + PyTypeObject *type, PyObject *name, uint16_t immortal, uint16_t mortal) { // The cached value may be dead, so we need to do the lookup again... :( @@ -285,7 +284,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction PyObject *lookup = _PyType_Lookup(type, name); if (lookup) { int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; - add_op(ctx, this_instr, out_len, opcode, 0, (uintptr_t)lookup); + ADD_OP(opcode, 0, (uintptr_t)lookup); PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); return sym_new_const(ctx, lookup); @@ -395,7 +394,9 @@ optimize_uops( frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; - int out_len = 0; + + int *cur_len = &ctx->out_len; + *cur_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -428,8 +429,8 @@ optimize_uops( Py_UNREACHABLE(); } // If no ADD_OP was called during this iteration, copy the original instruction - if (out_len == i) { - ctx->out_buffer[out_len++] = *this_instr; + if (*cur_len == i) { + ctx->out_buffer[(*cur_len)++] = *this_instr; } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -460,12 +461,12 @@ optimize_uops( * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); // Check that the trace ends with a proper terminator - if (out_len > 0) { - _PyUOpInstruction *last_uop = &ctx->out_buffer[out_len - 1]; + if (*cur_len > 0) { + _PyUOpInstruction *last_uop = &ctx->out_buffer[*cur_len - 1]; if (!is_terminator_uop(last_uop)) { // Copy remaining uops from original trace until we find a terminator - for (int i = out_len; i < trace_len; i++) { - ctx->out_buffer[out_len++] = trace[i]; + for (int i = *cur_len; i < trace_len; i++) { + ctx->out_buffer[(*cur_len)++] = trace[i]; if (is_terminator_uop(&trace[i])) { break; } @@ -473,7 +474,7 @@ optimize_uops( } } - return out_len; + return *cur_len; error: DPRINTF(3, "\n"); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6b1bfaa5bad457..91b72e4d7665ff 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -43,13 +43,12 @@ extern int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - int *out_len, JitOptSymbol *value, JitOptSymbol **result_ptr, bool insert_mode); extern void -eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, int *out_len, bool exit); +eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool exit); extern PyCodeObject *get_code(_PyUOpInstruction *op); @@ -58,7 +57,6 @@ dummy_func(void) { PyCodeObject *co; int oparg; - int out_len; JitOptSymbol *flag; JitOptSymbol *left; JitOptSymbol *right; @@ -389,21 +387,21 @@ dummy_func(void) { } op(_TO_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- value)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &value, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &value, false); if (!already_bool) { sym_set_type(value, &PyBool_Type); } } op(_TO_BOOL_INT, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { sym_set_type(value, &PyLong_Type); res = sym_new_truthiness(ctx, value, true); @@ -411,14 +409,14 @@ dummy_func(void) { } op(_TO_BOOL_LIST, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { res = sym_new_type(ctx, &PyBool_Type); } } op(_TO_BOOL_NONE, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { sym_set_const(value, Py_None); res = sym_new_const(ctx, Py_False); @@ -444,7 +442,7 @@ dummy_func(void) { } op(_TO_BOOL_STR, (value -- res, v)) { - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, true); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, true); v = value; if (!already_bool) { res = sym_new_truthiness(ctx, value, true); @@ -707,7 +705,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -716,7 +714,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -725,7 +723,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); } @@ -734,7 +732,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -744,7 +742,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -754,7 +752,7 @@ dummy_func(void) { (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -775,7 +773,7 @@ dummy_func(void) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } @@ -785,7 +783,7 @@ dummy_func(void) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); } @@ -1119,7 +1117,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); } @@ -1164,7 +1162,7 @@ dummy_func(void) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); } @@ -1173,11 +1171,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, &out_len, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); } @@ -1186,11 +1184,11 @@ dummy_func(void) { if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, &out_len, false); + eliminate_pop_guard(this_instr, ctx, false); } } @@ -1525,7 +1523,7 @@ dummy_func(void) { ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->out_buffer[out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f1f5f20fe934d4..8924b6f3db1d22 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -311,7 +311,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { res = sym_new_truthiness(ctx, value, true); } @@ -322,7 +322,7 @@ case _TO_BOOL_BOOL: { JitOptRef value; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &value, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &value, false); if (!already_bool) { sym_set_type(value, &PyBool_Type); } @@ -334,7 +334,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { sym_set_type(value, &PyLong_Type); res = sym_new_truthiness(ctx, value, true); @@ -371,7 +371,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { res = sym_new_type(ctx, &PyBool_Type); } @@ -383,7 +383,7 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, false); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, false); if (!already_bool) { sym_set_const(value, Py_None); res = sym_new_const(ctx, Py_False); @@ -424,7 +424,7 @@ JitOptRef res; JitOptRef v; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, &out_len, value, &res, true); + int already_bool = optimize_to_bool(this_instr, ctx, value, &res, true); v = value; if (!already_bool) { res = sym_new_truthiness(ctx, value, true); @@ -1530,7 +1530,7 @@ ctx->frame->globals_watched = true; } if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { - REPLACE_OP(&ctx->out_buffer[out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); + REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1], _GUARD_GLOBALS_VERSION, 0, version); ctx->frame->globals_checked_version = version; } if (ctx->frame->globals_checked_version == version) { @@ -1927,7 +1927,7 @@ (void)descr; PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2693,7 +2693,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2714,7 +2714,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2734,7 +2734,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2749,7 +2749,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _POP_TOP_LOAD_CONST_INLINE_BORROW, _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; @@ -2769,7 +2769,7 @@ (void)descr; PyTypeObject *type = sym_get_type(owner); PyObject *name = get_co_name(ctx, oparg >> 1); - attr = lookup_attr(ctx, dependencies, this_instr, &out_len, type, name, + attr = lookup_attr(ctx, dependencies, this_instr, type, name, _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; @@ -2823,7 +2823,7 @@ if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { assert(PyFunction_Check(sym_get_const(ctx, callable))); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -2841,7 +2841,7 @@ PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - ctx->out_buffer[out_len - 1].operand1 = (uintptr_t)method->im_func; + ctx->out_buffer[ctx->out_len - 1].operand1 = (uintptr_t)method->im_func; } sym_set_type(callable, &PyMethod_Type); break; @@ -3680,7 +3680,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_True); + eliminate_pop_guard(this_instr, ctx, value != Py_True); } sym_set_const(flag, Py_True); CHECK_STACK_BOUNDS(-1); @@ -3695,7 +3695,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, value != Py_False); + eliminate_pop_guard(this_instr, ctx, value != Py_False); } sym_set_const(flag, Py_False); CHECK_STACK_BOUNDS(-1); @@ -3710,11 +3710,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, !Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, &out_len, true); + eliminate_pop_guard(this_instr, ctx, true); } sym_set_const(val, Py_None); CHECK_STACK_BOUNDS(-1); @@ -3729,11 +3729,11 @@ if (sym_is_const(ctx, val)) { PyObject *value = sym_get_const(ctx, val); assert(value != NULL); - eliminate_pop_guard(this_instr, ctx, &out_len, Py_IsNone(value)); + eliminate_pop_guard(this_instr, ctx, Py_IsNone(value)); } else if (sym_has_type(val)) { assert(!sym_matches_type(val, &_PyNone_Type)); - eliminate_pop_guard(this_instr, ctx, &out_len, false); + eliminate_pop_guard(this_instr, ctx, false); } CHECK_STACK_BOUNDS(-1); stack_pointer += -1; From 543971c2312c9e8293f7a4e1a4fd0d0ff58caa31 Mon Sep 17 00:00:00 2001 From: cocolato Date: Tue, 13 Jan 2026 02:51:35 +0800 Subject: [PATCH 9/9] remove cur_len --- Python/optimizer_analysis.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7697312a492c96..2a101df76c8803 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -395,8 +395,7 @@ optimize_uops( ctx->curr_frame_depth++; ctx->frame = frame; - int *cur_len = &ctx->out_len; - *cur_len = 0; + ctx->out_len = 0; _PyUOpInstruction *this_instr = NULL; JitOptRef *stack_pointer = ctx->frame->stack_pointer; @@ -429,8 +428,8 @@ optimize_uops( Py_UNREACHABLE(); } // If no ADD_OP was called during this iteration, copy the original instruction - if (*cur_len == i) { - ctx->out_buffer[(*cur_len)++] = *this_instr; + if (ctx->out_len == i) { + ctx->out_buffer[ctx->out_len++] = *this_instr; } assert(ctx->frame != NULL); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -461,12 +460,12 @@ optimize_uops( * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); // Check that the trace ends with a proper terminator - if (*cur_len > 0) { - _PyUOpInstruction *last_uop = &ctx->out_buffer[*cur_len - 1]; + if (ctx->out_len > 0) { + _PyUOpInstruction *last_uop = &ctx->out_buffer[ctx->out_len - 1]; if (!is_terminator_uop(last_uop)) { // Copy remaining uops from original trace until we find a terminator - for (int i = *cur_len; i < trace_len; i++) { - ctx->out_buffer[(*cur_len)++] = trace[i]; + for (int i = ctx->out_len; i < trace_len; i++) { + ctx->out_buffer[ctx->out_len++] = trace[i]; if (is_terminator_uop(&trace[i])) { break; } @@ -474,7 +473,7 @@ optimize_uops( } } - return *cur_len; + return ctx->out_len; error: DPRINTF(3, "\n");