Handle all stack layouts in (delayed) TRACE vmevent.

Thanks to Sergey Bronnikov and Peter Cawley.

(cherry picked from commit b138ccf)

When recording a trace using a Lua code like
`repeat until `a >= 'b' > 'b'` a Lua error is encountered
(`attempt to compare string with nil`), which (along with
raising the error) causes an asynchronous trace abort.
The trace abort remains pending until the call of
`lua_pcall(L, 2, 0, 0)` that actually calls `jit.attach(trace_cb, nil)`.

On handling abort LuaJIT is searching for the topmost Lua frame
on the stack, that was active when the trace abort happened,
it is needed to generate a better error message.
Unfortunately, because the abort was due to an error, and
the error was caught by a `lua_pcall` with unspecified
error function (4th argument), the Lua frame that caused the abort
was already removed as part of error processing, so the search
cannot find it. Furthermore, in this particular case, there are
no Lua frames on the stack, which isn't something the search code
had considered possible.

lj_ccallback_leave

Sergey Bronnikov:
* added the description and the test for the problem

Part of tarantool/tarantool#9145

src/lj_trace.c

@@ -616,21 +616,27 @@ static int trace_abort(jit_State *J)
     J->cur.link = 0;
     J->cur.linktype = LJ_TRLINK_NONE;
     lj_vmevent_send(L, TRACE,
-      TValue *frame;
+      cTValue *bot = tvref(L->stack)+LJ_FR2;
+      cTValue *frame;
       const BCIns *pc;
-      GCfunc *fn;
+      BCPos pos = 0;
       setstrV(L, L->top++, lj_str_newlit(L, "abort"));
       setintV(L->top++, traceno);
       /* Find original Lua function call to generate a better error message. */
-      frame = J->L->base-1;
-      pc = J->pc;
-      while (!isluafunc(frame_func(frame))) {
-	pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
-	frame = frame_prev(frame);
+      for (frame = J->L->base-1, pc = J->pc; ; frame = frame_prev(frame)) {
+	if (isluafunc(frame_func(frame))) {
+	  pos = proto_bcpos(funcproto(frame_func(frame)), pc);
+	  break;
+	} else if (frame_prev(frame) <= bot) {
+	  break;
+	} else if (frame_iscont(frame)) {
+	  pc = frame_contpc(frame) - 1;
+	} else {
+	  pc = frame_pc(frame) - 1;
+	}
       }
-      fn = frame_func(frame);
-      setfuncV(L, L->top++, fn);
-      setintV(L->top++, proto_bcpos(funcproto(fn), pc));
+      setfuncV(L, L->top++, frame_func(frame));
+      setintV(L->top++, pos);
       copyTV(L, L->top++, restorestack(L, errobj));
       copyTV(L, L->top++, &J->errinfo);
     );

test/tarantool-c-tests/CMakeLists.txt

@@ -36,6 +36,10 @@ add_test_suite_target(tarantool-c-tests
   DEPENDS libluajit libtest tarantool-c-tests-build
 )
 
+if(NOT LUAJIT_DISABLE_JIT)
+  AppendFlags(TESTS_C_FLAGS "-DLJ_HASJIT")
+endif(NOT LUAJIT_DISABLE_JIT)
+
 set(CTEST_SRC_SUFFIX ".test.c")
 file(GLOB tests "${CMAKE_CURRENT_SOURCE_DIR}/*${CTEST_SRC_SUFFIX}")
 foreach(test_source ${tests})

test/tarantool-c-tests/lj-1087-vm-handler-call.test.c

@@ -0,0 +1,174 @@
+#include <assert.h>
+#include <stdbool.h>
+
+#include "lua.h"
+#include "lauxlib.h"
+
+#include "test.h"
+#include "utils.h"
+
+#define UNUSED(x)       ((void)(x))     /* to avoid warnings */
+
+/*
+ * Test file to demonstrate a segmentation fault under
+ * AddressSanitizer, when C function is used as a VM handler
+ * in LuaJIT:
+ *
+ * Program received signal SIGSEGV, Segmentation fault.
+ * 0x000055555557e77d in trace_abort (J=0x7ffff7f9b6b8) at lj_trace.c:615
+ * 615         lj_vmevent_send(L, TRACE,
+ * (gdb) bt
+ *
+ * See details in https://github.com/LuaJIT/LuaJIT/issues/1087.
+ */
+
+int is_cb_called = 0;
+
+static void jit_attach(lua_State *L, void *cb, const char *event)
+{
+	lua_getglobal(L, "jit");
+	lua_getfield(L, -1, "attach");
+	lua_pushcfunction(L, (lua_CFunction)cb);
+	if (event != NULL) {
+		lua_pushstring(L, event);
+	} else {
+		lua_pushnil(L);
+	}
+	lua_pcall(L, 2, 0, 0);
+}
+
+static int trace_cb(lua_State *L) {
+	(void)L;
+	is_cb_called = 1;
+	return 0;
+}
+
+static int handle_luafunc_frame(void *test_state)
+{
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	/* Loading and executing of a broken Lua code. */
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	assert(rc == 1);
+
+	/* Generates a Lua frame. */
+	rc = luaL_dostring(L, "return function() end");
+	assert(rc == 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int nop(lua_State *L)
+{
+	return 0;
+}
+
+static int cframe(lua_State *L)
+{
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	assert(rc == 1);
+	lua_pop(L, 1);		/* Remove errmsg. */
+
+	lua_pushcfunction(L, nop);
+	lua_call(L, 0, 0);
+
+	return 0;
+}
+
+static int handle_c_frame(void *test_state)
+{
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	lua_pushcfunction(L, cframe);
+	lua_call(L, 0, 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int global_f(lua_State* L)
+{
+	/* int rc = luaL_dostring(L, "repeat until b > 1"); */
+	/* UNUSED(rc); */
+	lua_pushstring(L, "");
+	lua_pushstring(L, "");
+	lua_concat(L, 2);
+
+    return 1;
+}
+
+static int handle_cont_frame(void *test_state)
+{
+	/* Setup. */
+	lua_State *L = test_state;
+	jit_attach(L, (void *)trace_cb, "trace");
+	int res = luaL_dostring(L, "jit.opt.start('minstitch=34952')");
+	assert(res == 0);
+	lua_pushcfunction(L, global_f);
+	lua_setglobal(L, "global_f");
+
+	res = luaL_loadstring(L, " \
+local t = setmetatable({}, { __index = global_f }) \
+local r \
+local i = 1 \
+while i <= 4 do \
+  i = i + 1 \
+  r = t[1] \
+  if i == 4 then assert(nil) end \
+end");
+	assert(res == 0);
+	lua_pcall(L, 0, LUA_MULTRET, 0);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+static int handle_bottom_frame(void *test_state)
+{
+	lua_State *L = test_state;
+
+	/* Attach VM call handler. */
+	jit_attach(L, (void *)trace_cb, "trace");
+
+	/* Load a Lua code that generate a trace abort. */
+	int rc = luaL_dostring(L, "repeat until nil > 1");
+	UNUSED(rc);
+
+	/* Triggers segmentation fault. */
+	jit_attach(L, (void *)trace_cb, NULL);
+
+	/* Teardown. */
+	lua_settop(L, 0);
+
+	return TEST_EXIT_SUCCESS;
+}
+
+int main(void)
+{
+	lua_State *L = utils_lua_init();
+	const struct test_unit tgroup[] = {
+#ifdef LJ_HASJIT
+		test_unit_def(handle_luafunc_frame),
+		test_unit_def(handle_bottom_frame),
+		test_unit_def(handle_cont_frame),
+		test_unit_def(handle_c_frame),
+#endif /* LJ_HASJIT */
+	};
+	luaL_openlibs(L);
+	int res = luaL_dostring(L, "jit.opt.start('hotloop=1')");
+	assert(res == 0);
+	const int test_result = test_run_group(tgroup, L);
+	utils_lua_close(L);
+	return test_result;
+}