issue #82 disable some Debug lib functions when debugger is running

include/lstate.h

@@ -149,6 +149,7 @@ typedef struct global_State {
   ravi_Writeline ravi_writeline;
   ravi_Writestring ravi_writestring;
   ravi_Writestringerror ravi_writestringerror;
+  void * ravi_debugger_data;
 } global_State;
 
 

include/lua.h

@@ -512,6 +512,9 @@ LUA_API void ravi_writestring(lua_State *L, const char *s, size_t len);
 LUA_API void ravi_writeline(lua_State *L);
 LUA_API void ravi_writestringerror(lua_State *L, const char *fmt, const char *p);
 
+/* The debugger can set some data - but only once */
+LUA_API void ravi_set_debugger_data(lua_State *L, void *data);
+LUA_API void *ravi_get_debugger_data(lua_State *L);
 
 /////////////////////////////////////////////////////////////////////////////
 /*

src/lapi.c

@@ -1630,6 +1630,18 @@ LUA_API void ravi_set_writefuncs(lua_State *L, ravi_Writestring writestr, ravi_W
   g->ravi_writestringerror = writestringerr;
 }
 
+LUA_API void ravi_set_debugger_data(lua_State *L, void *data) {
+  global_State *g = G(L);
+  /* we only allow this to be set once */
+  if (g->ravi_debugger_data == NULL)
+    g->ravi_debugger_data = data;
+}
+
+LUA_API void *ravi_get_debugger_data(lua_State *L) {
+  global_State *g = G(L);
+  return g->ravi_debugger_data;
+}
+
 LUA_API void ravi_writestring(lua_State *L, const char *s, size_t len) {
   global_State *g = G(L);
   g->ravi_writestring(s, len);

src/ldblib.c

@@ -347,6 +347,9 @@ static char *unmakemask (int mask, char *smask) {
 static int db_sethook (lua_State *L) {
   int arg, mask, count;
   lua_Hook func;
+  /* Ravi change - disallow when debugging via VSCode */
+  if (ravi_get_debugger_data(L) != NULL)
+    return 0;
   lua_State *L1 = getthread(L, &arg);
   if (lua_isnoneornil(L, arg+1)) {  /* no hook? */
     lua_settop(L, arg+1);
@@ -400,6 +403,9 @@ static int db_gethook (lua_State *L) {
 
 
 static int db_debug (lua_State *L) {
+  /* Ravi change - disallow when debugging via VSCode */
+  if (ravi_get_debugger_data(L) != NULL)
+    return 0;
   for (;;) {
     char buffer[250];
     lua_writestringerror("%s", "lua_debug> ");

src/ldebug.c

@@ -72,6 +72,10 @@ static void swapextra (lua_State *L) {
 ** this function can be called asynchronous (e.g. during a signal)
 */
 LUA_API void lua_sethook (lua_State *L, lua_Hook func, int mask, int count) {
+  /* Ravi change - disable setting of hook if debugger is running the Lua instance */
+  global_State *G = G(L);
+  if (L->hook != NULL && G->ravi_debugger_data != NULL)
+    return;
   if (func == NULL || mask == 0) {  /* turn off hooks? */
     mask = 0;
     func = NULL;

src/lstate.c

@@ -323,6 +323,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
   g->ravi_writeline = ravi_default_writeline;
   g->ravi_writestring = ravi_default_writestring;
   g->ravi_writestringerror = ravi_default_writestringerror;
+  g->ravi_debugger_data = NULL;
   L->next = NULL;
   L->tt = LUA_TTHREAD;
   g->currentwhite = bitmask(WHITE0BIT);

vscode-debugger/src/ravidebug.c

@@ -664,6 +664,7 @@ int main(void) {
   luaL_openlibs(L); /* open standard libraries */
   lua_sethook(L, ravi_debughook, LUA_MASKCALL | LUA_MASKLINE | LUA_MASKRET, 0);
   debugger_state = DEBUGGER_BIRTH;
+  ravi_set_debugger_data(L, &debugger_state); /* This is useless data right now but it ensures that the hook cannot be removed */
   debugger(L, NULL, stdin, stdout);
   lua_close(L);
   fclose(my_logger);