New feature: Override player input with machine learning models (#17407)

* Add dummy game ai subsystem

* First working prototype of a machine learning model that can override player input

* Update README.md

* Update README.md

* Fix loading path on Windows

* Change ai override to player 2

* Added quick menu show game ai option

* Implemented Quick Menu entry for Game AI options

* Redirect debug logs to retroarch log system + properly support player override

* Added support to use framebuffer as input to the AI

* Added pixel format parameter to API

* Fix game name

* code clean-up of game_ai.cpp

* Update README.md - Windows Build

* Update README.md

* Update README.md

* Update README.md

* Update config.params.sh

turn off GAME_AI feature by default

* Fix compile error in menu_displaylist.c

* Add missing #define in menu_cbs_title.c

* Added new game_ai entry in griffin_cpp

* Remove GAME_AI entry in  msg_hash_us.c

* Fix compile error in menu_displaylist.h

* Removed GAME AI references from README.md

* Fixes coding style + add GameAI lib API header

* Convert comment to legacy + remove unused code

* Additional coding style fixes to game_ai.cpp

* Fix identation issues in game_ai.cpp

* Removed some debug code in game_ai.cpp

* Add game_ai_lib in deps

* Replace assert with retro_assert

* Update Makefile.common

* Converting game_ai from cpp to c. First step.

* Convert game_ai from CPP to C. STEP 2: add C function calls

* Convert game_ai from CPP to C. Final Step

* Added shutdown function for game ai lib

* Update game_ai_lib README

* Fix crash when loading/unloading multiple games

Makefile.common

@@ -2648,6 +2648,11 @@ ifeq ($(HAVE_ODROIDGO2), 1)
          gfx/drivers/oga_gfx.o
 endif
 
+ifeq ($(HAVE_GAME_AI),1)
+   DEFINES += -DHAVE_GAME_AI
+   OBJ += ai/game_ai.o
+endif
+
 # Detect the operating system
 UNAME := $(shell uname -s)
 
@@ -2673,7 +2678,6 @@ else
 $(warning Windows NT version macro (_WIN32_WINNT) is not defined.)
 endif
 
-
 endif
 
 ##################################

ai/game_ai.c

@@ -0,0 +1,220 @@
+#include "game_ai.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include <retro_assert.h>
+
+#include "../deps/game_ai_lib/GameAI.h"
+
+#define GAME_AI_MAX_PLAYERS 2
+
+void *                     ga = NULL;
+volatile void *            g_ram_ptr = NULL;
+volatile int               g_ram_size = 0;
+volatile signed short int  g_buttons_bits[GAME_AI_MAX_PLAYERS] = {0};
+volatile int               g_frameCount = 0;
+volatile char              game_ai_lib_path[1024] = {0};
+volatile char              g_game_name[1024] = {0};
+retro_log_printf_t         g_log = NULL;
+
+#ifdef _WIN32
+HINSTANCE                  g_lib_handle = NULL;
+#else
+void *                     g_lib_handle = NULL;
+#endif
+
+/* GameAI Lib API*/
+create_game_ai_t              create_game_ai = NULL;
+destroy_game_ai_t             destroy_game_ai = NULL;
+game_ai_lib_init_t            game_ai_lib_init = NULL;
+game_ai_lib_think_t           game_ai_lib_think = NULL;
+game_ai_lib_set_show_debug_t  game_ai_lib_set_show_debug = NULL;
+game_ai_lib_set_debug_log_t   game_ai_lib_set_debug_log = NULL;
+
+/* Helper functions */
+void game_ai_debug_log(int level, const char *fmt, ...)
+{
+   va_list vp;
+   va_start(vp, fmt);
+
+   if (g_log)
+      g_log((enum retro_log_level)level, fmt, vp);
+
+   va_end(vp);
+}
+
+void array_to_bits_16(volatile signed short * result, const bool b[16])
+{
+   for (int bit = 0; bit <= 15; bit++)
+   {
+      *result |= b[bit] ? (1 << bit) : 0;
+   }
+}
+
+/* Interface to RA */
+
+extern signed short int game_ai_input(unsigned int port, unsigned int device, unsigned int idx, unsigned int id, signed short int result)
+{
+   if (ga == NULL)
+      return 0;
+
+   if (port < GAME_AI_MAX_PLAYERS)
+      return g_buttons_bits[port];
+
+   return 0;
+}
+
+extern void game_ai_init()
+{
+   if (create_game_ai == NULL)
+   {
+#ifdef _WIN32
+   BOOL fFreeResult, fRunTimeLinkSuccess = FALSE;
+
+   g_lib_handle = LoadLibrary(TEXT("game_ai.dll"));
+   retro_assert(hinstLib);
+
+   char full_module_path[MAX_PATH];
+   DWORD dwLen = GetModuleFileNameA(g_lib_handle, static_cast<char*>(&full_module_path), MAX_PATH);
+
+   if (hinstLib != NULL)
+   {
+      create_game_ai = (create_game_ai_t) GetProcAddress(hinstLib, "create_game_ai");
+      retro_assert(create_game_ai);
+
+      destroy_game_ai = (destroy_game_ai_t) GetProcAddress(hinstLib, "destroy_game_ai");
+      retro_assert(destroy_game_ai);
+
+      game_ai_lib_init = (game_ai_lib_init_t) GetProcAddress(hinstLib, "game_ai_lib_init");
+      retro_assert(game_ai_lib_init);
+
+      game_ai_lib_think = (game_ai_lib_think_t) GetProcAddress(hinstLib, "game_ai_lib_think");
+      retro_assert(game_ai_lib_think);
+
+      game_ai_lib_set_show_debug = (game_ai_lib_set_show_debug_t) GetProcAddress(hinstLib, "game_ai_lib_set_show_debug");
+      retro_assert(game_ai_lib_set_show_debug);
+
+      game_ai_lib_set_debug_log = (game_ai_lib_set_debug_log_t) GetProcAddress(hinstLib, "game_ai_lib_set_debug_log");
+      retro_assert(game_ai_lib_set_debug_log);
+   }
+#else
+      g_lib_handle = dlopen("libgame_ai.so", RTLD_NOW);
+      retro_assert(g_lib_handle);
+
+      if(g_lib_handle != NULL)
+      {
+         dlinfo(g_lib_handle, RTLD_DI_ORIGIN, (void *) &game_ai_lib_path);
+
+         create_game_ai = (create_game_ai_t)(dlsym(g_lib_handle, "create_game_ai"));
+         retro_assert(create_game_ai);
+
+         destroy_game_ai = (destroy_game_ai_t)(dlsym(g_lib_handle, "destroy_game_ai"));
+         retro_assert(destroy_game_ai);
+
+         game_ai_lib_init = (game_ai_lib_init_t)(dlsym(g_lib_handle, "game_ai_lib_init"));
+         retro_assert(game_ai_lib_init);
+
+         game_ai_lib_think = (game_ai_lib_think_t)(dlsym(g_lib_handle, "game_ai_lib_think"));
+         retro_assert(game_ai_lib_think);
+
+         game_ai_lib_set_show_debug = (game_ai_lib_set_show_debug_t)(dlsym(g_lib_handle, "game_ai_lib_set_show_debug"));
+         retro_assert(game_ai_lib_set_show_debug);
+
+         game_ai_lib_set_debug_log  = (game_ai_lib_set_debug_log_t)(dlsym(g_lib_handle, "game_ai_lib_set_debug_log"));
+         retro_assert(game_ai_lib_set_debug_log);
+      }
+#endif
+   }
+}
+
+extern void game_ai_shutdown()
+{
+   if (g_lib_handle)
+   {
+      if (ga)
+      {
+         destroy_game_ai(ga);
+         ga = NULL;
+      }
+#ifdef _WIN32
+      FreeLibrary(g_lib_handle);
+#else
+      dlclose(g_lib_handle);
+#endif
+   }
+}
+
+extern void game_ai_load(const char * name, void * ram_ptr, int ram_size, retro_log_printf_t log)
+{
+   strcpy((char *) &g_game_name[0], name);
+
+   g_ram_ptr = ram_ptr;
+   g_ram_size = ram_size;
+
+   g_log = log;
+
+   if (ga)
+   {
+      destroy_game_ai(ga);
+      ga = NULL;
+   }
+}
+
+extern void game_ai_think(bool override_p1, bool override_p2, bool show_debug, const void *frame_data, unsigned int frame_width, unsigned int frame_height, unsigned int frame_pitch, unsigned int pixel_format)
+{
+   if (ga)
+      game_ai_lib_set_show_debug(ga, show_debug);
+
+   if (ga == NULL && g_ram_ptr != NULL)
+   {
+      ga = create_game_ai((char *) &g_game_name[0]);
+      retro_assert(ga);
+
+      if (ga)
+      {
+         char data_path[1024] = {0};
+         strcpy(&data_path[0], (char *)game_ai_lib_path);
+         strcat(&data_path[0], "/data/");
+         strcat(&data_path[0], (char *)g_game_name);
+
+         game_ai_lib_init(ga, (void *) g_ram_ptr, g_ram_size);
+         game_ai_lib_set_debug_log(ga, game_ai_debug_log);
+      }
+   }
+
+   if (g_frameCount >= (GAMEAI_SKIPFRAMES - 1))
+   {
+      if (ga)
+      {
+         bool b[GAMEAI_MAX_BUTTONS] = {0};
+
+         g_buttons_bits[0]=0;
+         g_buttons_bits[1]=0;
+
+         if (override_p1)
+         {
+            game_ai_lib_think(ga, b, 0, frame_data, frame_width, frame_height, frame_pitch, pixel_format);
+            array_to_bits_16(&g_buttons_bits[0], b);
+         }
+
+         if (override_p2)
+         {
+            game_ai_lib_think(ga, b, 1, frame_data, frame_width, frame_height, frame_pitch, pixel_format);
+            array_to_bits_16(&g_buttons_bits[1], b);
+         }
+      }
+      g_frameCount=0;
+   }
+   else
+   {
+      g_frameCount++;
+   }
+}

ai/game_ai.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <libretro.h>
+
+signed short int game_ai_input(unsigned int port, unsigned int device, unsigned int idx, unsigned int id, signed short int result);
+void game_ai_init();
+void game_ai_shutdown();
+void game_ai_load(const char * name, void * ram_ptr, int ram_size, retro_log_printf_t log);
+void game_ai_think(bool override_p1, bool override_p2, bool show_debug, const void *frame_data, unsigned int frame_width, unsigned int frame_height, unsigned int frame_pitch, unsigned int pixel_format);

configuration.c

@@ -2227,6 +2227,10 @@ static struct config_bool_setting *populate_settings_bool(
    SETTING_BOOL("gcdwebserver_alert",            &settings->bools.gcdwebserver_alert, true, true, false);
 #endif
 
+#ifdef HAVE_GAME_AI
+   SETTING_BOOL("quick_menu_show_game_ai",  &settings->bools.quick_menu_show_game_ai, true, 1, false);
+#endif
+
    *size = count;
 
    return tmp;

configuration.h

@@ -1095,6 +1095,14 @@ typedef struct settings
 #if defined(HAVE_COCOATOUCH)
       bool gcdwebserver_alert;
 #endif
+
+#ifdef HAVE_GAME_AI
+      bool quick_menu_show_game_ai;
+      bool game_ai_override_p1;
+      bool game_ai_override_p2;
+      bool game_ai_show_debug;
+#endif
+
    } bools;
 
    uint8_t flags;

deps/game_ai_lib/.gitignore

@@ -0,0 +1,8 @@
+libtorch/
+build/
+CMakeFiles/
+Debug/
+libtorch/
+win/
+*.zip
+.vscode/

deps/game_ai_lib/CMakeLists.txt

@@ -0,0 +1,25 @@
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+project(custom_ops)
+
+find_package(Torch REQUIRED)
+find_package(OpenCV REQUIRED)
+include_directories(${OpenCV_INCLUDE_DIRS})
+
+add_executable(test test.cpp RetroModel.cpp)
+target_link_libraries(test "${TORCH_LIBRARIES}" ${OpenCV_LIBS})
+
+add_library(game_ai SHARED GameAILocal.cpp RetroModel.cpp games/NHL94GameAI.cpp games/NHL94GameData.cpp games/DefaultGameAI.cpp utils/data.cpp utils/memory.cpp utils/utils.cpp)
+
+target_link_libraries(game_ai "${TORCH_LIBRARIES}" ${OpenCV_LIBS})
+
+set_property(TARGET test PROPERTY CXX_STANDARD 17)
+set_property(TARGET game_ai PROPERTY CXX_STANDARD 17)
+
+if (MSVC)
+  file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll")
+  add_custom_command(TARGET test
+                     POST_BUILD
+                     COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                     ${TORCH_DLLS}
+                     $<TARGET_FILE_DIR:test>)
+endif (MSVC)

deps/game_ai_lib/GameAI.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#ifdef __cplusplus
+
+#include <bitset>
+#include <string>
+#include <filesystem>
+#include <vector>
+#include <queue>
+
+#endif
+
+
+typedef void (*debug_log_t)(int level, const char *fmt, ...);
+
+#define GAMEAI_MAX_BUTTONS 16
+#define GAMEAI_SKIPFRAMES 4
+
+
+#ifdef __cplusplus
+
+class GameAI {
+public:
+        virtual void    Init(void * ram_ptr, int ram_size) {};
+        virtual void    Think(bool buttons[GAMEAI_MAX_BUTTONS], int player, const void *frame_data, unsigned int frame_width, unsigned int frame_height, unsigned int frame_pitch, unsigned int pixel_format) {};
+        void            SetShowDebug(const bool show){ this->showDebug = show; };
+        void            SetDebugLog(debug_log_t func){debugLogFunc = func;};
+
+private:
+        bool            showDebug;
+        debug_log_t     debugLogFunc;
+};
+
+#endif
+
+typedef void * (*create_game_ai_t)(const char *);
+typedef void (*destroy_game_ai_t)(void * obj_ptr);
+typedef void (*game_ai_lib_init_t)(void * obj_ptr, void * ram_ptr, int ram_size);
+typedef void (*game_ai_lib_think_t)(void * obj_ptr, bool buttons[GAMEAI_MAX_BUTTONS], int player, const void *frame_data, unsigned int frame_width, unsigned int frame_height, unsigned int frame_pitch, unsigned int pixel_format);
+typedef void (*game_ai_lib_set_show_debug_t)(void * obj_ptr, const bool show);
+typedef void (*game_ai_lib_set_debug_log_t)(void * obj_ptr, debug_log_t func);