Adds new module API test to test the scripting engine module API

This commit adds a module with a very simple stack based scripting
language implementation to test the new module API that allows to
implement new scripting engines as modules.

Signed-off-by: Ricardo Dias <[email protected]>

tests/modules/Makefile

@@ -64,7 +64,8 @@ TEST_MODULES = \
     moduleauthtwo.so \
     rdbloadsave.so \
     crash.so \
-    cluster.so
+    cluster.so \
+    helloscripting.so
 
 .PHONY: all
 

tests/modules/helloscripting.c

@@ -0,0 +1,276 @@
+#include "valkeymodule.h"
+
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+
+
+typedef enum HelloInstKind {
+    FUNCTION = 0,
+    CONSTI,
+    ARGS,
+    RETURN,
+    _END,
+} HelloInstKind;
+
+const char *HelloInstKindStr[] = {
+    "FUNCTION",
+    "CONSTI",
+    "ARGS",
+    "RETURN",
+};
+
+typedef struct HelloInst {
+    HelloInstKind kind;
+    union {
+        uint32_t integer;
+        const char *string;
+    } param;
+} HelloInst;
+
+typedef struct HelloFunc {
+    char *name;
+    HelloInst instructions[256];
+    uint32_t num_instructions;
+} HelloFunc;
+
+typedef struct HelloProgram {
+    HelloFunc *functions[16];
+    uint32_t num_functions;
+} HelloProgram;
+
+typedef struct HelloLangCtx {
+    HelloProgram *program;
+} HelloLangCtx;
+
+
+static HelloLangCtx *hello_ctx = NULL;
+
+
+static HelloInstKind helloLangParseInstruction(const char *token) {
+    for (HelloInstKind i = 0; i < _END; i++) {
+        if (strcmp(HelloInstKindStr[i], token) == 0) {
+            return i;
+        }
+    }
+    return _END;
+}
+
+static void helloLangParseFunction(HelloFunc *func) {
+    char *token = strtok(NULL, " \n");
+    ValkeyModule_Assert(token != NULL);
+    func->name = ValkeyModule_Alloc(sizeof(char) * strlen(token) + 1);
+    strcpy(func->name, token);
+}
+
+static uint32_t str2int(const char *str) {
+    char *end;
+    errno = 0;
+    uint32_t val = (uint32_t)strtoul(str, &end, 10);
+    ValkeyModule_Assert(errno == 0);
+    return val;
+}
+
+static void helloLangParseIntegerParam(HelloFunc *func) {
+    char *token = strtok(NULL, " \n");
+    func->instructions[func->num_instructions].param.integer = str2int(token);
+}
+
+static void helloLangParseConstI(HelloFunc *func) {
+    helloLangParseIntegerParam(func);
+    func->num_instructions++;
+}
+
+static void helloLangParseArgs(HelloFunc *func) {
+    helloLangParseIntegerParam(func);
+    func->num_instructions++;
+}
+
+static HelloProgram *helloLangParseCode(const char *code, HelloProgram *program) {
+    char *_code = ValkeyModule_Alloc(sizeof(char) * strlen(code) + 1);
+    strcpy(_code, code);
+
+    HelloFunc *currentFunc = NULL;
+
+    char *token = strtok(_code, " \n");
+    while (token != NULL) {
+        HelloInstKind kind = helloLangParseInstruction(token);
+
+        if (currentFunc != NULL) {
+            currentFunc->instructions[currentFunc->num_instructions].kind = kind;
+        }
+
+        switch (kind) {
+        case FUNCTION:
+            ValkeyModule_Assert(currentFunc == NULL);
+            currentFunc = ValkeyModule_Alloc(sizeof(HelloFunc));
+            program->functions[program->num_functions++] = currentFunc;
+            helloLangParseFunction(currentFunc);
+            break;
+        case CONSTI:
+            ValkeyModule_Assert(currentFunc != NULL);
+            helloLangParseConstI(currentFunc);
+            break;
+        case ARGS:
+            ValkeyModule_Assert(currentFunc != NULL);
+            helloLangParseArgs(currentFunc);
+            break;
+        case RETURN:
+            ValkeyModule_Assert(currentFunc != NULL);
+            currentFunc->num_instructions++;
+            currentFunc = NULL;
+            break;
+        case _END:
+            ValkeyModule_Assert(0);
+        }
+
+        token = strtok(NULL, " \n");
+    }
+
+    ValkeyModule_Free(_code);
+
+    return program;
+}
+
+static uint32_t executeHelloLangFunction(HelloFunc *func, ValkeyModuleString **args, int nargs) {
+    uint32_t stack[64];
+    int sp = 0;
+
+    for (uint32_t pc = 0; pc < func->num_instructions; pc++) {
+        HelloInst instr = func->instructions[pc];
+        switch (instr.kind) {
+        case CONSTI:
+            stack[sp++] = instr.param.integer;
+            break;
+        case ARGS:
+            uint32_t idx = instr.param.integer;
+            ValkeyModule_Assert(idx < (uint32_t)nargs);
+            size_t len;
+            const char *argStr = ValkeyModule_StringPtrLen(args[idx], &len);
+            uint32_t arg = str2int(argStr);
+            stack[sp++] = arg;
+            break;
+        case RETURN:
+            uint32_t val = stack[--sp];
+            ValkeyModule_Assert(sp == 0);
+            return val;
+        case FUNCTION:
+        case _END:
+            ValkeyModule_Assert(0);
+        }
+    }
+
+    ValkeyModule_Assert(0);
+    return 0;
+}
+
+static size_t engineGetUsedMemoy(void *engine_ctx) {
+    VALKEYMODULE_NOT_USED(engine_ctx);
+    return 0;
+}
+
+static size_t engineMemoryOverhead(void *engine_ctx) {
+    HelloLangCtx *ctx = (HelloLangCtx *)engine_ctx;
+    size_t overhead = ValkeyModule_MallocSize(engine_ctx);
+    if (ctx->program != NULL) {
+        overhead += ValkeyModule_MallocSize(ctx->program);
+    }
+    return overhead;
+}
+
+static size_t engineFunctionMemoryOverhead(void *compiled_function) {
+    HelloFunc *func = (HelloFunc *)compiled_function;
+    return ValkeyModule_MallocSize(func->name);
+}
+
+static void engineFreeFunction(void *engine_ctx, void *compiled_function) {
+    VALKEYMODULE_NOT_USED(engine_ctx);
+    HelloFunc *func = (HelloFunc *)compiled_function;
+    ValkeyModule_Free(func->name);
+    func->name = NULL;
+    ValkeyModule_Free(func);
+}
+
+static int createHelloLangEngine(void *engine_ctx, ValkeyModuleScriptingEngineFunctionLibrary *li, const char *code, size_t timeout, char **err) {
+    VALKEYMODULE_NOT_USED(timeout);
+
+    HelloLangCtx *ctx = (HelloLangCtx *)engine_ctx;
+
+    if (ctx->program == NULL) {
+        ctx->program = ValkeyModule_Alloc(sizeof(HelloProgram));
+        memset(ctx->program, 0, sizeof(HelloProgram));
+    } else {
+        ctx->program->num_functions = 0;
+    }
+
+    ctx->program = helloLangParseCode(code, ctx->program);
+
+    for (uint32_t i = 0; i < ctx->program->num_functions; i++) {
+        HelloFunc *func = ctx->program->functions[i];
+        int ret = ValkeyModule_RegisterScriptingEngineFunction(func->name, func, li, NULL, 0, err);
+        if (ret != 0) {
+            // We need to cleanup all parsed functions that were not registered.
+            for (uint32_t j=i; j < ctx->program->num_functions; j++) {
+                engineFreeFunction(NULL, ctx->program->functions[j]);
+            }
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void callHelloLangFunction(ValkeyModuleScriptingEngineFunctionCallCtx *func_ctx,
+                                  void *engine_ctx,
+                                  void *compiled_function,
+                                  ValkeyModuleString **keys,
+                                  size_t nkeys,
+                                  ValkeyModuleString **args,
+                                  size_t nargs) {
+    VALKEYMODULE_NOT_USED(engine_ctx);
+    VALKEYMODULE_NOT_USED(keys);
+    VALKEYMODULE_NOT_USED(nkeys);
+
+    ValkeyModuleCtx *ctx = ValkeyModule_GetModuleCtxFromFunctionCallCtx(func_ctx);
+
+    HelloFunc *func = (HelloFunc *)compiled_function;
+    uint32_t result = executeHelloLangFunction(func, args, nargs);
+
+    ValkeyModule_ReplyWithLongLong(ctx, result);
+}
+
+int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc) {
+    VALKEYMODULE_NOT_USED(argv);
+    VALKEYMODULE_NOT_USED(argc);
+
+    if (ValkeyModule_Init(ctx, "helloengine", 1, VALKEYMODULE_APIVER_1) == VALKEYMODULE_ERR) return VALKEYMODULE_ERR;
+
+    hello_ctx = ValkeyModule_Alloc(sizeof(HelloLangCtx));
+    hello_ctx->program = NULL;
+
+    ValkeyModule_RegisterScriptingEngine(ctx,
+                                         "HELLO",
+                                         hello_ctx,
+                                         createHelloLangEngine,
+                                         callHelloLangFunction,
+                                         engineGetUsedMemoy,
+                                         engineFunctionMemoryOverhead,
+                                         engineMemoryOverhead,
+                                         engineFreeFunction);
+
+    return VALKEYMODULE_OK;
+}
+
+int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) {
+    if (ValkeyModule_UnregisterScriptingEngine(ctx, "HELLO") != VALKEYMODULE_OK) {
+        ValkeyModule_Log(ctx, "error", "Failed to unregister engine");
+        return VALKEYMODULE_ERR;
+    }
+
+    ValkeyModule_Free(hello_ctx->program);
+    hello_ctx->program = NULL;
+    ValkeyModule_Free(hello_ctx);
+    hello_ctx = NULL;
+
+    return VALKEYMODULE_OK;
+}

tests/unit/functions.tcl

@@ -604,7 +604,7 @@ start_server {tags {"scripting"}} {
             }
         } e
         set _ $e
-    } {*Library names can only contain letters, numbers, or underscores(_) and must be at least one character long*}
+    } {*Function names can only contain letters, numbers, or underscores(_) and must be at least one character long*}
 
     test {LIBRARIES - test registration with empty name} {
         catch {
@@ -613,7 +613,7 @@ start_server {tags {"scripting"}} {
             }
         } e
         set _ $e
-    } {*Library names can only contain letters, numbers, or underscores(_) and must be at least one character long*}
+    } {*Function names can only contain letters, numbers, or underscores(_) and must be at least one character long*}
 
     test {LIBRARIES - math.random from function load} {
         catch {

tests/unit/moduleapi/scriptingengine.tcl

@@ -0,0 +1,82 @@
+set testmodule [file normalize tests/modules/helloscripting.so]
+
+set HELLO_PROGRAM "#!hello name=mylib\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION bar\nCONSTI 432\nRETURN"
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    r function load $HELLO_PROGRAM
+
+    test {Load script with invalid library name} {
+        assert_error {ERR Library names can only contain letters, numbers, or underscores(_) and must be at least one character long} {r function load "#!hello name=my-lib\nFUNCTION foo\nARGS 0\nRETURN"}
+    }
+
+    test {Load script with existing library} {
+        assert_error {ERR Library 'mylib' already exists} {r function load $HELLO_PROGRAM}
+    }
+
+    test {Load script with invalid engine} {
+        assert_error {ERR Engine 'wasm' not found} {r function load "#!wasm name=mylib2\nFUNCTION foo\nARGS 0\nRETURN"}
+    }
+
+    test {Load script with no functions} {
+        assert_error {ERR No functions registered} {r function load "#!hello name=mylib2\n"}
+    }
+
+    test {Load script with duplicate function} {
+        assert_error {ERR Function foo already exists} {r function load "#!hello name=mylib2\nFUNCTION foo\nARGS 0\nRETURN"}
+    }
+
+    test {Load script with no metadata header} {
+        assert_error {ERR Missing library metadata} {r function load "FUNCTION foo\nARGS 0\nRETURN"}
+    }
+
+    test {Load script with header without lib name} {
+        assert_error {ERR Library name was not given} {r function load "#!hello \n"}
+    }
+
+    test {Load script with header with unknown param} {
+        assert_error {ERR Invalid metadata value given: nme=mylib} {r function load "#!hello nme=mylib\n"}
+    }
+
+    test {Load script with header with lib name passed twice} {
+        assert_error {ERR Invalid metadata value, name argument was given multiple times} {r function load "#!hello name=mylib2 name=mylib3\n"}
+    }
+
+    test {Load script with invalid function name} {
+        assert_error {ERR Function names can only contain letters, numbers, or underscores(_) and must be at least one character long} {r function load "#!hello name=mylib2\nFUNCTION foo-bar\nARGS 0\nRETURN"}
+    }
+
+    test {Load script with duplicate function} {
+        assert_error {ERR Function already exists in the library} {r function load "#!hello name=mylib2\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION foo\nARGS 0\nRETURN"}
+    }
+
+    test {Call scripting engine function: calling foo works} {
+        r fcall foo 0 134
+    } {134}
+
+    test {Call scripting engine function: calling bar works} {
+        r fcall bar 0
+    } {432}
+
+    test {Replace function library and call functions} {
+        set result [r function load replace "#!hello name=mylib\nFUNCTION foo\nARGS 0\nRETURN\nFUNCTION bar\nCONSTI 500\nRETURN"]
+        assert_equal $result "mylib"
+
+        set result [r fcall foo 0 132]
+        assert_equal $result 132
+
+        set result [r fcall bar 0]
+        assert_equal $result 500
+    }
+
+    test {List scripting engine functions} {
+        r function load replace "#!hello name=mylib\nFUNCTION foobar\nARGS 0\nRETURN"
+        r function list
+    } {{library_name mylib engine HELLO functions {{name foobar description {} flags {}}}}}
+
+    test {Unload scripting engine module} {
+        set result [r module unload helloengine]
+        assert_equal $result "OK"
+    }
+}