Refactoring the rust api

.github/workflows/linux.yml

@@ -0,0 +1,40 @@
+name: Test ubuntu latest
+
+on:
+  push:
+    branches: [ master, refactoring_rust_api ]
+  pull_request:
+    branches: [ master, refactoring_rust_api ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        test_args: ["", "-V --vg-suppressions ../leakcheck.supp"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Checkout submodules
+      run: git submodule update --init --recursive
+    - name: format
+      run: cargo fmt -- --check
+    - name: install rltest
+      run: python3 -m pip install RLTest gevent
+    - name: install redis
+      run: git clone https://github.com/redis/redis; cd redis; git checkout 7.0.5; BUILD_TLS=yes make valgrind install
+    - name: install valgrind
+      run: sudo apt-get install valgrind
+    - name: Build
+      run: |
+        cd tests/mr_test_module/
+        cargo build --verbose
+    - name: Run tests
+      run: |
+        cd tests/mr_test_module/pytests/
+        DEBUG=1 ./run_full_tests.sh ${{ matrix.test_args }}

.github/workflows/macos.yml

@@ -0,0 +1,42 @@
+name: Test Macos
+
+on:
+  push:
+    branches: [ master, refactoring_rust_api ]
+  pull_request:
+    branches: [ master, refactoring_rust_api ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Checkout submodules
+      run: git submodule update --init --recursive
+    - name: format
+      run: cargo fmt -- --check
+    - name: install rltest
+      run: python3 -m pip install RLTest gevent
+    - name: install redis
+      run: git clone https://github.com/redis/redis; cd redis; git checkout 7.0.5; BUILD_TLS=yes make install
+    - name: install automake
+      run: brew install automake
+    - name: install openssl
+      run: brew install [email protected]
+    - name: Build
+      run: |
+        cd tests/mr_test_module/
+        export PKG_CONFIG_PATH=/usr/local/opt/[email protected]/lib/pkgconfig/
+        cargo build --verbose
+    - name: Run tests
+      run: |
+        cd tests/mr_test_module/pytests/
+        DEBUG=1 ./run_full_tests.sh

Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "lib_mr"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+#redis-module = { version="0.22.0", features = ["experimental-api"]}
+redis-module = { git = "https://github.com/RedisLabsModules/redismodule-rs", branch = "api_extentions", features = ["experimental-api"]}
+serde_json = "1.0"
+serde = "1.0"
+serde_derive = "1.0"
+libc = "0.2"
+linkme = "0.3"
+
+[build-dependencies]
+bindgen = "0.59.2"
+
+[lib]
+crate-type = ["rlib"]
+name = "mr"
+path = "rust_api/lib.rs"

LibMRDerive/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "lib_mr_derive"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = "1.0"
+quote = "1.0"
+
+[lib]
+name = "mr_derive"
+path = "src/lib.rs"
+proc-macro = true

LibMRDerive/src/lib.rs

@@ -0,0 +1,28 @@
+extern crate proc_macro;
+use proc_macro::TokenStream;
+use quote::quote;
+use quote::format_ident;
+use syn;
+
+#[proc_macro_derive(BaseObject)]
+pub fn base_object_derive(item: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(item).unwrap();
+    let name = &ast.ident;
+
+    let func_name = format_ident!("register_{}", name.to_string().to_lowercase());
+
+    let gen = quote! {
+        impl mr::libmr::base_object::BaseObject for #name {
+            fn get_name() -> &'static str {
+                concat!(stringify!(#name), "\0")
+            }
+        }
+
+        #[linkme::distributed_slice(mr::libmr::REGISTER_LIST)]
+        fn #func_name() {
+            #name::register();        
+        }
+    };
+
+    gen.into()
+}

Makefile

@@ -4,9 +4,11 @@ clean: clean_libmr
 
 build_deps:
 	make -C deps/
-	
-libmr: build_deps
+
+libmr_only:
 	make -C src/
+
+libmr: build_deps libmr_only
 
 run_tests:
 	make -C ./tests/mr_test_module/ test

build.rs

@@ -0,0 +1,79 @@
+extern crate bindgen;
+
+use std::env;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn probe_paths<'a>(paths: &'a [&'a str]) -> Option<&'a str> {
+    paths.iter().find(|path| Path::new(path).exists()).copied()
+}
+
+fn find_macos_openssl_prefix_path() -> &'static str {
+    const PATHS: [&str; 3] = [
+        "/usr/local/opt/openssl",
+        "/usr/local/opt/[email protected]",
+        "/opt/homebrew/opt/[email protected]",
+    ];
+    probe_paths(&PATHS).unwrap_or("")
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=src/*.c");
+    println!("cargo:rerun-if-changed=src/*.h");
+    println!("cargo:rerun-if-changed=src/utils/*.h");
+    println!("cargo:rerun-if-changed=src/utils/*.c");
+
+    if !Command::new("make")
+        .env(
+            "MODULE_NAME",
+            std::env::var("MODULE_NAME").expect("module name was not given"),
+        )
+        .status()
+        .expect("failed to compile libmr")
+        .success()
+    {
+        panic!("failed to compile libmr");
+    }
+
+    let output_dir = env::var("OUT_DIR").expect("Can not find out directory");
+
+    if !Command::new("cp")
+        .args(["src/libmr.a", &output_dir])
+        .status()
+        .expect("failed copy libmr.a to output directory")
+        .success()
+    {
+        panic!("failed copy libmr.a to output directory");
+    }
+
+    let build = bindgen::Builder::default();
+
+    let bindings = build
+        .header("src/mr.h")
+        .size_t_is_usize(true)
+        .layout_tests(false)
+        .generate()
+        .expect("error generating bindings");
+
+    let out_path = PathBuf::from(&output_dir);
+    bindings
+        .write_to_file(out_path.join("libmr_bindings.rs"))
+        .expect("failed to write bindings to file");
+
+    let open_ssl_prefix_path = match std::option_env!("OPENSSL_PREFIX") {
+        Some(p) => p,
+        None if std::env::consts::OS == "macos" => find_macos_openssl_prefix_path(),
+        _ => "",
+    };
+
+    let open_ssl_lib_path_link_argument = if open_ssl_prefix_path.is_empty() {
+        "".to_owned()
+    } else {
+        format!("-L{open_ssl_prefix_path}/lib/")
+    };
+
+    println!(
+        "cargo:rustc-flags=-L{} {} -lmr -lssl -lcrypto",
+        output_dir, open_ssl_lib_path_link_argument
+    );
+}

deps/Makefile

@@ -1,11 +1,23 @@
 all: build_hiredis build_libevent
 
 build_hiredis:
+ifneq ("$(wildcard built_hiredis)","")
+	echo hiredis already built
+else
 	MAKEFLAGS='' USE_SSL=1 make -C ./hiredis/
+endif
+	touch built_hiredis
 
 build_libevent:
-	cd libevent; autoreconf -v -i -f; CFLAGS=-fPIC ./configure; make
-
+ifneq ("$(wildcard built_libevent)","")
+	echo libevent already built
+else
+	cd libevent; autoreconf -v -i -f; CFLAGS=-fPIC ./configure PKG_CONFIG_PATH=$(PKG_CONFIG_PATH); make
+endif
+	touch built_libevent
+
 clean:
 	make -C ./hiredis/ clean
 	make -C ./libevent/ clean
+	rm built_libevent
+	rm built_hiredis

rust_api/lib.rs

@@ -0,0 +1,11 @@
+/*
+ * Copyright Redis Ltd. 2021 - present
+ * Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or
+ * the Server Side Public License v1 (SSPLv1).
+ */
+
+#[macro_use]
+extern crate serde_derive;
+
+pub mod libmr;
+pub mod libmr_c_raw;

rust_api/libmr/accumulator.rs

@@ -0,0 +1,65 @@
+/*
+ * Copyright Redis Ltd. 2021 - present
+ * Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or
+ * the Server Side Public License v1 (SSPLv1).
+ */
+
+use crate::libmr_c_raw::bindings::{
+    ExecutionCtx, MR_ExecutionCtxSetError, MR_RegisterAccumulator, Record,
+};
+
+use crate::libmr::base_object::{register, BaseObject};
+use crate::libmr::record;
+use crate::libmr::record::MRBaseRecord;
+use crate::libmr::RustMRError;
+
+use std::os::raw::{c_char, c_void};
+
+use std::ptr;
+
+pub extern "C" fn rust_accumulate<Step: AccumulateStep>(
+    ectx: *mut ExecutionCtx,
+    accumulator: *mut Record,
+    r: *mut Record,
+    args: *mut c_void,
+) -> *mut Record {
+    let s = unsafe { &*(args as *mut Step) };
+    let accumulator = if accumulator.is_null() {
+        None
+    } else {
+        let mut accumulator =
+            unsafe { *Box::from_raw(accumulator as *mut MRBaseRecord<Step::Accumulator>) };
+        Some(accumulator.record.take().unwrap())
+    };
+    let mut r = unsafe { Box::from_raw(r as *mut MRBaseRecord<Step::InRecord>) };
+    let res = match s.accumulate(accumulator, r.record.take().unwrap()) {
+        Ok(res) => res,
+        Err(e) => {
+            unsafe { MR_ExecutionCtxSetError(ectx, e.as_ptr() as *mut c_char, e.len()) };
+            return ptr::null_mut();
+        }
+    };
+    Box::into_raw(Box::new(MRBaseRecord::new(res))) as *mut Record
+}
+
+pub trait AccumulateStep: BaseObject {
+    type InRecord: record::Record;
+    type Accumulator: record::Record;
+
+    fn accumulate(
+        &self,
+        accumulator: Option<Self::Accumulator>,
+        r: Self::InRecord,
+    ) -> Result<Self::Accumulator, RustMRError>;
+
+    fn register() {
+        let obj = register::<Self>();
+        unsafe {
+            MR_RegisterAccumulator(
+                Self::get_name().as_ptr() as *mut c_char,
+                Some(rust_accumulate::<Self>),
+                obj,
+            );
+        }
+    }
+}