Calling MQF from Rust

.gitignore

@@ -75,3 +75,6 @@ ThirdParty/stxxl
 
 .idea/*
 build/*
+
+Cargo.lock
+target

.travis.yml

@@ -5,22 +5,36 @@ compiler:
 - g++
 os:
 - linux
-
-
-before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get install -y make automake autotools-dev cmake
-script:
-  - mkdir -p build
-  - cd build
-  - cmake ..
-  - make
-  - ctest
-  - gcov -n -o . src/gqf.cpp > /dev/null;
 branches:
   only:
   - mqfDevelopmenet
   - master
+cache:
+  directories:
+    - "$HOME/.cargo"
+    - "target"
 
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
+jobs:
+  include:
+    - &test
+      stage: test
+      before_install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y make automake autotools-dev cmake
+      script:
+        - mkdir -p build
+        - cd build
+        - cmake ..
+        - make
+        - ctest
+        - gcov -n -o . src/gqf.cpp > /dev/null;
+      after_success:
+        - bash <(curl -s https://codecov.io/bash)
+    - <<: *test
+      name: Rust FFI bindings
+      language: rust
+      rust: stable
+      env:
+        - RUSTFLAGS="-Clink-arg=-fuse-ld=gold"
+      script:
+        - cargo test

Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "mqf"
+version = "0.1.0"
+authors = ["Luiz Irber <[email protected]>"]
+links = "libmqf"
+
+[build-dependencies]
+bindgen = "0.51"
+cmake = "0.1.42"
+
+[dependencies]

build.rs

@@ -0,0 +1,56 @@
+use std::env;
+use std::path::PathBuf;
+
+extern crate cmake;
+use cmake::Config;
+
+fn main() {
+    let dst = Config::new(".").define("BUILD_STATIC_LIBS", "ON").build();
+
+    // TODO: there are probably better ways to do this...
+    let target = env::var("TARGET").unwrap();
+    if target.contains("apple") {
+        println!("cargo:rustc-link-lib=dylib=c++");
+    } else if target.contains("linux") {
+        println!("cargo:rustc-link-lib=dylib=stdc++");
+    } else {
+        unimplemented!();
+    }
+
+    println!("cargo:rustc-link-search=native={}/build/src", dst.display());
+    println!(
+        "cargo:rustc-link-search=native={}/build/ThirdParty/stxxl/lib",
+        dst.display()
+    );
+    // TODO: static libs are being generated in lib too,
+    // cmake seems to be just copying it from the right locations.
+    // But not sure we should be using them...
+    // println!("cargo:rustc-link-search=native=lib");
+
+    println!("cargo:rustc-link-lib=static=MQF");
+    // TODO: there are two names for stxxl, depending on being built on release
+    // or debug mode...
+    //println!("cargo:rustc-link-lib=static=stxxl");
+    println!("cargo:rustc-link-lib=static=stxxl_debug");
+
+    let bindings = bindgen::Builder::default()
+        .clang_arg("-I./include")
+        .clang_arg("-x")
+        .clang_arg("c++")
+        .clang_arg("-std=c++11")
+        .header("include/gqf.h")
+        .whitelist_type("QF")
+        .whitelist_function("qf_init")
+        .whitelist_function("qf_insert")
+        .whitelist_function("qf_count_key")
+        .whitelist_function("qf_destroy")
+        .whitelist_function("qf_copy")
+        .blacklist_type("std::*")
+        .generate()
+        .expect("Unable to generate bindings");
+
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    bindings
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("couldn't write bindings!");
+}

include/gqf.h

@@ -109,7 +109,7 @@ extern "C" {
 
 	void qf_destroy(QF *qf);
 
-	void qf_copy(QF *dest, QF *src);
+	void qf_copy(QF *dest, const QF *src);
 
 	/*!
 	 	@breif Increment the counter for this item by count.

src/gqf.cpp

@@ -1982,7 +1982,7 @@ bool qf_getBlockLabel_pointer_byItem(const QF *qf, uint64_t key,char *&res){
 
 /* The caller should call qf_init on the dest QF before calling this function.
  */
-void qf_copy(QF *dest, QF *src)
+void qf_copy(QF *dest, const QF *src)
 {
 	memcpy(dest->mem, src->mem, sizeof(qfmem));
 	memcpy(dest->metadata, src->metadata, sizeof(qfmetadata));

src/lib.rs

@@ -0,0 +1,110 @@
+mod raw;
+
+use std::ffi::CString;
+use std::ptr;
+
+pub struct MQF {
+    inner: raw::QF,
+}
+
+impl Default for MQF {
+    fn default() -> MQF {
+        MQF {
+            inner: raw::QF {
+                mem: ptr::null_mut(),
+                metadata: ptr::null_mut(),
+                blocks: ptr::null_mut(),
+            },
+        }
+    }
+}
+
+impl Drop for MQF {
+    fn drop(&mut self) {
+        unsafe { raw::qf_destroy(&mut self.inner) };
+    }
+}
+
+impl Clone for MQF {
+    fn clone(&self) -> Self {
+        let mut new_qf = MQF::default();
+        unsafe {
+            raw::qf_copy(&mut new_qf.inner, &self.inner);
+        };
+        new_qf
+    }
+}
+
+impl MQF {
+    pub fn new(counter_size: u64, qbits: u64) -> MQF {
+        let mut mqf = MQF::default();
+
+        let num_hash_bits = qbits + 8;
+        let maximum_count = (1u64 << counter_size) - 1;
+
+        let s = CString::new("").unwrap();
+
+        unsafe {
+            raw::qf_init(
+                &mut mqf.inner,
+                1u64 << qbits, // nslots
+                num_hash_bits, // key_bits
+                0,             // label_bits
+                counter_size,  // fixed_counter_size
+                0,             // blocksLabelSize
+                true,          // mem
+                s.as_ptr(),    // path
+                0,             // seed (doesn't matter)
+            );
+        };
+
+        mqf
+    }
+
+    pub fn insert(&mut self, key: u64, count: u64) {
+        unsafe { raw::qf_insert(&mut self.inner, key, count, false, false) };
+    }
+
+    pub fn count_key(&self, key: u64) -> u64 {
+        unsafe { raw::qf_count_key(&self.inner, key) }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn simple_counting_test_api() {
+        //except first item is inserted 5 times to full test _insert1
+        let counter_size = 2;
+        let qbits = 5;
+        let mut qf: MQF = MQF::new(counter_size, qbits);
+
+        let mut count = 0;
+        let mut fixed_counter = 0;
+
+        for i in 0..=10 {
+            qf.insert(100, 1);
+            count = qf.count_key(100);
+            dbg!((count, fixed_counter));
+            assert_eq!(count, 1 + i);
+        }
+
+        qf.insert(1500, 50);
+
+        count = qf.count_key(1500);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 50);
+
+        qf.insert(1600, 60);
+        count = qf.count_key(1600);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 60);
+
+        qf.insert(2000, 4000);
+        count = qf.count_key(2000);
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 4000);
+    }
+}

src/raw.rs

@@ -0,0 +1,77 @@
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(dead_code)]
+include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::ffi::CString;
+    use std::ptr;
+
+    #[test]
+    fn simple_counting_test() {
+        //except first item is inserted 5 times to full test _insert1
+        let mut qf: QF = QF {
+            mem: ptr::null_mut(),
+            metadata: ptr::null_mut(),
+            blocks: ptr::null_mut(),
+        };
+
+        let counter_size = 2;
+        let qbits = 5;
+        let num_hash_bits = qbits + 8;
+        let maximum_count = (1u64 << counter_size) - 1;
+        let mut count = 0;
+        let mut fixed_counter = 0;
+
+        let s = CString::new("").unwrap();
+
+        //INFO("Counter size = "<<counter_size<<" max count= "<<maximum_count);
+        unsafe {
+            qf_init(
+                &mut qf,
+                1u64 << qbits,
+                num_hash_bits,
+                0,
+                counter_size,
+                0,
+                true,
+                s.as_ptr(),
+                2038074761,
+            );
+        }
+
+        for i in 0..=10 {
+            unsafe {
+                qf_insert(&mut qf, 100, 1, false, false);
+            };
+            count = unsafe { qf_count_key(&qf, 100) };
+            dbg!((count, fixed_counter));
+            assert_eq!(count, 1 + i);
+        }
+
+        unsafe {
+            qf_insert(&mut qf, 1500, 50, false, false);
+        }
+
+        count = unsafe { qf_count_key(&qf, 1500) };
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 50);
+
+        unsafe {
+            qf_insert(&mut qf, 1600, 60, false, false);
+        }
+        count = unsafe { qf_count_key(&qf, 1600) };
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 60);
+
+        unsafe {
+            qf_insert(&mut qf, 2000, 4000, false, false);
+        }
+        count = unsafe { qf_count_key(&qf, 2000) };
+        dbg!((count, fixed_counter));
+        assert_eq!(count, 4000);
+    }
+}