From 6ae33482195a225a4b7d2eb831301249e9c43b2c Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 25 Jan 2024 11:19:10 +0100 Subject: [PATCH 1/2] Allow user to configure cache size + cURL debug logging --- README.md | 18 ++++++------ cloud_sqlite_vfs.go | 67 +++++++++++++++++++++++++++------------------ example/main.go | 3 +- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 1a2b9ef..b89b546 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # go-cloud-sqlite-vfs -# Description +## Description This project wraps the [Cloud Backed SQLite](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) (CBS) -solution into a golang package. The project uses [The SQLite OS Interface or "VFS"](https://www.sqlite.org/vfs.html) +solution into a Go package. The project uses [The SQLite OS Interface or "VFS"](https://www.sqlite.org/vfs.html) concept to create a VFS which is backed by either Azure Blob Storage or Google Cloud Storage. The VFS can be used -with every SQLite golang package as long as it supports setting a custom VFS name. +with every SQLite Go package as long as it supports setting a custom VFS name. Below are some examples about how to use this package, for further information about the workings of this package please read the [documentation](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) of the CBS project. -# Installation +## Installation This package can be installed with the `go get` command: @@ -18,9 +18,9 @@ This package can be installed with the `go get` command: go get github.com/PDOK/go-cloud-sqlite-vfs ``` -**go-cloud-sqlite-vfs is cgo package**. If you want to build your app using go-cloud-sqlite-vfs , you need a C-compiler like gcc. +**go-cloud-sqlite-vfs is cgo package**. If you want to build your app using go-cloud-sqlite-vfs, you need a C-compiler like gcc. -# Usage +## Usage ```go package main @@ -64,9 +64,9 @@ func main() { } ``` -# Example project +## Example project -1. Build `blockcachevfsd` cli. For instuctions see the [CBS website](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) +1. Build `blockcachevfsd` cli. For instructions see the [CBS website](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki) 2. Start Azurite ```bash docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0 @@ -89,7 +89,7 @@ func main() { go run ./ ``` -# Dev +## Dev Because the C code of the [CBS project](https://sqlite.org/cloudsqlite/dir?ci=tip) needs to be included in the package it can be updated with the `download-c-code.sh` script located in the root of this project. diff --git a/cloud_sqlite_vfs.go b/cloud_sqlite_vfs.go index 6af15fd..1cbe4f0 100644 --- a/cloud_sqlite_vfs.go +++ b/cloud_sqlite_vfs.go @@ -18,6 +18,16 @@ import ( "unsafe" ) +const ( + // Argument sets the maximum size of the cachefile in bytes. This option is only available for daemonless mode VFSs. + sqliteBcvCacheSize = 1 + // The argument enables (non-zero) or disables (zero) verbose libcurl logging. + sqliteBcvCurlVerbose = 4 +) + +var KEY = "" + +// VFS represent a SQLite virtual file systems backed by (Azure/Google) cloud object storage. type VFS struct { bcvfs *C.sqlite3_bcvfs cacheDir string @@ -26,8 +36,6 @@ type VFS struct { Daemonless bool } -var KEY = "" - //export csAuthCb func csAuthCb(pCtx *C.void, zStorage *C.char, zAccount *C.char, zContainer *C.char, pzAuthToken **C.char) C.int { cKey := C.CString(KEY) @@ -43,16 +51,18 @@ func removeCacheDir(cacheDir string) error { func createCacheDir(cacheDir string) error { if _, err := os.Stat(cacheDir); os.IsNotExist(err) { - err := os.Mkdir(cacheDir, 0750) + err = os.Mkdir(cacheDir, 0750) if err != nil { return err } } - return nil } -func NewVFS(vfsName string, storage string, account string, key string, containerName string, cacheDir string) (VFS, error) { +// NewVFS build a new VFS. Note: when provided cacheSize is 0 the default CBS cache size of 1Gb will be used +func NewVFS(vfsName string, storage string, account string, key string, + containerName string, cacheDir string, cacheSize int64, curlVerbose bool) (VFS, error) { + KEY = key vfs := &VFS{} @@ -72,39 +82,44 @@ func NewVFS(vfsName string, storage string, account string, key string, containe defer C.free(unsafe.Pointer(cCacheDir)) defer C.free(unsafe.Pointer(cVFSName)) - if rc == C.SQLITE_OK { - if C.sqlite3_bcvfs_isdaemon(pVfs) == 1 { - vfs.Daemonless = false - } else { - vfs.Daemonless = true - } - } else { + if rc != C.SQLITE_OK { _ = removeCacheDir(cacheDir) return *vfs, fmt.Errorf("unable to create virtual filesystem with error: %s", C.GoString(zErr)) } - if rc == C.SQLITE_OK { - C.sqlite3_bcvfs_auth_callback(pVfs, nil, (*[0]byte)(unsafe.Pointer(C.csAuthCb))) + if curlVerbose { + C.sqlite3_bcvfs_config(pVfs, sqliteBcvCurlVerbose, 1) + } + if C.sqlite3_bcvfs_isdaemon(pVfs) == 1 { + vfs.Daemonless = false + } else { + vfs.Daemonless = true - cStorage := C.CString(storage) - cAccount := C.CString(account) - cContainerName := C.CString(containerName) + if cacheSize > 0 { + // cache only works in daemonless mode + C.sqlite3_bcvfs_config(pVfs, sqliteBcvCacheSize, C.longlong(cacheSize)) + } + } - rc = C.sqlite3_bcvfs_attach(pVfs, cStorage, cAccount, cContainerName, nil, C.SQLITE_BCV_ATTACH_IFNOT, &zErr) + C.sqlite3_bcvfs_auth_callback(pVfs, nil, (*[0]byte)(unsafe.Pointer(C.csAuthCb))) - defer C.free(unsafe.Pointer(cStorage)) - defer C.free(unsafe.Pointer(cAccount)) - defer C.free(unsafe.Pointer(cContainerName)) + cStorage := C.CString(storage) + cAccount := C.CString(account) + cContainerName := C.CString(containerName) - if rc != C.SQLITE_OK { - _ = removeCacheDir(cacheDir) - return *vfs, fmt.Errorf("unable to attach virtual filesystem with error: %s", C.GoString(zErr)) - } + rc = C.sqlite3_bcvfs_attach(pVfs, cStorage, cAccount, cContainerName, nil, C.SQLITE_BCV_ATTACH_IFNOT, &zErr) + + defer C.free(unsafe.Pointer(cStorage)) + defer C.free(unsafe.Pointer(cAccount)) + defer C.free(unsafe.Pointer(cContainerName)) + + if rc != C.SQLITE_OK { + _ = removeCacheDir(cacheDir) + return *vfs, fmt.Errorf("unable to attach virtual filesystem with error: %s", C.GoString(zErr)) } vfs.bcvfs = pVfs vfs.cacheDir = cacheDir - return *vfs, nil } diff --git a/example/main.go b/example/main.go index 1b7b139..a10851a 100644 --- a/example/main.go +++ b/example/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/PDOK/go-cloud-sqlite-vfs" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" @@ -23,7 +24,7 @@ type Genre struct { } func main() { - vfs, err := cloud_sqlite_vfs.NewVFS(VFS_NAME, STORAGE, ACCOUNT, KEY, CONTAINER_NAME, CACHE_DIR) + vfs, err := cloud_sqlite_vfs.NewVFS(VFS_NAME, STORAGE, ACCOUNT, KEY, CONTAINER_NAME, CACHE_DIR, 0, false) if err != nil { fmt.Println(err) return From a1e7352a0343fe08819423d91271b708960df2e4 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 25 Jan 2024 12:52:16 +0100 Subject: [PATCH 2/2] Use constants from C source --- cloud_sqlite_vfs.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cloud_sqlite_vfs.go b/cloud_sqlite_vfs.go index 1cbe4f0..114de67 100644 --- a/cloud_sqlite_vfs.go +++ b/cloud_sqlite_vfs.go @@ -18,13 +18,6 @@ import ( "unsafe" ) -const ( - // Argument sets the maximum size of the cachefile in bytes. This option is only available for daemonless mode VFSs. - sqliteBcvCacheSize = 1 - // The argument enables (non-zero) or disables (zero) verbose libcurl logging. - sqliteBcvCurlVerbose = 4 -) - var KEY = "" // VFS represent a SQLite virtual file systems backed by (Azure/Google) cloud object storage. @@ -88,7 +81,7 @@ func NewVFS(vfsName string, storage string, account string, key string, } if curlVerbose { - C.sqlite3_bcvfs_config(pVfs, sqliteBcvCurlVerbose, 1) + C.sqlite3_bcvfs_config(pVfs, C.SQLITE_BCV_CURLVERBOSE, 1) } if C.sqlite3_bcvfs_isdaemon(pVfs) == 1 { vfs.Daemonless = false @@ -97,7 +90,7 @@ func NewVFS(vfsName string, storage string, account string, key string, if cacheSize > 0 { // cache only works in daemonless mode - C.sqlite3_bcvfs_config(pVfs, sqliteBcvCacheSize, C.longlong(cacheSize)) + C.sqlite3_bcvfs_config(pVfs, C.SQLITE_BCV_CACHESIZE, C.longlong(cacheSize)) } }