Merge pull request #3 from PDOK/bump-cbs-and-sqlite

Bump SQLite + Cloud-Backed SQLite to latest version

bcv_int.h

@@ -362,7 +362,7 @@ Manifest *bcvManifestRef(Manifest *p);
 int bcvManifestDup(Manifest *p, Manifest **ppNew);
 void bcvManifestExpand(int*, Manifest**, int);
 
-void bcvBlockidToText(Manifest *p, const u8 *pBlk, char *aBuf);
+void bcvBlockidToText(int nName, const u8 *pBlk, char *aBuf);
 
 int bcvWritefile(sqlite3_file *pFd, const u8 *aData, int nData, i64 iOff);
 int bcvOpenLocal(const char *, int bWal, int bReadonly, sqlite3_file **ppFd);

bcvutil.c

@@ -76,6 +76,7 @@ struct sqlite3_bcv {
 
   bcv_log_cb xBcvLog;
   void *pBcvLogCtx;
+  BcvLog *pBcvLogObj;
 };
 
 typedef struct sqlite3_bcv_job BcvDispatchJob;
@@ -1099,14 +1100,14 @@ static int bcvManifestNameToIndex(Manifest *p, const char *zDb){
 }
 
 /*
-** The first argument points to a NAMEBYTES byte buffer
+** The second argument points to a nName byte buffer
 ** containing a block-id. Format the block-id as text and write it to
-** buffer aBuf[], which must be at leat BCV_FILESYSTEM_BLOCKID_BYTES
+** buffer aBuf[], which must be at least BCV_FILESYSTEM_BLOCKID_BYTES
 ** in size.
 */
-void bcvBlockidToText(Manifest *p, const u8 *pBlk, char *aBuf){
-  hex_encode(pBlk, NAMEBYTES(p), aBuf, 1);
-  memcpy(&aBuf[NAMEBYTES(p)*2], ".bcv", 5);
+void bcvBlockidToText(int nName, const u8 *pBlk, char *aBuf){
+  hex_encode(pBlk, nName, aBuf, 1);
+  memcpy(&aBuf[nName*2], ".bcv", 5);
 }
 
 static sqlite3_bcv sqlite3_bcv_oom_handle = {
@@ -1152,6 +1153,7 @@ void sqlite3_bcv_close(sqlite3_bcv *p){
   if( p && p!=&sqlite3_bcv_oom_handle ){
     bcvContainerClose(p->pCont);
     bcvDispatchFree(p->pDisp);
+    bcvLogDelete(p->pBcvLogObj);
     sqlite3_free(p->zErrmsg);
     sqlite3_free(p);
   }
@@ -1194,8 +1196,13 @@ int sqlite3_bcv_config(sqlite3_bcv *p, int eOp, ...){
       case SQLITE_BCVCONFIG_LOG: {
         p->pBcvLogCtx = va_arg(ap, void*);
         p->xBcvLog = (bcv_log_cb)va_arg(ap, bcv_log_cb);
+        if( p->pBcvLogObj==0 ){
+          p->pBcvLogObj = bcvLogNew();
+          if( p->pBcvLogObj==0 ) rc = SQLITE_NOMEM;
+        }
         if( p->xBcvLog ){
           bcvDispatchLog(p->pDisp, (void*)p, bcvLogWrapper);
+          bcvDispatchLogObj(p->pDisp, p->pBcvLogObj);
         }else{
           bcvDispatchLog(p->pDisp, 0, 0);
         }
@@ -1330,7 +1337,7 @@ static void bcvUploadOneBlockRetry(BcvUploadJob *pJob, int *pbRetry){
       sqlite3_randomness(NAMEBYTES(pMan), aBlk);
     }
 
-    bcvBlockidToText(pMan, aBlk, aBuf);
+    bcvBlockidToText(NAMEBYTES(pMan), aBlk, aBuf);
     rc = bcvDispatchPut(
         pBcv->pDisp, pBcv->pCont, aBuf, 0, aSpace, pMan->szBlk, 
         pJob, bcvUploadOneBlockCb
@@ -1677,7 +1684,7 @@ static void bcvDownloadOneFile(BcvDownloadJob *pJob){
 
   if( pBcv->errCode==SQLITE_OK ){
     char aBuf[BCV_MAX_FSNAMEBYTES];
-    bcvBlockidToText(pMan, aBlk, aBuf);
+    bcvBlockidToText(NAMEBYTES(pMan), aBlk, aBuf);
     pJob->iBlk = iBlk;
     rc = bcvDispatchFetch(
         pBcv->pDisp, pBcv->pCont, aBuf, 0, pMd5, pJob, bcvDownloadOneFileCb
@@ -1875,7 +1882,7 @@ static void bcvExtraLogManifest(
       u8 *aEntry = &pMan->aDelBlk[i*GCENTRYBYTES(pMan)];
       i64 t;
       char aBuf[BCV_MAX_FSNAMEBYTES];
-      bcvBlockidToText(pMan, aEntry, aBuf);
+      bcvBlockidToText(NAMEBYTES(pMan), aEntry, aBuf);
       t = bcvGetU64(&aEntry[NAMEBYTES(pMan)]);
       bcvExtraLog(p, "    %s (t=%lld)", aBuf, t);
     }
@@ -1889,7 +1896,9 @@ static void bcvExtraLogManifest(
       bcvExtraLog(p, "Database %d block list: (%d blocks)", i,pDb->nBlkLocal);
       for(j=0; j<pDb->nBlkLocal; j++){
         char aBuf[BCV_MAX_FSNAMEBYTES];
-        bcvBlockidToText(pMan, &pDb->aBlkLocal[j*NAMEBYTES(pMan)], aBuf);
+        bcvBlockidToText(
+            NAMEBYTES(pMan), &pDb->aBlkLocal[j*NAMEBYTES(pMan)], aBuf
+        );
         bcvExtraLog(p, "    %s", aBuf);
       }
     }
@@ -1969,7 +1978,7 @@ static void bcvfsDeleteOneBlock(DeleteCtx *pCtx){
       /* Do some debug logging */
       if( pCtx->p->nLogLevel>=BCV_LOGLEVEL_DEBUG ){
         char zDebug[BCV_MAX_FSNAMEBYTES];
-        bcvBlockidToText(pCtx->pMan, &pCtx->aOld[iOff], zDebug);
+        bcvBlockidToText(NAMEBYTES(pCtx->pMan), &pCtx->aOld[iOff], zDebug);
         if( bDel==0 ){
           bcvExtraLog(pCtx->p, "cleanup: not deleting block %s "
               "(not eligible for another %lldms)", zDebug, iTime-pCtx->iDelTime
@@ -1981,7 +1990,7 @@ static void bcvfsDeleteOneBlock(DeleteCtx *pCtx){
 
       if( pCtx->iDelTime==0 || iTime<=pCtx->iDelTime ){
         char zFile[BCV_MAX_FSNAMEBYTES];
-        bcvBlockidToText(pCtx->pMan, &pCtx->aOld[iOff], zFile);
+        bcvBlockidToText(NAMEBYTES(pCtx->pMan), &pCtx->aOld[iOff], zFile);
         pCtx->p->errCode = bcvDispatchDelete(
             pCtx->pDisp, pCtx->pBcv, zFile, 0, (void*)pCtx, bcvfsDeleteBlockDone
         );

blockcachevfs.c

@@ -1442,7 +1442,6 @@ static int bcvfsReadWriteDatabase(
     assert( bWrite==0 );
     return SQLITE_IOERR_SHORT_READ;
   }
-  assert( (iAmt & (iAmt-1))==0 );
 
   /* Take the VFS mutex and do three things under its cover:
   **
@@ -4723,7 +4722,7 @@ static void bcvfsUploadOneBlockTry(UploadCtx2 *pCtx, int *pbRetry){
       memcpy(aName, aNew, nName);
       if( rc==SQLITE_OK && bSkip==0 ){
         char zFile[BCV_MAX_FSNAMEBYTES];
-        bcvBlockidToText(pCtx->pMan, aNew, zFile);
+        bcvBlockidToText(NAMEBYTES(pCtx->pMan), aNew, zFile);
         p->pCtx = pCtx;
         rc = bcvDispatchPut(pCtx->pDisp, pCtx->pBcv,
             zFile, 0, aBuf, nBuf, (void*)p, bcvfsUploadBlockDone

download-c-code.sh

@@ -1,26 +1,38 @@
 #!/bin/sh
 
-VERSION=8535bc08c4
+CBS_VERSION=060e73c89a
+SQLITE_VERSION=3460000
 
-wget -qO ./tmp.zip https://sqlite.org/cloudsqlite/zip/${VERSION}/cloudsqlite-${VERSION}.zip
+# Download and install cloud-backed-sqlite
+wget -qO ./tmp.zip https://sqlite.org/cloudsqlite/zip/${CBS_VERSION}/cloudsqlite-${CBS_VERSION}.zip
 unzip ./tmp.zip
 rm tmp.zip
 
 rm -f *.c
 rm -f *.h
-cp -f ./cloudsqlite-${VERSION}/src/bcv_int.h ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvutil.c ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvutil.h ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvmodule.c ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvmodule.h ./
-cp -f ./cloudsqlite-${VERSION}/src/blockcachevfs.c ./
-cp -f ./cloudsqlite-${VERSION}/src/blockcachevfs.h ./
-cp -f ./cloudsqlite-${VERSION}/src/simplexml.c ./
-cp -f ./cloudsqlite-${VERSION}/src/simplexml.h ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvencrypt.c ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvencrypt.h ./
-cp -f ./cloudsqlite-${VERSION}/src/bcvlog.c ./
-cp -f ./cloudsqlite-${VERSION}/src/sqlite3.c ./
-cp -f ./cloudsqlite-${VERSION}/src/sqlite3.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcv_int.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvutil.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvutil.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvmodule.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvmodule.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/blockcachevfs.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/blockcachevfs.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/simplexml.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/simplexml.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvencrypt.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvencrypt.h ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/bcvlog.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/sqlite3.c ./
+cp -f ./cloudsqlite-${CBS_VERSION}/src/sqlite3.h ./
 
-rm -rf ./cloudsqlite-${VERSION}
+rm -rf ./cloudsqlite-${CBS_VERSION}
+
+# Download sqlite and upgrade the default sqlite version shipped with cloud-backed-sqlite
+wget -qO ./tmp.zip https://sqlite.org/2024/sqlite-amalgamation-${SQLITE_VERSION}.zip
+unzip ./tmp.zip
+rm tmp.zip
+
+cp -f ./sqlite-amalgamation-${SQLITE_VERSION}/sqlite3.c ./
+cp -f ./sqlite-amalgamation-${SQLITE_VERSION}/sqlite3.h ./
+
+rm -rf ./sqlite-amalgamation-${SQLITE_VERSION}