[feat] cffi

binding/dal.cc

@@ -33,8 +33,8 @@ static Napi::Object RowIterator(const Napi::CallbackInfo& args) {
 
 
 static Napi::Object Init(Napi::Env env, Napi::Object exports) {
-  exports["InitSQLite"] = Napi::Function::New(env, _InitSQLite);
-  exports["RowIterator"] = Napi::Function::New(env, RowIterator);
+  exports["initSQLite"] = Napi::Function::New(env, _InitSQLite);
+  exports["rowIterator"] = Napi::Function::New(env, RowIterator);
   return exports;
 }
 

binding/dal.go

@@ -4,7 +4,6 @@ package main
 // #include <stdio.h>
 import "C"
 import (
-	"fmt"
 	"strings"
 	"unsafe"
 
@@ -13,8 +12,10 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 )
 
-var iterators = make(map[int]*facade.RowsIter)
-var itersize = make(map[int]C.int)
+var (
+	iterators = make(map[int]*facade.RowsIter)
+	itersize  = make(map[int]C.int)
+)
 
 //export InitSQLite
 func InitSQLite(pragmas string) {
@@ -28,7 +29,6 @@ func CreateRowIterator(input []byte) C.int {
 	it.Exec(input)
 	ptr := C.int(len(iterators))
 	iterators[len(iterators)] = it
-	defer fmt.Println(ptr)
 	return ptr
 }
 

clib/clib.h

@@ -0,0 +1,91 @@
+/* Code generated by cmd/cgo; DO NOT EDIT. */
+
+/* package command-line-arguments */
+
+
+#line 1 "cgo-builtin-export-prolog"
+
+#include <stddef.h>
+
+#ifndef GO_CGO_EXPORT_PROLOGUE_H
+#define GO_CGO_EXPORT_PROLOGUE_H
+
+#ifndef GO_CGO_GOSTRING_TYPEDEF
+typedef struct { const char *p; ptrdiff_t n; } _GoString_;
+#endif
+
+#endif
+
+/* Start of preamble from import "C" comments.  */
+
+
+#line 3 "main.go"
+ #include <stdlib.h>
+ #include <stdio.h>
+
+#line 1 "cgo-generated-wrapper"
+
+
+/* End of preamble from import "C" comments.  */
+
+
+/* Start of boilerplate cgo prologue.  */
+#line 1 "cgo-gcc-export-header-prolog"
+
+#ifndef GO_CGO_PROLOGUE_H
+#define GO_CGO_PROLOGUE_H
+
+typedef signed char GoInt8;
+typedef unsigned char GoUint8;
+typedef short GoInt16;
+typedef unsigned short GoUint16;
+typedef int GoInt32;
+typedef unsigned int GoUint32;
+typedef long long GoInt64;
+typedef unsigned long long GoUint64;
+typedef GoInt64 GoInt;
+typedef GoUint64 GoUint;
+typedef size_t GoUintptr;
+typedef float GoFloat32;
+typedef double GoFloat64;
+#ifdef _MSC_VER
+#include <complex.h>
+typedef _Fcomplex GoComplex64;
+typedef _Dcomplex GoComplex128;
+#else
+typedef float _Complex GoComplex64;
+typedef double _Complex GoComplex128;
+#endif
+
+/*
+  static assertion to make sure the file is being used on architecture
+  at least with matching size of GoInt.
+*/
+typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];
+
+#ifndef GO_CGO_GOSTRING_TYPEDEF
+typedef _GoString_ GoString;
+#endif
+typedef void *GoMap;
+typedef void *GoChan;
+typedef struct { void *t; void *v; } GoInterface;
+typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;
+
+#endif
+
+/* End of boilerplate cgo prologue.  */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern void InitSQLite(char* params);
+extern int CreateRowIterator(char* data, int size);
+extern void* NextRow(int itid);
+extern int GetLen(int idx);
+extern void Cleanup(int itid);
+extern void Free(void* ptr);
+
+#ifdef __cplusplus
+}
+#endif

clib/go.mod

@@ -0,0 +1,16 @@
+module l12.xyz/x/dal/clib
+
+go 1.22.6
+
+require (
+	github.com/mattn/go-sqlite3 v1.14.22
+	l12.xyz/x/dal v0.0.0
+)
+
+require (
+	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/tinylib/msgp v1.2.0 // indirect
+)
+
+replace l12.xyz/x/dal => ../

clib/go.sum

@@ -0,0 +1,8 @@
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
+github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
+github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=

clib/main.go

@@ -0,0 +1,74 @@
+package main
+
+// #include <stdlib.h>
+// #include <stdio.h>
+import "C"
+
+import (
+	"fmt"
+	"strings"
+	"unsafe"
+
+	"l12.xyz/x/dal/pkg/facade"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var (
+	iterators = make(map[int]*facade.RowsIter)
+	itersize  = make(map[int]C.int)
+)
+
+//export InitSQLite
+func InitSQLite(params *C.char) {
+	pragmas := C.GoString(params)
+	pragmasArray := strings.Split(pragmas, ";")
+	facade.InitSQLite(pragmasArray)
+}
+
+//export CreateRowIterator
+func CreateRowIterator(data *C.char, size C.int) C.int {
+	var it = &facade.RowsIter{}
+	input := C.GoBytes(unsafe.Pointer(data), size)
+	fmt.Println("input", input)
+	it.Exec(input)
+	ptr := C.int(len(iterators))
+	iterators[len(iterators)] = it
+	return ptr
+}
+
+//export NextRow
+func NextRow(itid C.int) unsafe.Pointer {
+	it := iterators[int(itid)]
+	if it.Result != nil {
+		itersize[int(itid)] = C.int(len(it.Result))
+		return C.CBytes(it.Result)
+	}
+	data := it.Next()
+	if data == nil {
+		return nil
+	}
+	itersize[int(itid)] = C.int(len(data))
+	res := C.CBytes(data)
+	return res
+}
+
+//export GetLen
+func GetLen(idx C.int) C.int {
+	return itersize[int(idx)]
+}
+
+//export Cleanup
+func Cleanup(itid C.int) {
+	it := iterators[int(itid)]
+	it.Close()
+	delete(iterators, int(itid))
+	delete(itersize, int(itid))
+}
+
+//export Free
+func Free(ptr unsafe.Pointer) {
+	C.free(ptr)
+}
+
+func main() {}

dal/Binding.ts

@@ -5,7 +5,7 @@ type RowIterator = {
   free: () => void;
 };
 type SQLite = {
-  InitSQLite: (pragmas: Buffer) => void;
-  RowIterator: (input: Buffer) => RowIterator;
+  initSQLite: (pragmas: Buffer) => void;
+  rowIterator: (input: Buffer) => RowIterator;
 };
 export default require("../build/Release/dal.node") as SQLite;

dal/Bunding.ts

@@ -0,0 +1,68 @@
+/**
+ * This file is responsible for binding the C library to the Bun runtime.
+ */
+import { dlopen, FFIType, suffix, ptr, toBuffer } from "bun:ffi";
+
+const libname = `lib/clib.${suffix}`;
+const libpath = libname;
+
+const {
+  symbols: { InitSQLite, CreateRowIterator, NextRow, GetLen, Free, Cleanup },
+} = dlopen(libpath, {
+  InitSQLite: {
+    args: [FFIType.cstring],
+    returns: FFIType.void,
+  },
+  CreateRowIterator: {
+    args: [FFIType.cstring, FFIType.i32],
+    returns: FFIType.i32,
+  },
+  NextRow: {
+    args: [FFIType.i32],
+    returns: FFIType.ptr,
+  },
+  GetLen: {
+    args: [FFIType.i32],
+    returns: FFIType.i32,
+  },
+  Free: {
+    args: [FFIType.ptr],
+    returns: FFIType.void,
+  },
+  Cleanup: {
+    args: [FFIType.i32],
+    returns: FFIType.void,
+  },
+});
+
+function initSQLite(pragmas: string) {
+  const buf = Buffer.from(pragmas);
+  InitSQLite(ptr(buf));
+}
+
+function rowIterator(buf: Buffer) {
+  const iter = CreateRowIterator(ptr(buf), buf.length);
+  const next = () => {
+    const pointer = NextRow(iter);
+    if (pointer === null) {
+      return null;
+    }
+    const buf = toBuffer(pointer, 0, GetLen(iter));
+    //Free(pointer) //should be resolved by GC;
+    return buf;
+  };
+
+  const cleanup = () => {
+    Cleanup(iter);
+  };
+
+  return {
+    next,
+    cleanup,
+  };
+}
+
+export default {
+  initSQLite,
+  rowIterator,
+};

dal/__test__/bench.node.cjs

@@ -35,12 +35,10 @@ total: ${Mb(this.avg_heapTotal)} Mb`);
   }
 }
 
-
 const stats = new Stats();
 let prevMem = process.memoryUsage();
 stats.add(prevMem);
 
-
 function MEM(when = "") {
   const mem = process.memoryUsage();
   stats.add(mem);
@@ -52,7 +50,6 @@ buffers: ${Mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}]
 total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`);
 }
 
-
 console.time("Time to end");
 MEM("START");
 

dal/bun.cffi.ts

@@ -0,0 +1,65 @@
+import Builder from "./Builder";
+import Bunding from "./Bunding";
+import { encodeRequest, decodeRows, decodeResponse } from "./Protocol";
+import type { ExecResult } from "./Protocol";
+
+type Options = {
+  database: string;
+};
+
+/**
+ * Allows to use SQLite databases in BunJS
+ */
+export default class CBuilder<
+  I extends abstract new (...args: any) => any,
+> extends Builder<I> {
+  constructor(opts: Options) {
+    super({ database: opts.database, url: "" });
+  }
+  /**
+   * TODO: handle responses
+   */
+  async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
+    this.formatRequest();
+    const req = Buffer.from(encodeRequest(this.request));
+    const iter = Bunding.rowIterator(req);
+    for (;;) {
+      const response = iter.next();
+      if (response === null) {
+        iter.cleanup();
+        break;
+      }
+      const rows = decodeRows(response);
+      if (rows.length === 0) {
+        iter.cleanup();
+        break;
+      }
+      for (const row of rows) {
+        if (this.headerRow === null) {
+          this.headerRow = row.r;
+          continue;
+        }
+        yield this.formatRow(row.r);
+      }
+    }
+  }
+  async Query<T = InstanceType<I>>(): Promise<T[]> {
+    const rows = this.Rows();
+    const result: T[] = [];
+    for await (const row of rows) {
+      result.push(row);
+    }
+    return result;
+  }
+  async Exec(): Promise<ExecResult> {
+    this.formatRequest();
+    const req = Buffer.from(encodeRequest(this.request));
+    const iter = Bunding.rowIterator(req);
+    const response = iter.next();
+    if (response === null) {
+      iter.cleanup();
+      throw new Error("No response");
+    }
+    return decodeResponse(response);
+  }
+}

dal/bun.ffi.ts

@@ -1,41 +0,0 @@
-import { dlopen, FFIType, suffix, ptr } from "bun:ffi";
-import { join } from "node:path";
-import fs from "node:fs";
-
-// TODO: build a shared library compatible with cffi
-const libname = `libdal-arm64.${suffix}`;
-const libpath = join(__dirname, '..', 'lib', libname);
-
-const {
-    symbols: {
-        InitSQLite,
-        CreateRowIterator,
-        NextRow,
-        GetLen,
-        FreeIter,
-    },
-  } = dlopen(
-    libpath,
-    {
-      InitSQLite: {
-        args: [ FFIType.cstring ],
-        returns: FFIType.void,
-      },
-      CreateRowIterator: {
-        args: [ FFIType.cstring ],
-        returns: FFIType.i32,
-      },
-      NextRow: {
-        args: [FFIType.i32],
-        returns: FFIType.cstring,
-      },
-      GetLen: {
-        args: [FFIType.i32],
-        returns: FFIType.i32,
-      },
-      FreeIter : {
-        args: [FFIType.i32],
-        returns: FFIType.void,
-      },
-    },
-  );

dal/node.napi.ts

@@ -9,7 +9,6 @@ type Options = {
 
 /**
  * Allows to use SQLite databases in a NodeJS process.
- * It is less memory-efficient than a seaparate server, and uses absolute path for database name.
  */
 export default class CBuilder<
   I extends abstract new (...args: any) => any,
@@ -23,7 +22,7 @@ export default class CBuilder<
   async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
     this.formatRequest();
     const req = Buffer.from(encodeRequest(this.request));
-    const iter = Binding.RowIterator(req);
+    const iter = Binding.rowIterator(req);
     for (;;) {
       const response = iter.next();
       const rows = decodeRows(response);
@@ -51,7 +50,7 @@ export default class CBuilder<
   async Exec(): Promise<ExecResult> {
     this.formatRequest();
     const req = Buffer.from(encodeRequest(this.request));
-    const iter = Binding.RowIterator(req);
+    const iter = Binding.rowIterator(req);
     const response = iter.next();
     return decodeResponse(response);
   }