[feat] prep for cffi

Signed-off-by: Anton Nesterov <anton@demiurg.io>

.gitignore

@@ -3,6 +3,9 @@
 *.a
 *.dylib
 build/
+database.sqlite
+database.sqlite-shm
+database.sqlite-wal
 # Logs
 
 logs

README.md

@@ -1,19 +1,22 @@
 # DAL
 
 **Data Access Layer**
+
 DAL is a proxy layer for SQL databases with a MongoDB inspired query interface.
-It can be used as a Go or NodeJS package (requires compiler) to create your own proxy and apply custom middlewares.
+It can be used as a Go or NodeJS package (requires compiler).
+It is modular and allows to create your own proxy and apply custom middlewares.
 
 _Notes:_
 
-- This project is still in early alpha. You need to build it yourself and use at your own risk.
+- This project is still in **early alpha**. You need to build it yourself and use at your own risk.
 - At the time only SQLite is implemented, however, other drivers might work.
 
 _Use cases:_
 
 - For IOT networks when MySQL/PG are too heavy.
 - If you need a layer between your application and the database (i.e. for caching).
-- If you want a MongoDB-like query interface for your SQL.
+- If you want a MongoDB-like query interface for your SQL db.
+- When you need a SQLite proxy (useful to share datasets with services)
 
 ## Usage
 
@@ -102,16 +105,26 @@ const db = new DAL({
   url: "http://localhost:8111",
 });
 
+// SELECT * FROM test t WHERE name GLOB '*son' AND age >= 18
 const rows = db
   .In("test t")
   .Find({
     name: { $glob: "*son" },
     age: { $gte: 18 },
-  }) // SELECT * FROM test t WHERE name GLOB '*son' AND age >= 18
+  })
   .As(UserDTO) // Map every row to DTO
-  .Rows(); // Get iterator
+  .Rows();
 
 for await (const row of rows) {
   console.log(row); // Jason, Jackson
 }
 ```
+
+## Internals
+
+The client uses a light builder and messagepack over http. It is relatively easy to implement a client in any language see [the docs](./doc/)
+
+## License
+
+While in alpha stage the project is free for research purposes.
+Later it will be released under MIT-like license with AI/dataset exclusion terms.

binding/dal.cc

@@ -7,17 +7,34 @@ static void _InitSQLite(const Napi::CallbackInfo& args) {
   InitSQLite(charstr);
 }
 
-static Napi::Value Handle(const Napi::CallbackInfo& args) {
+static Napi::Object RowIterator(const Napi::CallbackInfo& args) {
+  Napi::Env env = args.Env();
+  Napi::Object it = Napi::Object::New( env );
   Napi::Buffer<uint8_t> buf = args[0].As<Napi::Buffer<uint8_t>>();
   GoSlice input = {reinterpret_cast<void *>(buf.Data()), long(buf.Length()), long(buf.Length())};
-  GoSlice result = HandleQuery(input);
+  int iter = CreateRowIterator(input);
-  return Napi::Buffer<char>::Copy(args.Env(), reinterpret_cast<char *>(result.data), result.len);
+  auto next_row = [=](const Napi::CallbackInfo& a){
+    void* next = NextRow(iter);
+    if (next == nullptr) {
+      FreeIter(iter);
+      return Napi::Buffer<uint8_t>::New(env, 0);
+    }
+    Napi::Buffer<uint8_t> val = Napi::Buffer<uint8_t>::Copy(env, reinterpret_cast<uint8_t *>(next), GetLen(iter));
+    free(next);
+    return val;
+  };
+  auto free_iter = [=](const Napi::CallbackInfo& a){
+    FreeIter(iter);
+  };
+  it.Set("next", Napi::Function::New(env, next_row));
+  it.Set("free", Napi::Function::New(env, free_iter));
+  return it;
 }
 
 
 static Napi::Object Init(Napi::Env env, Napi::Object exports) {
   exports["InitSQLite"] = Napi::Function::New(env, _InitSQLite);
-  exports["Handle"] = Napi::Function::New(env, Handle);
+  exports["RowIterator"] = Napi::Function::New(env, RowIterator);
   return exports;
 }
 

binding/dal.go

@@ -1,25 +1,64 @@
 package main
 
+// #include <stdlib.h>
+// #include <stdio.h>
 import "C"
 import (
+	"fmt"
 	"strings"
+	"unsafe"
 
 	"github.com/nesterow/dal/pkg/facade"
 
 	_ "github.com/mattn/go-sqlite3"
 )
 
+var iterators = make(map[int]*facade.RowsIter)
+var itersize = make(map[int]C.int)
+
 //export InitSQLite
 func InitSQLite(pragmas string) {
 	pragmasArray := strings.Split(pragmas, ";")
 	facade.InitSQLite(pragmasArray)
 }
 
-//export HandleQuery
+//export CreateRowIterator
-func HandleQuery(input []byte) []byte {
+func CreateRowIterator(input []byte) C.int {
-	var out []byte
+	var it = &facade.RowsIter{}
-	facade.HandleQuery(&input, &out)
+	it.Exec(input)
-	return out
+	ptr := C.int(len(iterators))
+	iterators[len(iterators)] = it
+	defer fmt.Println(ptr)
+	return ptr
+}
+
+//export NextRow
+func NextRow(itid C.int) unsafe.Pointer {
+	it := iterators[int(itid)]
+	if it.Result != nil {
+		itersize[int(itid)] = C.int(len(it.Result))
+		return C.CBytes(it.Result)
+	}
+	data := it.Next()
+	if data == nil {
+		return nil
+	}
+	itersize[int(itid)] = C.int(len(data))
+	res := C.CBytes(data)
+	return res
+}
+
+//export GetLen
+func GetLen(idx C.int) C.int {
+	return itersize[int(idx)]
+}
+
+//export FreeIter
+func FreeIter(itid C.int) {
+	it := iterators[int(itid)]
+	it.Close()
+	delete(iterators, int(itid))
+	delete(itersize, int(itid))
 }
 
 func main() {}

binding/dal.h

@@ -19,6 +19,11 @@ typedef struct { const char *p; ptrdiff_t n; } _GoString_;
 /* Start of preamble from import "C" comments.  */
 
 
+#line 3 "dal.go"
+ #include <stdlib.h>
+ #include <stdio.h>
+
+#line 1 "cgo-generated-wrapper"
 
 
 /* End of preamble from import "C" comments.  */
@@ -75,7 +80,10 @@ extern "C" {
 #endif
 
 extern void InitSQLite(GoString pragmas);
-extern GoSlice HandleQuery(GoSlice input);
+extern int CreateRowIterator(GoSlice input);
+extern void* NextRow(int itid);
+extern int GetLen(int idx);
+extern void FreeIter(int itid);
 
 #ifdef __cplusplus
 }

binding/go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 // indirect
-	github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181 // indirect
+	github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894 // indirect
 	github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42 // indirect
 	github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect

binding/makefile

@@ -0,0 +1,5 @@
+build-lib:
+	CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o ../lib/libdal-arm64.dylib -buildmode=c-shared ./dal.go
+	for arch in amd64 arm64; do\
+		CC=x86_64-unknown-linux-gnu-gcc CGO_ENABLED=1 GOOS=linux GOARCH=$${arch} go build -o ../lib/libdal-$${arch}.so -buildmode=c-shared ./dal.go;\
+	done

dal/Binding.ts

@@ -1,7 +1,11 @@
 import { createRequire } from "node:module";
 const require = createRequire(import.meta.url);
+type RowIterator = {
+  next: () => Buffer;
+  free: () => void;
+};
 type SQLite = {
   InitSQLite: (pragmas: Buffer) => void;
-  Handle: (input: Buffer) => Buffer;
+  RowIterator: (input: Buffer) => RowIterator;
 };
 export default require("../build/Release/dal.node") as SQLite;

dal/__test__/bench.node.cjs

@@ -0,0 +1,88 @@
+const fs = require("fs");
+const dal = require("../../build/Release/dal.node");
+
+const Mb = (num) => Math.round(num / 1024 / 1024);
+
+class Stats {
+  constructor() {
+    this.calls = 0;
+    this.avg_rss = 0;
+    this.avg_heapTotal = 0;
+    this.avg_heapUsed = 0;
+    this.avg_external = 0;
+  }
+  add(mem) {
+    this.calls++;
+    this.avg_rss += mem.rss;
+    this.avg_heapTotal += mem.heapTotal;
+    this.avg_heapUsed += mem.heapUsed;
+    this.avg_external += mem.external;
+  }
+  avg() {
+    const n = this.calls;
+    this.avg_rss /= n;
+    this.avg_heapTotal /= n;
+    this.avg_heapUsed /= n;
+    this.avg_external /= n;
+  }
+  print() {
+    console.log(`
+AVERAGE:
+rss: ${Mb(this.avg_rss)} Mb
+external: ${Mb(this.avg_external)} Mb
+buffers: ${Mb(this.avg_heapUsed)} Mb
+total: ${Mb(this.avg_heapTotal)} Mb`);
+  }
+}
+
+
+const stats = new Stats();
+let prevMem = process.memoryUsage();
+stats.add(prevMem);
+
+
+function MEM(when = "") {
+  const mem = process.memoryUsage();
+  stats.add(mem);
+  console.log(`
+${when}
+rss: ${Mb(mem.rss)} Mb [delta> ${mem.rss - prevMem.rss}] 
+external: ${Mb(mem.external)} Mb [delta> ${mem.external - prevMem.external}] 
+buffers: ${Mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}]
+total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`);
+}
+
+
+console.time("Time to end");
+MEM("START");
+
+const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
+
+const Iterator = dal.RowIterator(buf);
+MEM("AFTER INIT");
+
+let dataTransferedBytes = 0;
+for (let i = 0; i < 100000000; i++) {
+  const b = Iterator.next();
+  if (b.length === 0) {
+    break;
+  }
+  dataTransferedBytes += b.length;
+  if (i % 1000000 === 0) {
+    MEM(`ITERATION ${i}`);
+  }
+}
+
+MEM("AFTER ITERATION");
+
+Iterator.free();
+MEM("AFTER CLEANUP");
+
+console.log("\nData transfered: ", Mb(dataTransferedBytes), "Mb");
+console.timeEnd("Time to end");
+
+setTimeout(() => {
+  MEM("AFTER SOME TIME");
+  stats.avg();
+  stats.print();
+}, 30000);

dal/__test__/builder.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "bun:test";
-import DAL from "../Builder";
+import { DAL } from "..";
 
 const options = {
   database: "test.sqlite",
@@ -22,7 +22,7 @@ test("Rows iter, no format", async () => {
     })
     .Rows<any[]>();
   for await (const row of rows) {
-    console.log(row);
+    //console.log(row);
     expect(row.length).toBe(3);
   }
   expect(true).toBe(true);
@@ -32,7 +32,7 @@ test("Rows iter, format", async () => {
   const dal = new DAL(options);
   const rows = dal.In("test t").Find({}).As(DTO).Rows();
   for await (const row of rows) {
-    console.log(row);
+    //console.log(row);
     //expect(row.id).toBe(1);
   }
   expect(true).toBe(true);

dal/__test__/sqlite.node.cjs

@@ -1,6 +0,0 @@
-const fs = require("fs");
-const dal = require("../../build/Release/dal.node");
-dal.InitSQLite(Buffer.from([]));
-const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
-data = dal.Handle(buf);
-console.log(data);

dal/__test__/srv/go.mod

@@ -2,6 +2,8 @@ module srv
 
 go 1.22.6
 
+replace github.com/nesterow/dal/pkg/handler => ../../../pkg/handler
+
 require (
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181
@@ -9,10 +11,10 @@ require (
 )
 
 require (
-	github.com/nesterow/dal/pkg/builder v0.0.0-20240820191021-7dcd046e6ca0 // indirect
+	github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 // indirect
-	github.com/nesterow/dal/pkg/filters v0.0.0-20240820191021-7dcd046e6ca0 // indirect
+	github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 // indirect
-	github.com/nesterow/dal/pkg/proto v0.0.0-20240820191021-7dcd046e6ca0 // indirect
+	github.com/nesterow/dal/pkg/proto v0.0.0-20240820192515-7a408c994181 // indirect
-	github.com/nesterow/dal/pkg/utils v0.0.0-20240820191021-7dcd046e6ca0 // indirect
+	github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tinylib/msgp v1.2.0 // indirect

dal/__test__/srv/go.sum

@@ -2,16 +2,14 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181 h1:wwWMAG/Uu/layF1wu6dJiUej0zisaVQRfyTNqjJpxac=
 github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181/go.mod h1:TCkoNn14Hgl+iVENr73GHaDVEuAg8KizH0oXaV2qMKs=
-github.com/nesterow/dal/pkg/builder v0.0.0-20240820191021-7dcd046e6ca0 h1:S37CEWRRLdeLA3V8J4XEa8CmmO4Z+auifYf1GfNRo64=
+github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 h1:Ex2XjUB7o4IRSWcgh8qgac8JQ2gbn6M+jUfom4zZccY=
-github.com/nesterow/dal/pkg/builder v0.0.0-20240820191021-7dcd046e6ca0/go.mod h1:EmozdaZEPYE2+4qpK1cw0eIS8yx9LcrVRKstvq2dgCg=
+github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181/go.mod h1:mVt1QOFEeGLdi+ZbMLzVw3LVtFB2tYWxZRouhkOThJU=
-github.com/nesterow/dal/pkg/filters v0.0.0-20240820191021-7dcd046e6ca0 h1:2CDPDKwR+PSGCI6dMbSqI5Zf8Ugg9nY2CMN87BDZ1Xg=
+github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 h1:Vi6v1T68/oayGfoX8k/FpgvfMsCUqxMsowTHKXY70VE=
-github.com/nesterow/dal/pkg/filters v0.0.0-20240820191021-7dcd046e6ca0/go.mod h1:1wKJbE/13P756EjkLU8xzbdhR+dJYfn6Vlw4J2jFJaA=
+github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181/go.mod h1:6aUr+2+D+184FTSXgziW6r1bvcKTY+Ie3TW4iET0g6I=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181 h1:dAzXf3pEYEKTF+WHJxmfy81XLPJ7BcFy3MEYYsLeyQE=
+github.com/nesterow/dal/pkg/proto v0.0.0-20240820192515-7a408c994181 h1:HcGM9fagbbGa0r/6TRrg+lzRbW79sOr6yTczYkO1miw=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181/go.mod h1:yxMmRLpnj8PmBylCcVr0PJwpy2uLwNjsMFk36kGNWOE=
+github.com/nesterow/dal/pkg/proto v0.0.0-20240820192515-7a408c994181/go.mod h1:MUS8KEmlNqjUuhNyK4TOkUwjT9893jmNm++ukVz2hzU=
-github.com/nesterow/dal/pkg/proto v0.0.0-20240820191021-7dcd046e6ca0 h1:p2VAOo7aMmFehU17S7aZ3tiwIMEjQUuzLHTNO4jQHWA=
+github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 h1:b/D2uq18D6CzAh+NUyD1cM7s/q/aQZEpXy57iKyzg8k=
-github.com/nesterow/dal/pkg/proto v0.0.0-20240820191021-7dcd046e6ca0/go.mod h1:okcJheB01HZrsNryT2Hun1DjYI7Hj/+/oD6+4PVW3TU=
+github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181/go.mod h1:0XWmrxZfyoEAZoCOxP3quEOP7ZNK2Lhvtpeux8C74mw=
-github.com/nesterow/dal/pkg/utils v0.0.0-20240820191021-7dcd046e6ca0 h1:CYDcRDYrXXD8fQdkXbbPMJvvLnBy4nI36BrRMQEsWUo=
-github.com/nesterow/dal/pkg/utils v0.0.0-20240820191021-7dcd046e6ca0/go.mod h1:0XWmrxZfyoEAZoCOxP3quEOP7ZNK2Lhvtpeux8C74mw=
 github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
 github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

dal/bun.ffi.ts

@@ -0,0 +1,41 @@
+import { dlopen, FFIType, suffix, ptr } from "bun:ffi";
+import { join } from "node:path";
+import fs from "node:fs";
+
+// TODO: build a shared library compatible with cffi
+const libname = `libdal-arm64.${suffix}`;
+const libpath = join(__dirname, '..', 'lib', libname);
+
+const {
+    symbols: {
+        InitSQLite,
+        CreateRowIterator,
+        NextRow,
+        GetLen,
+        FreeIter,
+    },
+  } = dlopen(
+    libpath,
+    {
+      InitSQLite: {
+        args: [ FFIType.cstring ],
+        returns: FFIType.void,
+      },
+      CreateRowIterator: {
+        args: [ FFIType.cstring ],
+        returns: FFIType.i32,
+      },
+      NextRow: {
+        args: [FFIType.i32],
+        returns: FFIType.cstring,
+      },
+      GetLen: {
+        args: [FFIType.i32],
+        returns: FFIType.i32,
+      },
+      FreeIter : {
+        args: [FFIType.i32],
+        returns: FFIType.void,
+      },
+    },
+  );

dal/index.ts

@@ -1,2 +1 @@
 export { default as DAL } from "./Builder";
-export { default as DALite } from "./CBuilder";

dal/node.napi.ts

@@ -18,14 +18,19 @@ export default class CBuilder<
     super({ database: opts.database, url: "" });
   }
   /**
-   * Not really an iterator, since addonn allocates memory for all rows
+   * TODO: handle responses
-   * but returns an iterator
    */
   async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
     this.formatRequest();
     const req = Buffer.from(encodeRequest(this.request));
-    const response = Binding.Handle(req);
+    const iter = Binding.RowIterator(req);
+    for (;;) {
+      const response = iter.next();
       const rows = decodeRows(response);
+      if (rows.length === 0) {
+        iter.free();
+        break;
+      }
       for (const row of rows) {
         if (this.headerRow === null) {
           this.headerRow = row.r;
@@ -34,6 +39,7 @@ export default class CBuilder<
         yield this.formatRow(row.r);
       }
     }
+  }
   async Query<T = InstanceType<I>>(): Promise<T[]> {
     const rows = this.Rows();
     const result: T[] = [];
@@ -45,7 +51,8 @@ export default class CBuilder<
   async Exec(): Promise<ExecResult> {
     this.formatRequest();
     const req = Buffer.from(encodeRequest(this.request));
-    const response = Binding.Handle(req);
+    const iter = Binding.RowIterator(req);
+    const response = iter.next();
     return decodeResponse(response);
   }
 }

go.mod

@@ -2,7 +2,7 @@ module github.com/nesterow/dal
 
 go 1.22.6
 
-require github.com/nesterow/dal/pkg/facade v0.0.0-20240821202841-36d7d315e298
+require github.com/nesterow/dal/pkg/facade v0.0.0-20240823050743-ae641b7ee894
 
 require github.com/mattn/go-sqlite3 v1.14.22
 
@@ -10,7 +10,7 @@ require (
 	github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 // indirect
-	github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181 // indirect
+	github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894 // indirect
 	github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42 // indirect
 	github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 // indirect
 	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect

go.sum

@@ -4,12 +4,12 @@ github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181 h1:wwWMAG
 github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181/go.mod h1:TCkoNn14Hgl+iVENr73GHaDVEuAg8KizH0oXaV2qMKs=
 github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 h1:Ex2XjUB7o4IRSWcgh8qgac8JQ2gbn6M+jUfom4zZccY=
 github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181/go.mod h1:mVt1QOFEeGLdi+ZbMLzVw3LVtFB2tYWxZRouhkOThJU=
-github.com/nesterow/dal/pkg/facade v0.0.0-20240821202841-36d7d315e298 h1:Mb2gg8lR3yn4LGeOyobCSZll4wPAg5UqfmSUT2/dTD0=
+github.com/nesterow/dal/pkg/facade v0.0.0-20240823050743-ae641b7ee894 h1:MkHuizDCwVhyvVa/9sSBFsX7MiWZQAeOODS50GTe9nU=
-github.com/nesterow/dal/pkg/facade v0.0.0-20240821202841-36d7d315e298/go.mod h1:/C5t/NOUVLCzEZJgpNPlEt0jqmEllVNUpngevE9aUBk=
+github.com/nesterow/dal/pkg/facade v0.0.0-20240823050743-ae641b7ee894/go.mod h1:MKaLSkZocQXy/wFqBzl+Bu7lb4xSCeItbTC7vjOPE0Q=
 github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 h1:Vi6v1T68/oayGfoX8k/FpgvfMsCUqxMsowTHKXY70VE=
 github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181/go.mod h1:6aUr+2+D+184FTSXgziW6r1bvcKTY+Ie3TW4iET0g6I=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181 h1:dAzXf3pEYEKTF+WHJxmfy81XLPJ7BcFy3MEYYsLeyQE=
+github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894 h1:nPlCoVDXzpFGX4cmInLOKoAB7cTpcnjGDevCahF1Db8=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181/go.mod h1:yxMmRLpnj8PmBylCcVr0PJwpy2uLwNjsMFk36kGNWOE=
+github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894/go.mod h1:aUueTAR2yb5ZCcXqgm4euJTTlScTCqTyT187oNQN5zY=
 github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42 h1:bUnae1O6MEYWEZepd2ir3iSkydfX9/8DLRoH0U+EWHM=
 github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42/go.mod h1:YGGcF6rUmoknTFR3csZyLjJhpuFBw9S+DQa61P8B8kY=
 github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 h1:b/D2uq18D6CzAh+NUyD1cM7s/q/aQZEpXy57iKyzg8k=

package.json

@@ -23,6 +23,7 @@
     "test:client": "bun test:*",
     "test:dal": "bun test dal/__test__",
     "test:serve": "cd dal/__test__/srv && go run main.go",
+    "bench:node": "node ./dal/__test__/bench.node.cjs",
     "fmt": "prettier --write .",
     "build": "tsc",
     "prepublish": "tsc",

pkg/facade/SQLIteShared.go

@@ -1,6 +1,7 @@
 package facade
 
 import (
+	"database/sql"
 	"reflect"
 
 	"github.com/nesterow/dal/pkg/adapter"
@@ -25,6 +26,80 @@ func InitSQLite(pragmas []string) {
 	}
 }
 
+type RowsIter struct {
+	Result  []byte
+	Columns []string
+	rows    *sql.Rows
+}
+
+func (r *RowsIter) Exec(input []byte) {
+	InitSQLite([]string{})
+	req := proto.Request{}
+	_, e := req.UnmarshalMsg(input)
+	query, err := req.Parse(adapter.GetDialect(db.Type))
+	if err != nil || e != nil {
+		res := proto.Response{
+			Msg: "failed to unmarshal request",
+		}
+		r.Result, _ = res.MarshalMsg(nil)
+		return
+	}
+	if query.Exec {
+		result, err := db.Exec(query)
+		if err != nil {
+			res := proto.Response{
+				Msg: err.Error(),
+			}
+			r.Result, _ = res.MarshalMsg(nil)
+			return
+		}
+		ra, _ := result.RowsAffected()
+		la, _ := result.LastInsertId()
+		res := proto.Response{
+			Id:           0,
+			RowsAffected: ra,
+			LastInsertId: la,
+		}
+		r.Result, _ = res.MarshalMsg(nil)
+		return
+	}
+	rows, err := db.Query(query)
+	if err != nil {
+		res := proto.Response{
+			Msg: err.Error(),
+		}
+		r.Result, _ = res.MarshalMsg(nil)
+		return
+	}
+	r.rows = rows
+}
+
+func (r *RowsIter) Close() {
+	if r.rows == nil {
+		return
+	}
+	r.rows.Close()
+}
+
+func (r *RowsIter) Next() []byte {
+	columns, _ := r.rows.Columns()
+	types, _ := r.rows.ColumnTypes()
+	if r.Columns == nil {
+		r.Columns = columns
+		cols, _ := proto.MarshalRow(columns)
+		return cols
+	}
+	data := make([]interface{}, len(columns))
+	r.rows.Next()
+	for i := range data {
+		typ := reflect.New(types[i].ScanType()).Interface()
+		data[i] = &typ
+	}
+	r.rows.Scan(data...)
+	cols, _ := proto.MarshalRow(data)
+	return cols
+}
+
 func HandleQuery(input *[]byte, output *[]byte) int {
 	InitSQLite([]string{})
 	req := proto.Request{}
@@ -83,6 +158,7 @@ func HandleQuery(input *[]byte, output *[]byte) int {
 		*output, _ = res.MarshalMsg(nil)
 		return 0
 	}
+	defer rows.Close()
 	columns, _ := rows.Columns()
 	types, _ := rows.ColumnTypes()
 	cols, _ := proto.MarshalRow(columns)

pkg/facade/go.mod

@@ -4,7 +4,7 @@ go 1.22.6
 
 require (
 	github.com/nesterow/dal/pkg/adapter v0.0.0-20240820192515-7a408c994181
-	github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181
+	github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894
 	github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42
 )
 

pkg/facade/go.sum

@@ -6,8 +6,8 @@ github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181 h1:Ex2XjU
 github.com/nesterow/dal/pkg/builder v0.0.0-20240820192515-7a408c994181/go.mod h1:mVt1QOFEeGLdi+ZbMLzVw3LVtFB2tYWxZRouhkOThJU=
 github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181 h1:Vi6v1T68/oayGfoX8k/FpgvfMsCUqxMsowTHKXY70VE=
 github.com/nesterow/dal/pkg/filters v0.0.0-20240820192515-7a408c994181/go.mod h1:6aUr+2+D+184FTSXgziW6r1bvcKTY+Ie3TW4iET0g6I=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181 h1:dAzXf3pEYEKTF+WHJxmfy81XLPJ7BcFy3MEYYsLeyQE=
+github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894 h1:nPlCoVDXzpFGX4cmInLOKoAB7cTpcnjGDevCahF1Db8=
-github.com/nesterow/dal/pkg/handler v0.0.0-20240820192515-7a408c994181/go.mod h1:yxMmRLpnj8PmBylCcVr0PJwpy2uLwNjsMFk36kGNWOE=
+github.com/nesterow/dal/pkg/handler v0.0.0-20240823050743-ae641b7ee894/go.mod h1:aUueTAR2yb5ZCcXqgm4euJTTlScTCqTyT187oNQN5zY=
 github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42 h1:bUnae1O6MEYWEZepd2ir3iSkydfX9/8DLRoH0U+EWHM=
 github.com/nesterow/dal/pkg/proto v0.0.0-20240820230328-f8e28d2a2e42/go.mod h1:YGGcF6rUmoknTFR3csZyLjJhpuFBw9S+DQa61P8B8kY=
 github.com/nesterow/dal/pkg/utils v0.0.0-20240820192515-7a408c994181 h1:b/D2uq18D6CzAh+NUyD1cM7s/q/aQZEpXy57iKyzg8k=

pkg/handler/query_handler.go

@@ -63,19 +63,17 @@ func QueryHandler(db adapter.DBAdapter) http.Handler {
 		}
 		defer rows.Close()
 
+		w.Header().Set("Connection", "Keep-Alive")
 		w.Header().Set("X-Content-Type-Options", "nosniff")
 		w.Header().Set("Content-Type", "application/x-msgpack")
 
-		flusher, ok := w.(http.Flusher)
-		if !ok {
-			http.Error(w, "expected http.ResponseWriter to be an http.Flusher", http.StatusInternalServerError)
-			return
-		}
 		columns, _ := rows.Columns()
 		types, _ := rows.ColumnTypes()
 		cols, _ := proto.MarshalRow(columns)
 		w.Write(cols)
-		flusher.Flush()
+
+		rc := http.NewResponseController(w)
+		rc.Flush()
 
 		for rows.Next() {
 			data := make([]interface{}, len(columns))
@@ -86,7 +84,8 @@ func QueryHandler(db adapter.DBAdapter) http.Handler {
 			rows.Scan(data...)
 			cols, _ := proto.MarshalRow(data)
 			w.Write(cols)
-			flusher.Flush()
+			rc.Flush()
 		}
+
 	})
 }

tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "files": ["dal/index.ts", "dal/Binding.ts"],
+  "files": ["dal/index.ts", "dal/Binding.ts", "dal/node.napi.ts"],
   "compilerOptions": {
     "target": "ESNext",
     "module": "ESNext",