diff --git a/binding/dal.cc b/binding/dal.cc index 9e1580a..911d415 100644 --- a/binding/dal.cc +++ b/binding/dal.cc @@ -33,8 +33,8 @@ static Napi::Object RowIterator(const Napi::CallbackInfo& args) { static Napi::Object Init(Napi::Env env, Napi::Object exports) { - exports["InitSQLite"] = Napi::Function::New(env, _InitSQLite); - exports["RowIterator"] = Napi::Function::New(env, RowIterator); + exports["initSQLite"] = Napi::Function::New(env, _InitSQLite); + exports["rowIterator"] = Napi::Function::New(env, RowIterator); return exports; } diff --git a/binding/dal.go b/binding/dal.go index af15ee9..7e3324d 100644 --- a/binding/dal.go +++ b/binding/dal.go @@ -4,7 +4,6 @@ package main // #include import "C" import ( - "fmt" "strings" "unsafe" @@ -13,8 +12,10 @@ import ( _ "github.com/mattn/go-sqlite3" ) -var iterators = make(map[int]*facade.RowsIter) -var itersize = make(map[int]C.int) +var ( + iterators = make(map[int]*facade.RowsIter) + itersize = make(map[int]C.int) +) //export InitSQLite func InitSQLite(pragmas string) { @@ -28,7 +29,6 @@ func CreateRowIterator(input []byte) C.int { it.Exec(input) ptr := C.int(len(iterators)) iterators[len(iterators)] = it - defer fmt.Println(ptr) return ptr } diff --git a/clib/clib.h b/clib/clib.h new file mode 100644 index 0000000..07af65d --- /dev/null +++ b/clib/clib.h @@ -0,0 +1,91 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "main.go" + #include + #include + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern void InitSQLite(char* params); +extern int CreateRowIterator(char* data, int size); +extern void* NextRow(int itid); +extern int GetLen(int idx); +extern void Cleanup(int itid); +extern void Free(void* ptr); + +#ifdef __cplusplus +} +#endif diff --git a/clib/go.mod b/clib/go.mod new file mode 100644 index 0000000..5b4e14a --- /dev/null +++ b/clib/go.mod @@ -0,0 +1,16 @@ +module l12.xyz/x/dal/clib + +go 1.22.6 + +require ( + github.com/mattn/go-sqlite3 v1.14.22 + l12.xyz/x/dal v0.0.0 +) + +require ( + github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/tinylib/msgp v1.2.0 // indirect +) + +replace l12.xyz/x/dal => ../ diff --git a/clib/go.sum b/clib/go.sum new file mode 100644 index 0000000..5cf32ca --- /dev/null +++ b/clib/go.sum @@ -0,0 +1,8 @@ +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4= +github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY= +github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro= diff --git a/clib/main.go b/clib/main.go new file mode 100644 index 0000000..155e64f --- /dev/null +++ b/clib/main.go @@ -0,0 +1,74 @@ +package main + +// #include +// #include +import "C" + +import ( + "fmt" + "strings" + "unsafe" + + "l12.xyz/x/dal/pkg/facade" + + _ "github.com/mattn/go-sqlite3" +) + +var ( + iterators = make(map[int]*facade.RowsIter) + itersize = make(map[int]C.int) +) + +//export InitSQLite +func InitSQLite(params *C.char) { + pragmas := C.GoString(params) + pragmasArray := strings.Split(pragmas, ";") + facade.InitSQLite(pragmasArray) +} + +//export CreateRowIterator +func CreateRowIterator(data *C.char, size C.int) C.int { + var it = &facade.RowsIter{} + input := C.GoBytes(unsafe.Pointer(data), size) + fmt.Println("input", input) + it.Exec(input) + ptr := C.int(len(iterators)) + iterators[len(iterators)] = it + return ptr +} + +//export NextRow +func NextRow(itid C.int) unsafe.Pointer { + it := iterators[int(itid)] + if it.Result != nil { + itersize[int(itid)] = C.int(len(it.Result)) + return C.CBytes(it.Result) + } + data := it.Next() + if data == nil { + return nil + } + itersize[int(itid)] = C.int(len(data)) + res := C.CBytes(data) + return res +} + +//export GetLen +func GetLen(idx C.int) C.int { + return itersize[int(idx)] +} + +//export Cleanup +func Cleanup(itid C.int) { + it := iterators[int(itid)] + it.Close() + delete(iterators, int(itid)) + delete(itersize, int(itid)) +} + +//export Free +func Free(ptr unsafe.Pointer) { + C.free(ptr) +} + +func main() {} diff --git a/dal/Binding.ts b/dal/Binding.ts index 45fe8ba..f58abf6 100644 --- a/dal/Binding.ts +++ b/dal/Binding.ts @@ -5,7 +5,7 @@ type RowIterator = { free: () => void; }; type SQLite = { - InitSQLite: (pragmas: Buffer) => void; - RowIterator: (input: Buffer) => RowIterator; + initSQLite: (pragmas: Buffer) => void; + rowIterator: (input: Buffer) => RowIterator; }; export default require("../build/Release/dal.node") as SQLite; diff --git a/dal/Bunding.ts b/dal/Bunding.ts new file mode 100644 index 0000000..dc81525 --- /dev/null +++ b/dal/Bunding.ts @@ -0,0 +1,68 @@ +/** + * This file is responsible for binding the C library to the Bun runtime. + */ +import { dlopen, FFIType, suffix, ptr, toBuffer } from "bun:ffi"; + +const libname = `lib/clib.${suffix}`; +const libpath = libname; + +const { + symbols: { InitSQLite, CreateRowIterator, NextRow, GetLen, Free, Cleanup }, +} = dlopen(libpath, { + InitSQLite: { + args: [FFIType.cstring], + returns: FFIType.void, + }, + CreateRowIterator: { + args: [FFIType.cstring, FFIType.i32], + returns: FFIType.i32, + }, + NextRow: { + args: [FFIType.i32], + returns: FFIType.ptr, + }, + GetLen: { + args: [FFIType.i32], + returns: FFIType.i32, + }, + Free: { + args: [FFIType.ptr], + returns: FFIType.void, + }, + Cleanup: { + args: [FFIType.i32], + returns: FFIType.void, + }, +}); + +function initSQLite(pragmas: string) { + const buf = Buffer.from(pragmas); + InitSQLite(ptr(buf)); +} + +function rowIterator(buf: Buffer) { + const iter = CreateRowIterator(ptr(buf), buf.length); + const next = () => { + const pointer = NextRow(iter); + if (pointer === null) { + return null; + } + const buf = toBuffer(pointer, 0, GetLen(iter)); + //Free(pointer) //should be resolved by GC; + return buf; + }; + + const cleanup = () => { + Cleanup(iter); + }; + + return { + next, + cleanup, + }; +} + +export default { + initSQLite, + rowIterator, +}; diff --git a/dal/__test__/bench.node.cjs b/dal/__test__/bench.node.cjs index ded5e27..5815283 100644 --- a/dal/__test__/bench.node.cjs +++ b/dal/__test__/bench.node.cjs @@ -35,12 +35,10 @@ total: ${Mb(this.avg_heapTotal)} Mb`); } } - const stats = new Stats(); let prevMem = process.memoryUsage(); stats.add(prevMem); - function MEM(when = "") { const mem = process.memoryUsage(); stats.add(mem); @@ -52,7 +50,6 @@ buffers: ${Mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}] total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`); } - console.time("Time to end"); MEM("START"); diff --git a/dal/bun.cffi.ts b/dal/bun.cffi.ts new file mode 100644 index 0000000..904b582 --- /dev/null +++ b/dal/bun.cffi.ts @@ -0,0 +1,65 @@ +import Builder from "./Builder"; +import Bunding from "./Bunding"; +import { encodeRequest, decodeRows, decodeResponse } from "./Protocol"; +import type { ExecResult } from "./Protocol"; + +type Options = { + database: string; +}; + +/** + * Allows to use SQLite databases in BunJS + */ +export default class CBuilder< + I extends abstract new (...args: any) => any, +> extends Builder { + constructor(opts: Options) { + super({ database: opts.database, url: "" }); + } + /** + * TODO: handle responses + */ + async *Rows>(): AsyncGenerator { + this.formatRequest(); + const req = Buffer.from(encodeRequest(this.request)); + const iter = Bunding.rowIterator(req); + for (;;) { + const response = iter.next(); + if (response === null) { + iter.cleanup(); + break; + } + const rows = decodeRows(response); + if (rows.length === 0) { + iter.cleanup(); + break; + } + for (const row of rows) { + if (this.headerRow === null) { + this.headerRow = row.r; + continue; + } + yield this.formatRow(row.r); + } + } + } + async Query>(): Promise { + const rows = this.Rows(); + const result: T[] = []; + for await (const row of rows) { + result.push(row); + } + return result; + } + async Exec(): Promise { + this.formatRequest(); + const req = Buffer.from(encodeRequest(this.request)); + const iter = Bunding.rowIterator(req); + const response = iter.next(); + if (response === null) { + iter.cleanup(); + throw new Error("No response"); + } + return decodeResponse(response); + } +} diff --git a/dal/bun.ffi.ts b/dal/bun.ffi.ts deleted file mode 100644 index 4d4632d..0000000 --- a/dal/bun.ffi.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { dlopen, FFIType, suffix, ptr } from "bun:ffi"; -import { join } from "node:path"; -import fs from "node:fs"; - -// TODO: build a shared library compatible with cffi -const libname = `libdal-arm64.${suffix}`; -const libpath = join(__dirname, '..', 'lib', libname); - -const { - symbols: { - InitSQLite, - CreateRowIterator, - NextRow, - GetLen, - FreeIter, - }, - } = dlopen( - libpath, - { - InitSQLite: { - args: [ FFIType.cstring ], - returns: FFIType.void, - }, - CreateRowIterator: { - args: [ FFIType.cstring ], - returns: FFIType.i32, - }, - NextRow: { - args: [FFIType.i32], - returns: FFIType.cstring, - }, - GetLen: { - args: [FFIType.i32], - returns: FFIType.i32, - }, - FreeIter : { - args: [FFIType.i32], - returns: FFIType.void, - }, - }, - ); diff --git a/dal/node.napi.ts b/dal/node.napi.ts index 0a872dd..9efe9f3 100644 --- a/dal/node.napi.ts +++ b/dal/node.napi.ts @@ -9,7 +9,6 @@ type Options = { /** * Allows to use SQLite databases in a NodeJS process. - * It is less memory-efficient than a seaparate server, and uses absolute path for database name. */ export default class CBuilder< I extends abstract new (...args: any) => any, @@ -23,7 +22,7 @@ export default class CBuilder< async *Rows>(): AsyncGenerator { this.formatRequest(); const req = Buffer.from(encodeRequest(this.request)); - const iter = Binding.RowIterator(req); + const iter = Binding.rowIterator(req); for (;;) { const response = iter.next(); const rows = decodeRows(response); @@ -51,7 +50,7 @@ export default class CBuilder< async Exec(): Promise { this.formatRequest(); const req = Buffer.from(encodeRequest(this.request)); - const iter = Binding.RowIterator(req); + const iter = Binding.rowIterator(req); const response = iter.next(); return decodeResponse(response); }