[feat] cffi

This commit is contained in:
Anton Nesterov 2024-08-29 22:13:48 +02:00
parent 103b7dac11
commit dad8d46e9b
No known key found for this signature in database
GPG key ID: 59121E8AE2851FB5
12 changed files with 332 additions and 55 deletions

View file

@ -33,8 +33,8 @@ static Napi::Object RowIterator(const Napi::CallbackInfo& args) {
static Napi::Object Init(Napi::Env env, Napi::Object exports) { static Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports["InitSQLite"] = Napi::Function::New(env, _InitSQLite); exports["initSQLite"] = Napi::Function::New(env, _InitSQLite);
exports["RowIterator"] = Napi::Function::New(env, RowIterator); exports["rowIterator"] = Napi::Function::New(env, RowIterator);
return exports; return exports;
} }

View file

@ -4,7 +4,6 @@ package main
// #include <stdio.h> // #include <stdio.h>
import "C" import "C"
import ( import (
"fmt"
"strings" "strings"
"unsafe" "unsafe"
@ -13,8 +12,10 @@ import (
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
) )
var iterators = make(map[int]*facade.RowsIter) var (
var itersize = make(map[int]C.int) iterators = make(map[int]*facade.RowsIter)
itersize = make(map[int]C.int)
)
//export InitSQLite //export InitSQLite
func InitSQLite(pragmas string) { func InitSQLite(pragmas string) {
@ -28,7 +29,6 @@ func CreateRowIterator(input []byte) C.int {
it.Exec(input) it.Exec(input)
ptr := C.int(len(iterators)) ptr := C.int(len(iterators))
iterators[len(iterators)] = it iterators[len(iterators)] = it
defer fmt.Println(ptr)
return ptr return ptr
} }

91
clib/clib.h Normal file
View file

@ -0,0 +1,91 @@
/* Code generated by cmd/cgo; DO NOT EDIT. */
/* package command-line-arguments */
#line 1 "cgo-builtin-export-prolog"
#include <stddef.h>
#ifndef GO_CGO_EXPORT_PROLOGUE_H
#define GO_CGO_EXPORT_PROLOGUE_H
#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef struct { const char *p; ptrdiff_t n; } _GoString_;
#endif
#endif
/* Start of preamble from import "C" comments. */
#line 3 "main.go"
#include <stdlib.h>
#include <stdio.h>
#line 1 "cgo-generated-wrapper"
/* End of preamble from import "C" comments. */
/* Start of boilerplate cgo prologue. */
#line 1 "cgo-gcc-export-header-prolog"
#ifndef GO_CGO_PROLOGUE_H
#define GO_CGO_PROLOGUE_H
typedef signed char GoInt8;
typedef unsigned char GoUint8;
typedef short GoInt16;
typedef unsigned short GoUint16;
typedef int GoInt32;
typedef unsigned int GoUint32;
typedef long long GoInt64;
typedef unsigned long long GoUint64;
typedef GoInt64 GoInt;
typedef GoUint64 GoUint;
typedef size_t GoUintptr;
typedef float GoFloat32;
typedef double GoFloat64;
#ifdef _MSC_VER
#include <complex.h>
typedef _Fcomplex GoComplex64;
typedef _Dcomplex GoComplex128;
#else
typedef float _Complex GoComplex64;
typedef double _Complex GoComplex128;
#endif
/*
static assertion to make sure the file is being used on architecture
at least with matching size of GoInt.
*/
typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];
#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef _GoString_ GoString;
#endif
typedef void *GoMap;
typedef void *GoChan;
typedef struct { void *t; void *v; } GoInterface;
typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;
#endif
/* End of boilerplate cgo prologue. */
#ifdef __cplusplus
extern "C" {
#endif
extern void InitSQLite(char* params);
extern int CreateRowIterator(char* data, int size);
extern void* NextRow(int itid);
extern int GetLen(int idx);
extern void Cleanup(int itid);
extern void Free(void* ptr);
#ifdef __cplusplus
}
#endif

16
clib/go.mod Normal file
View file

@ -0,0 +1,16 @@
module l12.xyz/x/dal/clib
go 1.22.6
require (
github.com/mattn/go-sqlite3 v1.14.22
l12.xyz/x/dal v0.0.0
)
require (
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/tinylib/msgp v1.2.0 // indirect
)
replace l12.xyz/x/dal => ../

8
clib/go.sum Normal file
View file

@ -0,0 +1,8 @@
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=

74
clib/main.go Normal file
View file

@ -0,0 +1,74 @@
package main
// #include <stdlib.h>
// #include <stdio.h>
import "C"
import (
"fmt"
"strings"
"unsafe"
"l12.xyz/x/dal/pkg/facade"
_ "github.com/mattn/go-sqlite3"
)
var (
iterators = make(map[int]*facade.RowsIter)
itersize = make(map[int]C.int)
)
//export InitSQLite
func InitSQLite(params *C.char) {
pragmas := C.GoString(params)
pragmasArray := strings.Split(pragmas, ";")
facade.InitSQLite(pragmasArray)
}
//export CreateRowIterator
func CreateRowIterator(data *C.char, size C.int) C.int {
var it = &facade.RowsIter{}
input := C.GoBytes(unsafe.Pointer(data), size)
fmt.Println("input", input)
it.Exec(input)
ptr := C.int(len(iterators))
iterators[len(iterators)] = it
return ptr
}
//export NextRow
func NextRow(itid C.int) unsafe.Pointer {
it := iterators[int(itid)]
if it.Result != nil {
itersize[int(itid)] = C.int(len(it.Result))
return C.CBytes(it.Result)
}
data := it.Next()
if data == nil {
return nil
}
itersize[int(itid)] = C.int(len(data))
res := C.CBytes(data)
return res
}
//export GetLen
func GetLen(idx C.int) C.int {
return itersize[int(idx)]
}
//export Cleanup
func Cleanup(itid C.int) {
it := iterators[int(itid)]
it.Close()
delete(iterators, int(itid))
delete(itersize, int(itid))
}
//export Free
func Free(ptr unsafe.Pointer) {
C.free(ptr)
}
func main() {}

View file

@ -5,7 +5,7 @@ type RowIterator = {
free: () => void; free: () => void;
}; };
type SQLite = { type SQLite = {
InitSQLite: (pragmas: Buffer) => void; initSQLite: (pragmas: Buffer) => void;
RowIterator: (input: Buffer) => RowIterator; rowIterator: (input: Buffer) => RowIterator;
}; };
export default require("../build/Release/dal.node") as SQLite; export default require("../build/Release/dal.node") as SQLite;

68
dal/Bunding.ts Normal file
View file

@ -0,0 +1,68 @@
/**
* This file is responsible for binding the C library to the Bun runtime.
*/
import { dlopen, FFIType, suffix, ptr, toBuffer } from "bun:ffi";
const libname = `lib/clib.${suffix}`;
const libpath = libname;
const {
symbols: { InitSQLite, CreateRowIterator, NextRow, GetLen, Free, Cleanup },
} = dlopen(libpath, {
InitSQLite: {
args: [FFIType.cstring],
returns: FFIType.void,
},
CreateRowIterator: {
args: [FFIType.cstring, FFIType.i32],
returns: FFIType.i32,
},
NextRow: {
args: [FFIType.i32],
returns: FFIType.ptr,
},
GetLen: {
args: [FFIType.i32],
returns: FFIType.i32,
},
Free: {
args: [FFIType.ptr],
returns: FFIType.void,
},
Cleanup: {
args: [FFIType.i32],
returns: FFIType.void,
},
});
function initSQLite(pragmas: string) {
const buf = Buffer.from(pragmas);
InitSQLite(ptr(buf));
}
function rowIterator(buf: Buffer) {
const iter = CreateRowIterator(ptr(buf), buf.length);
const next = () => {
const pointer = NextRow(iter);
if (pointer === null) {
return null;
}
const buf = toBuffer(pointer, 0, GetLen(iter));
//Free(pointer) //should be resolved by GC;
return buf;
};
const cleanup = () => {
Cleanup(iter);
};
return {
next,
cleanup,
};
}
export default {
initSQLite,
rowIterator,
};

View file

@ -35,12 +35,10 @@ total: ${Mb(this.avg_heapTotal)} Mb`);
} }
} }
const stats = new Stats(); const stats = new Stats();
let prevMem = process.memoryUsage(); let prevMem = process.memoryUsage();
stats.add(prevMem); stats.add(prevMem);
function MEM(when = "") { function MEM(when = "") {
const mem = process.memoryUsage(); const mem = process.memoryUsage();
stats.add(mem); stats.add(mem);
@ -52,7 +50,6 @@ buffers: ${Mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}]
total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`); total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`);
} }
console.time("Time to end"); console.time("Time to end");
MEM("START"); MEM("START");

65
dal/bun.cffi.ts Normal file
View file

@ -0,0 +1,65 @@
import Builder from "./Builder";
import Bunding from "./Bunding";
import { encodeRequest, decodeRows, decodeResponse } from "./Protocol";
import type { ExecResult } from "./Protocol";
type Options = {
database: string;
};
/**
* Allows to use SQLite databases in BunJS
*/
export default class CBuilder<
I extends abstract new (...args: any) => any,
> extends Builder<I> {
constructor(opts: Options) {
super({ database: opts.database, url: "" });
}
/**
* TODO: handle responses
*/
async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
this.formatRequest();
const req = Buffer.from(encodeRequest(this.request));
const iter = Bunding.rowIterator(req);
for (;;) {
const response = iter.next();
if (response === null) {
iter.cleanup();
break;
}
const rows = decodeRows(response);
if (rows.length === 0) {
iter.cleanup();
break;
}
for (const row of rows) {
if (this.headerRow === null) {
this.headerRow = row.r;
continue;
}
yield this.formatRow(row.r);
}
}
}
async Query<T = InstanceType<I>>(): Promise<T[]> {
const rows = this.Rows();
const result: T[] = [];
for await (const row of rows) {
result.push(row);
}
return result;
}
async Exec(): Promise<ExecResult> {
this.formatRequest();
const req = Buffer.from(encodeRequest(this.request));
const iter = Bunding.rowIterator(req);
const response = iter.next();
if (response === null) {
iter.cleanup();
throw new Error("No response");
}
return decodeResponse(response);
}
}

View file

@ -1,41 +0,0 @@
import { dlopen, FFIType, suffix, ptr } from "bun:ffi";
import { join } from "node:path";
import fs from "node:fs";
// TODO: build a shared library compatible with cffi
const libname = `libdal-arm64.${suffix}`;
const libpath = join(__dirname, '..', 'lib', libname);
const {
symbols: {
InitSQLite,
CreateRowIterator,
NextRow,
GetLen,
FreeIter,
},
} = dlopen(
libpath,
{
InitSQLite: {
args: [ FFIType.cstring ],
returns: FFIType.void,
},
CreateRowIterator: {
args: [ FFIType.cstring ],
returns: FFIType.i32,
},
NextRow: {
args: [FFIType.i32],
returns: FFIType.cstring,
},
GetLen: {
args: [FFIType.i32],
returns: FFIType.i32,
},
FreeIter : {
args: [FFIType.i32],
returns: FFIType.void,
},
},
);

View file

@ -9,7 +9,6 @@ type Options = {
/** /**
* Allows to use SQLite databases in a NodeJS process. * Allows to use SQLite databases in a NodeJS process.
* It is less memory-efficient than a seaparate server, and uses absolute path for database name.
*/ */
export default class CBuilder< export default class CBuilder<
I extends abstract new (...args: any) => any, I extends abstract new (...args: any) => any,
@ -23,7 +22,7 @@ export default class CBuilder<
async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> { async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
this.formatRequest(); this.formatRequest();
const req = Buffer.from(encodeRequest(this.request)); const req = Buffer.from(encodeRequest(this.request));
const iter = Binding.RowIterator(req); const iter = Binding.rowIterator(req);
for (;;) { for (;;) {
const response = iter.next(); const response = iter.next();
const rows = decodeRows(response); const rows = decodeRows(response);
@ -51,7 +50,7 @@ export default class CBuilder<
async Exec(): Promise<ExecResult> { async Exec(): Promise<ExecResult> {
this.formatRequest(); this.formatRequest();
const req = Buffer.from(encodeRequest(this.request)); const req = Buffer.from(encodeRequest(this.request));
const iter = Binding.RowIterator(req); const iter = Binding.rowIterator(req);
const response = iter.next(); const response = iter.next();
return decodeResponse(response); return decodeResponse(response);
} }