From ddc2e1ba0e4d659db1d31cf053254dff686086d0 Mon Sep 17 00:00:00 2001 From: Anton Nesterov Date: Fri, 30 Aug 2024 08:53:32 +0200 Subject: [PATCH] [ref] ts client; [fix] free mem Bun ffi --- {dal => client}/Binding.ts | 0 {dal => client}/Builder.ts | 2 +- {dal => client}/Bunding.ts | 9 +-- client/Library.ts | 21 ++++++ {dal => client}/Protocol.ts | 0 client/__test__/bench.bun.ts | 90 ++++++++++++++++++++++++ {dal => client}/__test__/bench.node.cjs | 4 +- {dal => client}/__test__/builder.test.ts | 0 {dal => client}/__test__/srv/go.mod | 0 {dal => client}/__test__/srv/go.sum | 0 {dal => client}/__test__/srv/main.go | 0 {dal => client}/index.ts | 0 dal/node.napi.ts => client/libdal.ts | 2 +- {dal => client}/readme.md | 0 dal/bun.cffi.ts | 65 ----------------- package.json | 7 +- pkg/facade/SQLIteShared.go | 76 -------------------- tsconfig.json | 2 +- 18 files changed, 125 insertions(+), 153 deletions(-) rename {dal => client}/Binding.ts (100%) rename {dal => client}/Builder.ts (99%) rename {dal => client}/Bunding.ts (86%) create mode 100644 client/Library.ts rename {dal => client}/Protocol.ts (100%) create mode 100644 client/__test__/bench.bun.ts rename {dal => client}/__test__/bench.node.cjs (96%) rename {dal => client}/__test__/builder.test.ts (100%) rename {dal => client}/__test__/srv/go.mod (100%) rename {dal => client}/__test__/srv/go.sum (100%) rename {dal => client}/__test__/srv/main.go (100%) rename {dal => client}/index.ts (100%) rename dal/node.napi.ts => client/libdal.ts (97%) rename {dal => client}/readme.md (100%) delete mode 100644 dal/bun.cffi.ts diff --git a/dal/Binding.ts b/client/Binding.ts similarity index 100% rename from dal/Binding.ts rename to client/Binding.ts diff --git a/dal/Builder.ts b/client/Builder.ts similarity index 99% rename from dal/Builder.ts rename to client/Builder.ts index 20c4631..2cdcff5 100644 --- a/dal/Builder.ts +++ b/client/Builder.ts @@ -187,7 +187,7 @@ export default class Builder any> { } async Query>(): Promise { const rows = this.Rows(); - const result = []; + const result: T[] = []; for await (const row of rows) { result.push(row); } diff --git a/dal/Bunding.ts b/client/Bunding.ts similarity index 86% rename from dal/Bunding.ts rename to client/Bunding.ts index a6d8596..cff2cfa 100644 --- a/dal/Bunding.ts +++ b/client/Bunding.ts @@ -2,9 +2,10 @@ * This file is responsible for binding the C library to the Bun runtime. */ import { dlopen, FFIType, suffix, ptr, toBuffer } from "bun:ffi"; +import { join } from "path"; -const libname = `../clib/clib.${suffix}`; -const libpath = libname; +const libname = `clib.${suffix}`; +const libpath = join("clib", libname); const { symbols: { InitSQLite, CreateRowIterator, NextRow, GetLen, Free, Cleanup }, @@ -46,8 +47,8 @@ function rowIterator(buf: Buffer) { if (pointer === null) { return null; } - const buf = toBuffer(pointer, 0, GetLen(iter)); - //Free(pointer) //should be resolved by GC; + const buf = Buffer.from(toBuffer(pointer, 0, GetLen(iter))); + Free(pointer); return buf; }; diff --git a/client/Library.ts b/client/Library.ts new file mode 100644 index 0000000..55a2242 --- /dev/null +++ b/client/Library.ts @@ -0,0 +1,21 @@ +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); + +type RowIterator = { + next: () => Buffer; + cleanup: () => void; +}; +type SQLite = { + initSQLite: (pragmas: Buffer) => void; + rowIterator: (input: Buffer) => RowIterator; +}; + +let Library: SQLite; + +if (process.isBun) { + Library = require("./Bunding") as SQLite; +} else { + Library = require("./Binding") as SQLite; +} + +export default Library; diff --git a/dal/Protocol.ts b/client/Protocol.ts similarity index 100% rename from dal/Protocol.ts rename to client/Protocol.ts diff --git a/client/__test__/bench.bun.ts b/client/__test__/bench.bun.ts new file mode 100644 index 0000000..c973b76 --- /dev/null +++ b/client/__test__/bench.bun.ts @@ -0,0 +1,90 @@ +import fs from "fs"; +import dal from "../Bunding"; + +const Mb = (num) => Math.round(num / 1024 / 1024); + +class Stats { + avg_rss; + avg_heapTotal; + avg_heapUsed; + avg_external; + calls; + constructor() { + this.calls = 0; + this.avg_rss = 0; + this.avg_heapTotal = 0; + this.avg_heapUsed = 0; + this.avg_external = 0; + } + add(mem) { + this.calls++; + this.avg_rss += mem.rss; + this.avg_heapTotal += mem.heapTotal; + this.avg_heapUsed += mem.heapUsed; + this.avg_external += mem.external; + } + avg() { + const n = this.calls; + this.avg_rss /= n; + this.avg_heapTotal /= n; + this.avg_heapUsed /= n; + this.avg_external /= n; + } + print() { + console.log(` +AVERAGE: +rss: ${Mb(this.avg_rss)} Mb +external: ${Mb(this.avg_external)} Mb +buffers: ${Mb(this.avg_heapUsed)} Mb +total: ${Mb(this.avg_heapTotal)} Mb`); + } +} + +const stats = new Stats(); +let prevMem = process.memoryUsage(); +stats.add(prevMem); + +function MEM(when = "") { + const mem = process.memoryUsage(); + stats.add(mem); + console.log(` +${when} +rss: ${Mb(mem.rss)} Mb [delta> ${mem.rss - prevMem.rss}] +external: ${Mb(mem.external)} Mb [delta> ${mem.external - prevMem.external}] +buffers: ${Mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}] +total: ${Mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`); +} + +console.time("Time to end"); +MEM("START"); + +const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack"); + +const Iterator = dal.rowIterator(buf); +MEM("AFTER INIT"); + +let dataTransferedBytes = 0; +for (let i = 0; i < 100000000; i++) { + const b = Iterator.next()!; + if (b.length === 0) { + break; + } + dataTransferedBytes += b.length; + if (i % 1000000 === 0) { + MEM(`ITERATION ${i}`); + } +} + +MEM("AFTER ITERATION"); + +Iterator.cleanup(); +MEM("AFTER CLEANUP"); + +console.log("\nData transfered: ", Mb(dataTransferedBytes), "Mb"); +console.timeEnd("Time to end"); + +setTimeout(() => { + MEM("AFTER SOME TIME"); + stats.avg(); + stats.print(); +}, 30000); diff --git a/dal/__test__/bench.node.cjs b/client/__test__/bench.node.cjs similarity index 96% rename from dal/__test__/bench.node.cjs rename to client/__test__/bench.node.cjs index 5815283..fe20f6c 100644 --- a/dal/__test__/bench.node.cjs +++ b/client/__test__/bench.node.cjs @@ -55,7 +55,7 @@ MEM("START"); const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack"); -const Iterator = dal.RowIterator(buf); +const Iterator = dal.rowIterator(buf); MEM("AFTER INIT"); let dataTransferedBytes = 0; @@ -72,7 +72,7 @@ for (let i = 0; i < 100000000; i++) { MEM("AFTER ITERATION"); -Iterator.free(); +Iterator.cleanup(); MEM("AFTER CLEANUP"); console.log("\nData transfered: ", Mb(dataTransferedBytes), "Mb"); diff --git a/dal/__test__/builder.test.ts b/client/__test__/builder.test.ts similarity index 100% rename from dal/__test__/builder.test.ts rename to client/__test__/builder.test.ts diff --git a/dal/__test__/srv/go.mod b/client/__test__/srv/go.mod similarity index 100% rename from dal/__test__/srv/go.mod rename to client/__test__/srv/go.mod diff --git a/dal/__test__/srv/go.sum b/client/__test__/srv/go.sum similarity index 100% rename from dal/__test__/srv/go.sum rename to client/__test__/srv/go.sum diff --git a/dal/__test__/srv/main.go b/client/__test__/srv/main.go similarity index 100% rename from dal/__test__/srv/main.go rename to client/__test__/srv/main.go diff --git a/dal/index.ts b/client/index.ts similarity index 100% rename from dal/index.ts rename to client/index.ts diff --git a/dal/node.napi.ts b/client/libdal.ts similarity index 97% rename from dal/node.napi.ts rename to client/libdal.ts index dd6eea5..9ad51d7 100644 --- a/dal/node.napi.ts +++ b/client/libdal.ts @@ -1,5 +1,5 @@ import Builder from "./Builder"; -import Binding from "./Binding"; +import Binding from "./Library"; import { encodeRequest, decodeRows, decodeResponse } from "./Protocol"; import type { ExecResult } from "./Protocol"; diff --git a/dal/readme.md b/client/readme.md similarity index 100% rename from dal/readme.md rename to client/readme.md diff --git a/dal/bun.cffi.ts b/dal/bun.cffi.ts deleted file mode 100644 index 904b582..0000000 --- a/dal/bun.cffi.ts +++ /dev/null @@ -1,65 +0,0 @@ -import Builder from "./Builder"; -import Bunding from "./Bunding"; -import { encodeRequest, decodeRows, decodeResponse } from "./Protocol"; -import type { ExecResult } from "./Protocol"; - -type Options = { - database: string; -}; - -/** - * Allows to use SQLite databases in BunJS - */ -export default class CBuilder< - I extends abstract new (...args: any) => any, -> extends Builder { - constructor(opts: Options) { - super({ database: opts.database, url: "" }); - } - /** - * TODO: handle responses - */ - async *Rows>(): AsyncGenerator { - this.formatRequest(); - const req = Buffer.from(encodeRequest(this.request)); - const iter = Bunding.rowIterator(req); - for (;;) { - const response = iter.next(); - if (response === null) { - iter.cleanup(); - break; - } - const rows = decodeRows(response); - if (rows.length === 0) { - iter.cleanup(); - break; - } - for (const row of rows) { - if (this.headerRow === null) { - this.headerRow = row.r; - continue; - } - yield this.formatRow(row.r); - } - } - } - async Query>(): Promise { - const rows = this.Rows(); - const result: T[] = []; - for await (const row of rows) { - result.push(row); - } - return result; - } - async Exec(): Promise { - this.formatRequest(); - const req = Buffer.from(encodeRequest(this.request)); - const iter = Bunding.rowIterator(req); - const response = iter.next(); - if (response === null) { - iter.cleanup(); - throw new Error("No response"); - } - return decodeResponse(response); - } -} diff --git a/package.json b/package.json index bd0ba0b..8b03628 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,10 @@ }, "scripts": { "test:client": "bun test:*", - "test:dal": "bun test dal/__test__", - "test:serve": "cd dal/__test__/srv && go run main.go", - "bench:node": "node ./dal/__test__/bench.node.cjs", + "test:dal": "bun test client/__test__", + "test:serve": "cd client/__test__/srv && go run main.go", + "bench:node": "node ./client/__test__/bench.node.cjs", + "bench:bun": "bun ./client/__test__/bench.bun.ts", "fmt": "prettier --write .", "build": "tsc", "prepublish": "tsc", diff --git a/pkg/facade/SQLIteShared.go b/pkg/facade/SQLIteShared.go index 0fb96b8..e1c6c11 100644 --- a/pkg/facade/SQLIteShared.go +++ b/pkg/facade/SQLIteShared.go @@ -99,79 +99,3 @@ func (r *RowsIter) Next() []byte { cols, _ := proto.MarshalRow(data) return cols } - -func HandleQuery(input *[]byte, output *[]byte) int { - InitSQLite([]string{}) - req := proto.Request{} - _, err := req.UnmarshalMsg(*input) - if err != nil { - res := proto.Response{ - Id: 0, - RowsAffected: -1, - LastInsertId: -1, - Msg: "failed to unmarshal request", - } - *output, _ = res.MarshalMsg(nil) - return 0 - } - query, err := req.Parse(adapter.GetDialect(db.Type)) - if err != nil { - res := proto.Response{ - Id: 0, - RowsAffected: -1, - LastInsertId: -1, - Msg: err.Error(), - } - *output, _ = res.MarshalMsg(nil) - return 0 - } - if query.Exec { - result, err := db.Exec(query) - if err != nil { - res := proto.Response{ - Id: 0, - RowsAffected: -1, - LastInsertId: -1, - Msg: err.Error(), - } - *output, _ = res.MarshalMsg(nil) - return 0 - } - ra, _ := result.RowsAffected() - la, _ := result.LastInsertId() - res := proto.Response{ - Id: 0, - RowsAffected: ra, - LastInsertId: la, - } - *output, _ = res.MarshalMsg(nil) - return 0 - } - rows, err := db.Query(query) - if err != nil { - res := proto.Response{ - Id: 0, - RowsAffected: -1, - LastInsertId: -1, - Msg: err.Error(), - } - *output, _ = res.MarshalMsg(nil) - return 0 - } - defer rows.Close() - columns, _ := rows.Columns() - types, _ := rows.ColumnTypes() - cols, _ := proto.MarshalRow(columns) - *output = append(*output, cols...) - for rows.Next() { - data := make([]interface{}, len(columns)) - for i := range data { - typ := reflect.New(types[i].ScanType()).Interface() - data[i] = &typ - } - rows.Scan(data...) - cols, _ := proto.MarshalRow(data) - *output = append(*output, cols...) - } - return 0 -} diff --git a/tsconfig.json b/tsconfig.json index 39a7800..799326b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,5 @@ { - "files": ["dal/index.ts", "dal/Binding.ts", "dal/node.napi.ts"], + "files": ["client/index.ts", "client/Binding.ts", "client/libdal.ts"], "compilerOptions": { "target": "ESNext", "module": "ESNext",