From bfcb0fc9939a35406d365272d91f44fab4219c32 Mon Sep 17 00:00:00 2001 From: Anton Nesterov Date: Wed, 21 Aug 2024 03:28:40 +0200 Subject: [PATCH] [feat] builder for in-process sqlite --- README.md | 1 - dal/Binding.ts | 7 +++++ dal/Builder.ts | 25 +++++++++++------- dal/CBuilder.ts | 51 ++++++++++++++++++++++++++++++++++++ dal/Protocol.ts | 31 +++++++++++++--------- dal/SQLite.ts | 3 --- dal/__test__/builder.test.ts | 5 +--- dal/__test__/sqlite.node.cjs | 8 +++--- dal/index.ts | 1 + doc/dal-internals.md | 13 ++++----- package.json | 4 +-- tsconfig.json | 7 ++--- 12 files changed, 108 insertions(+), 48 deletions(-) create mode 100644 dal/Binding.ts create mode 100644 dal/CBuilder.ts delete mode 100644 dal/SQLite.ts diff --git a/README.md b/README.md index b28a729..7961417 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ Data Accees Layer for SQL databases written in Go. - ## NodeJs Client Mongodb inspired query interface: diff --git a/dal/Binding.ts b/dal/Binding.ts new file mode 100644 index 0000000..0ad407e --- /dev/null +++ b/dal/Binding.ts @@ -0,0 +1,7 @@ +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); +type SQLite = { + InitSQLite: (pragmas: Buffer) => void; + Handle: (input: Buffer) => Buffer; +}; +export default require("../build/Release/dal.node") as SQLite; diff --git a/dal/Builder.ts b/dal/Builder.ts index 3c15786..5353db4 100644 --- a/dal/Builder.ts +++ b/dal/Builder.ts @@ -1,5 +1,10 @@ import type { Request, ExecResult } from "./Protocol"; -import { METHODS, encodeRequest, decodeResponse, decodeRowsIterator } from "./Protocol"; +import { + METHODS, + encodeRequest, + decodeResponse, + decodeRowsIterator, +} from "./Protocol"; type Primitive = string | number | boolean | null; @@ -40,12 +45,12 @@ type Options = { }; export default class Builder any> { - private request: Request; - private url: string; - private dtoTemplate: new (...args: any) => any = Object; - private methodCalls: Map = new Map(); // one call per method - private headerRow: unknown[] | null = null; - private httpHeaders: Record = {}; + protected request: Request; + protected url: string; + protected dtoTemplate: new (...args: any) => any = Object; + protected methodCalls: Map = new Map(); // one call per method + protected headerRow: unknown[] | null = null; + protected httpHeaders: Record = {}; constructor(opts: Options) { this.request = { id: 0, @@ -57,7 +62,7 @@ export default class Builder any> { this.httpHeaders = opts.headers; } } - private formatRequest(): void { + protected formatRequest(): void { this.request.commands = []; METHODS.forEach((method) => { const args = this.methodCalls.get(method); @@ -67,7 +72,7 @@ export default class Builder any> { this.request.commands.push({ method, args }); }); } - private formatRow(data: unknown[]) { + protected formatRow(data: unknown[]) { if (!this.dtoTemplate || this.dtoTemplate === Object) { return data; } @@ -81,7 +86,7 @@ export default class Builder any> { return instance; } Raw(sql: string, ...values: unknown[]): Builder { - this.methodCalls.set("Raw", [{s: sql, v: values}]); + this.methodCalls.set("Raw", [{ s: sql, v: values }]); return this; } In(table: string): Builder { diff --git a/dal/CBuilder.ts b/dal/CBuilder.ts new file mode 100644 index 0000000..71f1e3f --- /dev/null +++ b/dal/CBuilder.ts @@ -0,0 +1,51 @@ +import Builder from "./Builder"; +import Binding from "./Binding"; +import { encodeRequest, decodeRows, decodeResponse } from "./Protocol"; +import type { ExecResult } from "./Protocol"; + +type Options = { + database: string; +}; + +/** + * Allows to use SQLite databases in a NodeJS process. + * It is less memory-efficient than a seaparate server, and uses absolute path for database name. + */ +export default class CBuilder< + I extends abstract new (...args: any) => any, +> extends Builder { + constructor(opts: Options) { + super({ database: opts.database, url: "" }); + } + /** + * Not really an iterator, since addonn allocates memory for all rows + * but returns an iterator + */ + async *Rows>(): AsyncGenerator { + this.formatRequest(); + const req = Buffer.from(encodeRequest(this.request)); + const response = Binding.Handle(req); + const rows = decodeRows(response); + for (const row of rows) { + if (this.headerRow === null) { + this.headerRow = row.r; + continue; + } + yield this.formatRow(row.r); + } + } + async Query>(): Promise { + const rows = this.Rows(); + const result: T[] = []; + for await (const row of rows) { + result.push(row); + } + return result; + } + async Exec(): Promise { + this.formatRequest(); + const req = Buffer.from(encodeRequest(this.request)); + const response = Binding.Handle(req); + return decodeResponse(response); + } +} diff --git a/dal/Protocol.ts b/dal/Protocol.ts index 098beb9..9d0789a 100644 --- a/dal/Protocol.ts +++ b/dal/Protocol.ts @@ -12,33 +12,38 @@ export interface Request { } export interface ExecResult { - Id: number; - RowsAffected: number; - LastInsertId: number; - Msg?: string; + Id: number; + RowsAffected: number; + LastInsertId: number; + Msg?: string; } -interface Row { +export interface Row { r: unknown[]; } export const METHODS = "Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split( "|", -); + ); export function encodeRequest(request: Request): Uint8Array { return encode(request); } export function decodeResponse(input: Uint8Array): ExecResult { - const res = decode(input) as {i: number; ra: number; li: number, m?: string}; - return { - Id: res.i, - RowsAffected: res.ra, - LastInsertId: res.li, - Msg: res.m, - }; + const res = decode(input) as { + i: number; + ra: number; + li: number; + m?: string; + }; + return { + Id: res.i, + RowsAffected: res.ra, + LastInsertId: res.li, + Msg: res.m, + }; } const ROW_TAG = [0x81, 0xa1, 0x72]; diff --git a/dal/SQLite.ts b/dal/SQLite.ts deleted file mode 100644 index 29a42a1..0000000 --- a/dal/SQLite.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createRequire } from 'node:module'; -const require = createRequire(import.meta.url); -export default require('../build/Release/dal.node'); \ No newline at end of file diff --git a/dal/__test__/builder.test.ts b/dal/__test__/builder.test.ts index ee84c22..5132c65 100644 --- a/dal/__test__/builder.test.ts +++ b/dal/__test__/builder.test.ts @@ -62,10 +62,7 @@ test("Query format", async () => { test("Query raw", async () => { const dal = new DAL(options); - const rows = await dal - .Raw("SELECT * FROM test WHERE id = 1") - .As(DTO) - .Query(); + const rows = await dal.Raw("SELECT * FROM test WHERE id = 1").As(DTO).Query(); for (const row of rows) { expect(row.id).toBeDefined(); expect(row.age).toBeUndefined(); diff --git a/dal/__test__/sqlite.node.cjs b/dal/__test__/sqlite.node.cjs index eb31662..bd84643 100644 --- a/dal/__test__/sqlite.node.cjs +++ b/dal/__test__/sqlite.node.cjs @@ -1,6 +1,6 @@ -const fs = require('fs'); -const dal = require('../../build/Release/dal.node'); +const fs = require("fs"); +const dal = require("../../build/Release/dal.node"); dal.InitSQLite(Buffer.from([])); -const buf = fs.readFileSync('./pkg/__test__/proto_test.msgpack'); +const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack"); data = dal.Handle(buf); -console.log(data); \ No newline at end of file +console.log(data); diff --git a/dal/index.ts b/dal/index.ts index 5fcba6a..4937830 100644 --- a/dal/index.ts +++ b/dal/index.ts @@ -1 +1,2 @@ export { default as DAL } from "./Builder"; +export { default as DALite } from "./CBuilder"; diff --git a/doc/dal-internals.md b/doc/dal-internals.md index b21457e..cd8eefc 100644 --- a/doc/dal-internals.md +++ b/doc/dal-internals.md @@ -1,7 +1,7 @@ # DAL Internal Architecture - The Client is written in TypeScript. -- The DAL server written in Golang. +- The DAL server written in Golang. ## Components @@ -30,10 +30,10 @@ Client consists of a query builder and protocol decoder/encoder. ## Protocol - Protocol utilizes messagepack for encoding and decoding the messages. There following types of encoded data: + - Row stream - Query (request) - Response (exec result) @@ -54,7 +54,7 @@ Locations: ### Row Stream - The server sends streaming (chunked) data to the client, every chunk is a row. -- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}` +- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}` - The first row is the header row, which contains the column names. Parsing the row stream (pseudo code): @@ -76,6 +76,7 @@ output << header + buffer ``` MessagePack schema for the row stream: + ```go type Row struct { Data []interface{} `msg:"r"` @@ -91,8 +92,6 @@ type Row struct { - Db: string (required, database name) - Commands: []BuilderMethod (required, list of Builder arguments) - - ```go type BuilderMethod struct { Method string `msg:"method"` @@ -107,6 +106,7 @@ type Request struct { ``` ### Response + The response is inteneded for operation results that don't return rows. ```go @@ -140,6 +140,7 @@ Locations: ``` ### Builder Methods + Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing [TS Docs]() [Golang Docs]() @@ -157,4 +158,4 @@ Locations: |- Adapter |... |... -``` \ No newline at end of file +``` diff --git a/package.json b/package.json index be395d9..3f5696f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@nesterow/dal", - "version": "0.0.1", - "repository":"https://github.com/nesterow/dal.git", + "version": "0.0.2", + "repository": "https://github.com/nesterow/dal.git", "publishConfig": { "registry": "https://npm.pkg.github.com" }, diff --git a/tsconfig.json b/tsconfig.json index 3fa80ee..0a9d61b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,5 @@ { - "files": [ - "dal/index.ts", - "dal/SQLite.ts", - ], + "files": ["dal/index.ts", "dal/Binding.ts"], "compilerOptions": { "target": "ESNext", "module": "ESNext", @@ -23,6 +20,6 @@ "noUnusedLocals": false, "noUnusedParameters": false, "noPropertyAccessFromIndexSignature": false, - "outDir": "dist", + "outDir": "dist" } }