[feat] builder for in-process sqlite

README.md

@@ -2,7 +2,6 @@
 
 Data Accees Layer for SQL databases written in Go.
 
-
 ## NodeJs Client
 
 Mongodb inspired query interface:

dal/Binding.ts

@@ -0,0 +1,7 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+type SQLite = {
+  InitSQLite: (pragmas: Buffer) => void;
+  Handle: (input: Buffer) => Buffer;
+};
+export default require("../build/Release/dal.node") as SQLite;

dal/Builder.ts

@@ -1,5 +1,10 @@
 import type { Request, ExecResult } from "./Protocol";
-import { METHODS, encodeRequest, decodeResponse, decodeRowsIterator } from "./Protocol";
+import {
+  METHODS,
+  encodeRequest,
+  decodeResponse,
+  decodeRowsIterator,
+} from "./Protocol";
 
 type Primitive = string | number | boolean | null;
 
@@ -40,12 +45,12 @@ type Options = {
 };
 
 export default class Builder<I extends abstract new (...args: any) => any> {
-  private request: Request;
+  protected request: Request;
-  private url: string;
+  protected url: string;
-  private dtoTemplate: new (...args: any) => any = Object;
+  protected dtoTemplate: new (...args: any) => any = Object;
-  private methodCalls: Map<string, unknown[]> = new Map(); // one call per method
+  protected methodCalls: Map<string, unknown[]> = new Map(); // one call per method
-  private headerRow: unknown[] | null = null;
+  protected headerRow: unknown[] | null = null;
-  private httpHeaders: Record<string, string> = {};
+  protected httpHeaders: Record<string, string> = {};
   constructor(opts: Options) {
     this.request = {
       id: 0,
@@ -57,7 +62,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
       this.httpHeaders = opts.headers;
     }
   }
-  private formatRequest(): void {
+  protected formatRequest(): void {
     this.request.commands = [];
     METHODS.forEach((method) => {
       const args = this.methodCalls.get(method);
@@ -67,7 +72,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
       this.request.commands.push({ method, args });
     });
   }
-  private formatRow(data: unknown[]) {
+  protected formatRow(data: unknown[]) {
     if (!this.dtoTemplate || this.dtoTemplate === Object) {
       return data;
     }
@@ -81,7 +86,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
     return instance;
   }
   Raw(sql: string, ...values: unknown[]): Builder<I> {
-    this.methodCalls.set("Raw", [{s: sql, v: values}]);
+    this.methodCalls.set("Raw", [{ s: sql, v: values }]);
     return this;
   }
   In(table: string): Builder<I> {

dal/CBuilder.ts

@@ -0,0 +1,51 @@
+import Builder from "./Builder";
+import Binding from "./Binding";
+import { encodeRequest, decodeRows, decodeResponse } from "./Protocol";
+import type { ExecResult } from "./Protocol";
+
+type Options = {
+  database: string;
+};
+
+/**
+ * Allows to use SQLite databases in a NodeJS process.
+ * It is less memory-efficient than a seaparate server, and uses absolute path for database name.
+ */
+export default class CBuilder<
+  I extends abstract new (...args: any) => any,
+> extends Builder<I> {
+  constructor(opts: Options) {
+    super({ database: opts.database, url: "" });
+  }
+  /**
+   * Not really an iterator, since addonn allocates memory for all rows
+   * but returns an iterator
+   */
+  async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
+    this.formatRequest();
+    const req = Buffer.from(encodeRequest(this.request));
+    const response = Binding.Handle(req);
+    const rows = decodeRows(response);
+    for (const row of rows) {
+      if (this.headerRow === null) {
+        this.headerRow = row.r;
+        continue;
+      }
+      yield this.formatRow(row.r);
+    }
+  }
+  async Query<T = InstanceType<I>>(): Promise<T[]> {
+    const rows = this.Rows();
+    const result: T[] = [];
+    for await (const row of rows) {
+      result.push(row);
+    }
+    return result;
+  }
+  async Exec(): Promise<ExecResult> {
+    this.formatRequest();
+    const req = Buffer.from(encodeRequest(this.request));
+    const response = Binding.Handle(req);
+    return decodeResponse(response);
+  }
+}

dal/Protocol.ts

@@ -12,33 +12,38 @@ export interface Request {
 }
 
 export interface ExecResult {
-    Id: number;
+  Id: number;
-    RowsAffected: number;
+  RowsAffected: number;
-    LastInsertId: number;
+  LastInsertId: number;
-    Msg?: string;
+  Msg?: string;
 }
 
-interface Row {
+export interface Row {
   r: unknown[];
 }
 
 export const METHODS =
   "Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split(
     "|",
-);
+  );
 
 export function encodeRequest(request: Request): Uint8Array {
   return encode(request);
 }
 
 export function decodeResponse(input: Uint8Array): ExecResult {
-  const res = decode(input) as {i: number; ra: number; li: number, m?: string};
+  const res = decode(input) as {
-    return {
+    i: number;
-        Id: res.i,
+    ra: number;
-        RowsAffected: res.ra,
+    li: number;
-        LastInsertId: res.li,
+    m?: string;
-        Msg: res.m,
+  };
-    };
+  return {
+    Id: res.i,
+    RowsAffected: res.ra,
+    LastInsertId: res.li,
+    Msg: res.m,
+  };
 }
 
 const ROW_TAG = [0x81, 0xa1, 0x72];

dal/SQLite.ts

@@ -1,3 +0,0 @@
-import { createRequire } from 'node:module';
-const require = createRequire(import.meta.url);
-export default require('../build/Release/dal.node');

dal/__test__/builder.test.ts

@@ -62,10 +62,7 @@ test("Query format", async () => {
 
 test("Query raw", async () => {
   const dal = new DAL(options);
-  const rows = await dal
+  const rows = await dal.Raw("SELECT * FROM test WHERE id = 1").As(DTO).Query();
-    .Raw("SELECT * FROM test WHERE id = 1")
-    .As(DTO)
-    .Query();
   for (const row of rows) {
     expect(row.id).toBeDefined();
     expect(row.age).toBeUndefined();

dal/__test__/sqlite.node.cjs

@@ -1,6 +1,6 @@
-const fs = require('fs');
+const fs = require("fs");
-const dal = require('../../build/Release/dal.node');
+const dal = require("../../build/Release/dal.node");
 dal.InitSQLite(Buffer.from([]));
-const buf = fs.readFileSync('./pkg/__test__/proto_test.msgpack');
+const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
 data = dal.Handle(buf);
-console.log(data);
+console.log(data);

dal/index.ts

@@ -1 +1,2 @@
 export { default as DAL } from "./Builder";
+export { default as DALite } from "./CBuilder";

doc/dal-internals.md

@@ -1,7 +1,7 @@
 # DAL Internal Architecture
 
 - The Client is written in TypeScript.
-- The DAL server written in Golang. 
+- The DAL server written in Golang.
 
 ## Components
 
@@ -30,10 +30,10 @@ Client consists of a query builder and protocol decoder/encoder.
 
 ## Protocol
 
-
 Protocol utilizes messagepack for encoding and decoding the messages.
 
 There following types of encoded data:
+
 - Row stream
 - Query (request)
 - Response (exec result)
@@ -54,7 +54,7 @@ Locations:
 ### Row Stream
 
 - The server sends streaming (chunked) data to the client, every chunk is a row.
-- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}` 
+- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}`
 - The first row is the header row, which contains the column names.
 
 Parsing the row stream (pseudo code):
@@ -76,6 +76,7 @@ output << header + buffer
 ```
 
 MessagePack schema for the row stream:
+
 ```go
 type Row struct {
 	Data []interface{} `msg:"r"`
@@ -91,8 +92,6 @@ type Row struct {
   - Db: string (required, database name)
   - Commands: []BuilderMethod (required, list of Builder arguments)
 
-
-
 ```go
 type BuilderMethod struct {
 	Method string        `msg:"method"`
@@ -107,6 +106,7 @@ type Request struct {
 ```
 
 ### Response
+
 The response is inteneded for operation results that don't return rows.
 
 ```go
@@ -140,6 +140,7 @@ Locations:
 ```
 
 ### Builder Methods
+
 Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing
 [TS Docs]()
 [Golang Docs]()
@@ -157,4 +158,4 @@ Locations:
     |- Adapter
         |...
 |...
-```
+```

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@nesterow/dal",
-  "version": "0.0.1",
+  "version": "0.0.2",
-  "repository":"https://github.com/nesterow/dal.git",
+  "repository": "https://github.com/nesterow/dal.git",
   "publishConfig": {
     "registry": "https://npm.pkg.github.com"
   },

tsconfig.json

@@ -1,8 +1,5 @@
 {
-  "files": [
+  "files": ["dal/index.ts", "dal/Binding.ts"],
-    "dal/index.ts",
-    "dal/SQLite.ts",
-  ],
   "compilerOptions": {
     "target": "ESNext",
     "module": "ESNext",
@@ -23,6 +20,6 @@
     "noUnusedLocals": false,
     "noUnusedParameters": false,
     "noPropertyAccessFromIndexSignature": false,
-    "outDir": "dist",
+    "outDir": "dist"
   }
 }