[feat] builder for in-process sqlite

This commit is contained in:
Anton Nesterov 2024-08-21 03:28:40 +02:00
parent 1aa3ad7fca
commit bfcb0fc993
No known key found for this signature in database
GPG key ID: 59121E8AE2851FB5
12 changed files with 108 additions and 48 deletions

View file

@ -2,7 +2,6 @@
Data Accees Layer for SQL databases written in Go. Data Accees Layer for SQL databases written in Go.
## NodeJs Client ## NodeJs Client
Mongodb inspired query interface: Mongodb inspired query interface:

7
dal/Binding.ts Normal file
View file

@ -0,0 +1,7 @@
import { createRequire } from "node:module";
const require = createRequire(import.meta.url);
type SQLite = {
InitSQLite: (pragmas: Buffer) => void;
Handle: (input: Buffer) => Buffer;
};
export default require("../build/Release/dal.node") as SQLite;

View file

@ -1,5 +1,10 @@
import type { Request, ExecResult } from "./Protocol"; import type { Request, ExecResult } from "./Protocol";
import { METHODS, encodeRequest, decodeResponse, decodeRowsIterator } from "./Protocol"; import {
METHODS,
encodeRequest,
decodeResponse,
decodeRowsIterator,
} from "./Protocol";
type Primitive = string | number | boolean | null; type Primitive = string | number | boolean | null;
@ -40,12 +45,12 @@ type Options = {
}; };
export default class Builder<I extends abstract new (...args: any) => any> { export default class Builder<I extends abstract new (...args: any) => any> {
private request: Request; protected request: Request;
private url: string; protected url: string;
private dtoTemplate: new (...args: any) => any = Object; protected dtoTemplate: new (...args: any) => any = Object;
private methodCalls: Map<string, unknown[]> = new Map(); // one call per method protected methodCalls: Map<string, unknown[]> = new Map(); // one call per method
private headerRow: unknown[] | null = null; protected headerRow: unknown[] | null = null;
private httpHeaders: Record<string, string> = {}; protected httpHeaders: Record<string, string> = {};
constructor(opts: Options) { constructor(opts: Options) {
this.request = { this.request = {
id: 0, id: 0,
@ -57,7 +62,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
this.httpHeaders = opts.headers; this.httpHeaders = opts.headers;
} }
} }
private formatRequest(): void { protected formatRequest(): void {
this.request.commands = []; this.request.commands = [];
METHODS.forEach((method) => { METHODS.forEach((method) => {
const args = this.methodCalls.get(method); const args = this.methodCalls.get(method);
@ -67,7 +72,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
this.request.commands.push({ method, args }); this.request.commands.push({ method, args });
}); });
} }
private formatRow(data: unknown[]) { protected formatRow(data: unknown[]) {
if (!this.dtoTemplate || this.dtoTemplate === Object) { if (!this.dtoTemplate || this.dtoTemplate === Object) {
return data; return data;
} }
@ -81,7 +86,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
return instance; return instance;
} }
Raw(sql: string, ...values: unknown[]): Builder<I> { Raw(sql: string, ...values: unknown[]): Builder<I> {
this.methodCalls.set("Raw", [{s: sql, v: values}]); this.methodCalls.set("Raw", [{ s: sql, v: values }]);
return this; return this;
} }
In(table: string): Builder<I> { In(table: string): Builder<I> {

51
dal/CBuilder.ts Normal file
View file

@ -0,0 +1,51 @@
import Builder from "./Builder";
import Binding from "./Binding";
import { encodeRequest, decodeRows, decodeResponse } from "./Protocol";
import type { ExecResult } from "./Protocol";
type Options = {
database: string;
};
/**
* Allows to use SQLite databases in a NodeJS process.
* It is less memory-efficient than a seaparate server, and uses absolute path for database name.
*/
export default class CBuilder<
I extends abstract new (...args: any) => any,
> extends Builder<I> {
constructor(opts: Options) {
super({ database: opts.database, url: "" });
}
/**
* Not really an iterator, since addonn allocates memory for all rows
* but returns an iterator
*/
async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
this.formatRequest();
const req = Buffer.from(encodeRequest(this.request));
const response = Binding.Handle(req);
const rows = decodeRows(response);
for (const row of rows) {
if (this.headerRow === null) {
this.headerRow = row.r;
continue;
}
yield this.formatRow(row.r);
}
}
async Query<T = InstanceType<I>>(): Promise<T[]> {
const rows = this.Rows();
const result: T[] = [];
for await (const row of rows) {
result.push(row);
}
return result;
}
async Exec(): Promise<ExecResult> {
this.formatRequest();
const req = Buffer.from(encodeRequest(this.request));
const response = Binding.Handle(req);
return decodeResponse(response);
}
}

View file

@ -18,21 +18,26 @@ export interface ExecResult {
Msg?: string; Msg?: string;
} }
interface Row { export interface Row {
r: unknown[]; r: unknown[];
} }
export const METHODS = export const METHODS =
"Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split( "Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split(
"|", "|",
); );
export function encodeRequest(request: Request): Uint8Array { export function encodeRequest(request: Request): Uint8Array {
return encode(request); return encode(request);
} }
export function decodeResponse(input: Uint8Array): ExecResult { export function decodeResponse(input: Uint8Array): ExecResult {
const res = decode(input) as {i: number; ra: number; li: number, m?: string}; const res = decode(input) as {
i: number;
ra: number;
li: number;
m?: string;
};
return { return {
Id: res.i, Id: res.i,
RowsAffected: res.ra, RowsAffected: res.ra,

View file

@ -1,3 +0,0 @@
import { createRequire } from 'node:module';
const require = createRequire(import.meta.url);
export default require('../build/Release/dal.node');

View file

@ -62,10 +62,7 @@ test("Query format", async () => {
test("Query raw", async () => { test("Query raw", async () => {
const dal = new DAL(options); const dal = new DAL(options);
const rows = await dal const rows = await dal.Raw("SELECT * FROM test WHERE id = 1").As(DTO).Query();
.Raw("SELECT * FROM test WHERE id = 1")
.As(DTO)
.Query();
for (const row of rows) { for (const row of rows) {
expect(row.id).toBeDefined(); expect(row.id).toBeDefined();
expect(row.age).toBeUndefined(); expect(row.age).toBeUndefined();

View file

@ -1,6 +1,6 @@
const fs = require('fs'); const fs = require("fs");
const dal = require('../../build/Release/dal.node'); const dal = require("../../build/Release/dal.node");
dal.InitSQLite(Buffer.from([])); dal.InitSQLite(Buffer.from([]));
const buf = fs.readFileSync('./pkg/__test__/proto_test.msgpack'); const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
data = dal.Handle(buf); data = dal.Handle(buf);
console.log(data); console.log(data);

View file

@ -1 +1,2 @@
export { default as DAL } from "./Builder"; export { default as DAL } from "./Builder";
export { default as DALite } from "./CBuilder";

View file

@ -30,10 +30,10 @@ Client consists of a query builder and protocol decoder/encoder.
## Protocol ## Protocol
Protocol utilizes messagepack for encoding and decoding the messages. Protocol utilizes messagepack for encoding and decoding the messages.
There following types of encoded data: There following types of encoded data:
- Row stream - Row stream
- Query (request) - Query (request)
- Response (exec result) - Response (exec result)
@ -76,6 +76,7 @@ output << header + buffer
``` ```
MessagePack schema for the row stream: MessagePack schema for the row stream:
```go ```go
type Row struct { type Row struct {
Data []interface{} `msg:"r"` Data []interface{} `msg:"r"`
@ -91,8 +92,6 @@ type Row struct {
- Db: string (required, database name) - Db: string (required, database name)
- Commands: []BuilderMethod (required, list of Builder arguments) - Commands: []BuilderMethod (required, list of Builder arguments)
```go ```go
type BuilderMethod struct { type BuilderMethod struct {
Method string `msg:"method"` Method string `msg:"method"`
@ -107,6 +106,7 @@ type Request struct {
``` ```
### Response ### Response
The response is inteneded for operation results that don't return rows. The response is inteneded for operation results that don't return rows.
```go ```go
@ -140,6 +140,7 @@ Locations:
``` ```
### Builder Methods ### Builder Methods
Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing Raw|In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing
[TS Docs]() [TS Docs]()
[Golang Docs]() [Golang Docs]()

View file

@ -1,7 +1,7 @@
{ {
"name": "@nesterow/dal", "name": "@nesterow/dal",
"version": "0.0.1", "version": "0.0.2",
"repository":"https://github.com/nesterow/dal.git", "repository": "https://github.com/nesterow/dal.git",
"publishConfig": { "publishConfig": {
"registry": "https://npm.pkg.github.com" "registry": "https://npm.pkg.github.com"
}, },

View file

@ -1,8 +1,5 @@
{ {
"files": [ "files": ["dal/index.ts", "dal/Binding.ts"],
"dal/index.ts",
"dal/SQLite.ts",
],
"compilerOptions": { "compilerOptions": {
"target": "ESNext", "target": "ESNext",
"module": "ESNext", "module": "ESNext",
@ -23,6 +20,6 @@
"noUnusedLocals": false, "noUnusedLocals": false,
"noUnusedParameters": false, "noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false, "noPropertyAccessFromIndexSignature": false,
"outDir": "dist", "outDir": "dist"
} }
} }