diff --git a/README.md b/README.md index bf07984..b28a729 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ Data Accees Layer for SQL databases written in Go. + +## NodeJs Client + Mongodb inspired query interface: ```typescript diff --git a/dal/Builder.ts b/dal/Builder.ts index 0b30543..1916a03 100644 --- a/dal/Builder.ts +++ b/dal/Builder.ts @@ -1,5 +1,5 @@ -import type { Request } from "./Protocol"; -import { METHODS, encodeRequest, decodeRowsIterator } from "./Protocol"; +import type { Request, ExecResult } from "./Protocol"; +import { METHODS, encodeRequest, decodeResponse, decodeRowsIterator } from "./Protocol"; type Primitive = string | number | boolean | null; @@ -18,11 +18,13 @@ interface Filter extends Record { $between?: [Primitive, Primitive]; $nbetween?: [Primitive, Primitive]; } + interface FindFilter { [key: string]: Primitive | Filter | Filter[] | undefined; } type JoinCondition = "inner" | "left" | "cross" | "full outer"; + type JoinFilter = { $for: string; $do: FindFilter; @@ -34,6 +36,7 @@ type SortOptions = Record; type Options = { database: string; url: string; + headers?: Record; }; export default class Builder any> { @@ -42,6 +45,7 @@ export default class Builder any> { private dtoTemplate: new (...args: any) => any = Object; private methodCalls: Map = new Map(); // one call per method private headerRow: unknown[] | null = null; + private httpHeaders: Record = {}; constructor(opts: Options) { this.request = { id: 0, @@ -49,6 +53,9 @@ export default class Builder any> { commands: [], }; this.url = opts.url; + if (opts.headers) { + this.httpHeaders = opts.headers; + } } private formatRequest(): void { this.request.commands = []; @@ -143,6 +150,10 @@ export default class Builder any> { this.methodCalls.set("Tx", []); return this; } + As any>(template: T): Builder { + this.dtoTemplate = template; + return this; + } async *Rows>(): AsyncGenerator { this.formatRequest(); const response = await fetch(this.url, { @@ -150,12 +161,12 @@ export default class Builder any> { body: new Blob([encodeRequest(this.request)]), headers: { "Content-Type": "application/x-msgpack", + ...this.httpHeaders, }, }); if (response.status !== 200) { throw new Error(await response.text()); } - const iterator = decodeRowsIterator(response.body!); for await (const row of iterator) { if (this.headerRow === null) { @@ -166,10 +177,6 @@ export default class Builder any> { yield this.formatRow(row.r); } } - As any>(template: T): Builder { - this.dtoTemplate = template; - return this; - } async Query>(): Promise { const rows = this.Rows(); const result = []; @@ -178,4 +185,20 @@ export default class Builder any> { } return result; } + async Exec(): Promise { + this.formatRequest(); + const response = await fetch(this.url, { + method: "POST", + body: new Blob([encodeRequest(this.request)]), + headers: { + "Content-Type": "application/x-msgpack", + ...this.httpHeaders, + }, + }); + if (response.status !== 200) { + throw new Error(await response.text()); + } + const buf = await response.arrayBuffer(); + return decodeResponse(new Uint8Array(buf)); + } } diff --git a/dal/Protocol.ts b/dal/Protocol.ts index 069aef5..4623504 100644 --- a/dal/Protocol.ts +++ b/dal/Protocol.ts @@ -11,17 +11,32 @@ export interface Request { commands: Method[]; } +export interface ExecResult { + Id: number; + RowsAffected: number; + LastInsertId: number; +} + +interface Row { + r: unknown[]; +} + export const METHODS = "In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split( "|", - ); +); export function encodeRequest(request: Request): Uint8Array { return encode(request); } -export interface Row { - r: unknown[]; +export function decodeResponse(input: Uint8Array): ExecResult { + const res = decode(input) as {i: number; ra: number; li: number}; + return { + Id: res.i, + RowsAffected: res.ra, + LastInsertId: res.li, + }; } const ROW_TAG = [0x81, 0xa1, 0x72]; diff --git a/doc/dal-internals.jpg b/doc/dal-internals.jpg new file mode 100644 index 0000000..aaabbc1 Binary files /dev/null and b/doc/dal-internals.jpg differ diff --git a/doc/dal-internals.md b/doc/dal-internals.md new file mode 100644 index 0000000..8c3f1b4 --- /dev/null +++ b/doc/dal-internals.md @@ -0,0 +1,160 @@ +# DAL Internal Architecture + +- The Client is written in TypeScript. +- The DAL server written in Golang. + +## Components + +(Top to bottom) + +- NodeJS Client +- Protocol +- Builder +- DB Adapter + +## NodeJs Client + +Client consists of a query builder and protocol decoder/encoder. + +- Query Builder is a light builder which constructs the query object for the server. +- Protocol is a decoder/encoder that utilizes messagepack. + +```bash +------------------ +|- dal + |- Builder.ts + |- Protocol.ts + |_ index.ts +|... +``` + +## Protocol + + +Protocol utilizes messagepack for encoding and decoding the messages. + +There following types of encoded data: +- Row stream +- Query (request) +- Response (exec result) + +Locations: + +```bash +------------------ +|- dal + |- Protocol.ts +|_... +|- pkg + |- proto + |... +|... +``` + +### Row Stream + +- The server sends streaming (chunked) data to the client, every chunk is a row. +- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}` +- The first row is the header row, which contains the column names. + +Parsing the row stream (pseudo code): + +```python +header = [0x81, 0xa1, 0x72] +input: byte[] = ... +buffer: byte[] = [] +output: byte[][] = [] +while i < input.length: + if input[i] != 0x81: + buffer << input[i] + i += 1 + else if input[i:3] == header: + output << header + buffer + buffer = [] + i += 3 +output << header + buffer +``` + +MessagePack schema for the row stream: +```go +type Row struct { + Data []interface{} `msg:"r"` +} +// { r: [] } +``` + +### Query + +- The client utilizes a "light builder" which prepares a list of callbacks for the SQL query builder. +- The Query consits of the following fields: + - Id: uint32 (optional) + - Db: string (required, database name) + - Commands: []BuilderMethod (required, list of Builder arguments) + + + +```go +type BuilderMethod struct { + Method string `msg:"method"` + Args []interface{} `msg:"args"` +} + +type Request struct { + Id uint32 `msg:"id"` + Db string `msg:"db"` + Commands []BuilderMethod `msg:"commands"` +} +``` + +### Response +The response is inteneded for operation results that don't return rows. + +```go +type Response struct { + Id uint32 `msg:"i"` + RowsAffected int64 `msg:"ra"` + LastInsertId int64 `msg:"li"` +} +``` + +## Builder + +The builder is a set of methods that are used to construct the SQL query. + +- The sql query is constructed by the server. +- The client utilizes a "light builder" which prepares a list of callbacks for the server builer. + +Locations: + +```bash +------------------ +|- dal + |- Builder.ts +|_... +|- pkg + |- builder + |... + |- filters + |... +|... +``` + +### Builder Methods +In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing +[TS Docs]() +[Golang Docs]() + +## DB Adapter + +- Adapter provides the interface for the database driver. +- Adapter package also provides utilitities for specific SQL Dialects. + +Locations: + +```bash +------------------ +|- pkg + |- Adapter + |... +|... +``` \ No newline at end of file diff --git a/doc/dal-internals.pdf b/doc/dal-internals.pdf new file mode 100644 index 0000000..182f286 Binary files /dev/null and b/doc/dal-internals.pdf differ