[doc] initial docs

Signed-off-by: Anton Nesterov <anton@demiurg.io>
This commit is contained in:
Anton Nesterov 2024-08-16 06:18:45 +02:00
parent 6670c1f55a
commit e5152b2bc5
No known key found for this signature in database
GPG key ID: 59121E8AE2851FB5
6 changed files with 211 additions and 10 deletions

View file

@ -2,6 +2,9 @@
Data Accees Layer for SQL databases written in Go.
## NodeJs Client
Mongodb inspired query interface:
```typescript

View file

@ -1,5 +1,5 @@
import type { Request } from "./Protocol";
import { METHODS, encodeRequest, decodeRowsIterator } from "./Protocol";
import type { Request, ExecResult } from "./Protocol";
import { METHODS, encodeRequest, decodeResponse, decodeRowsIterator } from "./Protocol";
type Primitive = string | number | boolean | null;
@ -18,11 +18,13 @@ interface Filter extends Record<string, unknown> {
$between?: [Primitive, Primitive];
$nbetween?: [Primitive, Primitive];
}
interface FindFilter {
[key: string]: Primitive | Filter | Filter[] | undefined;
}
type JoinCondition = "inner" | "left" | "cross" | "full outer";
type JoinFilter = {
$for: string;
$do: FindFilter;
@ -34,6 +36,7 @@ type SortOptions = Record<string, 1 | -1 | "asc" | "desc">;
type Options = {
database: string;
url: string;
headers?: Record<string, string>;
};
export default class Builder<I extends abstract new (...args: any) => any> {
@ -42,6 +45,7 @@ export default class Builder<I extends abstract new (...args: any) => any> {
private dtoTemplate: new (...args: any) => any = Object;
private methodCalls: Map<string, unknown[]> = new Map(); // one call per method
private headerRow: unknown[] | null = null;
private httpHeaders: Record<string, string> = {};
constructor(opts: Options) {
this.request = {
id: 0,
@ -49,6 +53,9 @@ export default class Builder<I extends abstract new (...args: any) => any> {
commands: [],
};
this.url = opts.url;
if (opts.headers) {
this.httpHeaders = opts.headers;
}
}
private formatRequest(): void {
this.request.commands = [];
@ -143,6 +150,10 @@ export default class Builder<I extends abstract new (...args: any) => any> {
this.methodCalls.set("Tx", []);
return this;
}
As<T extends new (...args: any) => any>(template: T): Builder<T> {
this.dtoTemplate = template;
return this;
}
async *Rows<T = InstanceType<I>>(): AsyncGenerator<T> {
this.formatRequest();
const response = await fetch(this.url, {
@ -150,12 +161,12 @@ export default class Builder<I extends abstract new (...args: any) => any> {
body: new Blob([encodeRequest(this.request)]),
headers: {
"Content-Type": "application/x-msgpack",
...this.httpHeaders,
},
});
if (response.status !== 200) {
throw new Error(await response.text());
}
const iterator = decodeRowsIterator(response.body!);
for await (const row of iterator) {
if (this.headerRow === null) {
@ -166,10 +177,6 @@ export default class Builder<I extends abstract new (...args: any) => any> {
yield this.formatRow(row.r);
}
}
As<T extends new (...args: any) => any>(template: T): Builder<T> {
this.dtoTemplate = template;
return this;
}
async Query<T = InstanceType<I>>(): Promise<T[]> {
const rows = this.Rows();
const result = [];
@ -178,4 +185,20 @@ export default class Builder<I extends abstract new (...args: any) => any> {
}
return result;
}
async Exec(): Promise<ExecResult> {
this.formatRequest();
const response = await fetch(this.url, {
method: "POST",
body: new Blob([encodeRequest(this.request)]),
headers: {
"Content-Type": "application/x-msgpack",
...this.httpHeaders,
},
});
if (response.status !== 200) {
throw new Error(await response.text());
}
const buf = await response.arrayBuffer();
return decodeResponse(new Uint8Array(buf));
}
}

View file

@ -11,17 +11,32 @@ export interface Request {
commands: Method[];
}
export interface ExecResult {
Id: number;
RowsAffected: number;
LastInsertId: number;
}
interface Row {
r: unknown[];
}
export const METHODS =
"In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing|Tx".split(
"|",
);
);
export function encodeRequest(request: Request): Uint8Array {
return encode(request);
}
export interface Row {
r: unknown[];
export function decodeResponse(input: Uint8Array): ExecResult {
const res = decode(input) as {i: number; ra: number; li: number};
return {
Id: res.i,
RowsAffected: res.ra,
LastInsertId: res.li,
};
}
const ROW_TAG = [0x81, 0xa1, 0x72];

BIN
doc/dal-internals.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 89 KiB

160
doc/dal-internals.md Normal file
View file

@ -0,0 +1,160 @@
# DAL Internal Architecture
- The Client is written in TypeScript.
- The DAL server written in Golang.
## Components
(Top to bottom)
- NodeJS Client
- Protocol
- Builder
- DB Adapter
## NodeJs Client
Client consists of a query builder and protocol decoder/encoder.
- Query Builder is a light builder which constructs the query object for the server.
- Protocol is a decoder/encoder that utilizes messagepack.
```bash
------------------
|- dal
|- Builder.ts
|- Protocol.ts
|_ index.ts
|...
```
## Protocol
Protocol utilizes messagepack for encoding and decoding the messages.
There following types of encoded data:
- Row stream
- Query (request)
- Response (exec result)
Locations:
```bash
------------------
|- dal
|- Protocol.ts
|_...
|- pkg
|- proto
|...
|...
```
### Row Stream
- The server sends streaming (chunked) data to the client, every chunk is a row.
- Every row starts with a 3-byte header `{0x81, 0xa1, 0x72}`
- The first row is the header row, which contains the column names.
Parsing the row stream (pseudo code):
```python
header = [0x81, 0xa1, 0x72]
input: byte[] = ...
buffer: byte[] = []
output: byte[][] = []
while i < input.length:
if input[i] != 0x81:
buffer << input[i]
i += 1
else if input[i:3] == header:
output << header + buffer
buffer = []
i += 3
output << header + buffer
```
MessagePack schema for the row stream:
```go
type Row struct {
Data []interface{} `msg:"r"`
}
// { r: [] }
```
### Query
- The client utilizes a "light builder" which prepares a list of callbacks for the SQL query builder.
- The Query consits of the following fields:
- Id: uint32 (optional)
- Db: string (required, database name)
- Commands: []BuilderMethod (required, list of Builder arguments)
```go
type BuilderMethod struct {
Method string `msg:"method"`
Args []interface{} `msg:"args"`
}
type Request struct {
Id uint32 `msg:"id"`
Db string `msg:"db"`
Commands []BuilderMethod `msg:"commands"`
}
```
### Response
The response is inteneded for operation results that don't return rows.
```go
type Response struct {
Id uint32 `msg:"i"`
RowsAffected int64 `msg:"ra"`
LastInsertId int64 `msg:"li"`
}
```
## Builder
The builder is a set of methods that are used to construct the SQL query.
- The sql query is constructed by the server.
- The client utilizes a "light builder" which prepares a list of callbacks for the server builer.
Locations:
```bash
------------------
|- dal
|- Builder.ts
|_...
|- pkg
|- builder
|...
|- filters
|...
|...
```
### Builder Methods
In|Find|Select|Fields|Join|Group|Sort|Limit|Offset|Delete|Insert|Set|Update|OnConflict|DoUpdate|DoNothing
[TS Docs]()
[Golang Docs]()
## DB Adapter
- Adapter provides the interface for the database driver.
- Adapter package also provides utilitities for specific SQL Dialects.
Locations:
```bash
------------------
|- pkg
|- Adapter
|...
|...
```

BIN
doc/dal-internals.pdf Normal file

Binary file not shown.