[wip] #1 nodejs addon bench (memory)

Signed-off-by: Anton Nesterov <anton@demiurg.io>

.gitignore

@@ -3,6 +3,9 @@
 *.a
 *.dylib
 build/
+database.sqlite
+database.sqlite-shm
+database.sqlite-wal
 # Logs
 
 logs

dal/__test__/bench.node.cjs

@@ -0,0 +1,80 @@
+const fs = require("fs");
+const dal = require("../../build/Release/dal.node");
+
+class Stats {
+  constructor() {
+    this.calls = 0;
+    this.avg_rss = 0;
+    this.avg_heapTotal = 0;
+    this.avg_heapUsed = 0;
+    this.avg_external = 0;
+  }
+  add(mem) {
+    this.calls++;
+    this.avg_rss += mem.rss;
+    this.avg_heapTotal += mem.heapTotal;
+    this.avg_heapUsed += mem.heapUsed;
+    this.avg_external += mem.external;
+  }
+  avg() {
+    const n = this.calls;
+    this.avg_rss /= n;
+    this.avg_heapTotal /= n;
+    this.avg_heapUsed /= n;
+    this.avg_external /= n;
+  }
+  print() {
+    console.log(`
+AVERAGE:
+rss: ${mb(this.avg_rss)} Mb
+external: ${mb(this.avg_external)} Mb
+buffers: ${mb(this.avg_heapUsed)} Mb
+total: ${mb(this.avg_heapTotal)} Mb`);
+  }
+}
+const stats = new Stats();
+let prevMem = process.memoryUsage();
+stats.add(prevMem);
+const mb = (num) => Math.round(num / 1024 / 1024);
+function MEM(when = "") {
+  const mem = process.memoryUsage();
+  stats.add(mem);
+  console.log(`
+${when}
+rss: ${mb(mem.rss)} Mb [delta> ${mem.rss - prevMem.rss}] 
+external: ${mb(mem.external)} Mb [delta> ${mem.external - prevMem.external}] 
+buffers: ${mb(mem.heapUsed)} Mb [delta> ${mem.heapUsed - prevMem.heapUsed}]
+total: ${mb(mem.heapTotal)} Mb [delta> ${mem.heapTotal - prevMem.heapTotal}]`);
+}
+
+console.time("Time to end");
+MEM("START");
+
+const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
+
+const iter = dal.RowIterator(buf);
+MEM("AFTER INIT");
+let dataTransferedBytes = 0;
+for (let i = 0; i < 10000000000; i++) {
+  const b = iter.next();
+  if (b.length === 0) {
+    break;
+  }
+  dataTransferedBytes += b.length;
+  if (i % 1000000 === 0) {
+    MEM(`ITERATION ${i}`);
+  }
+}
+
+MEM("AFTER ITERATION");
+
+iter.free();
+MEM("AFTER CLEANUP");
+
+console.log("\nData transfered: ", mb(dataTransferedBytes), "Mb");
+console.timeEnd("Time to end");
+setTimeout(() => {
+  MEM("AFTER SOME TIME");
+  stats.avg();
+  stats.print();
+}, 30000);

dal/__test__/builder.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "bun:test";
-import {DAL} from "..";
+import { DAL } from "..";
 
 const options = {
   database: "test.sqlite",

dal/__test__/sqlite.node.cjs

@@ -1,15 +0,0 @@
-const fs = require("fs");
-const dal = require("../../build/Release/dal.node");
-//dal.InitSQLite(Buffer.from([]));
-const buf = fs.readFileSync("./pkg/__test__/proto_test.msgpack");
-//console.log(dal.Handle(buf));
-const iter = dal.RowIterator(buf);
-console.log(iter);
-for (let i = 0; i < 10; i++) {
-  const b = iter.next();
-  if (b.length === 0) {
-    break;
-  }
-  console.log(b.toString());
-}
-iter.free();

pkg/facade/SQLIteShared.go

@@ -28,6 +28,7 @@ func InitSQLite(pragmas []string) {
 
 type RowsIter struct {
 	Result  []byte
+	Columns []string
 	rows    *sql.Rows
 }
 
@@ -83,7 +84,13 @@ func (r *RowsIter) Close() {
 func (r *RowsIter) Next() []byte {
 	columns, _ := r.rows.Columns()
 	types, _ := r.rows.ColumnTypes()
+	if r.Columns == nil {
+		r.Columns = columns
+		cols, _ := proto.MarshalRow(columns)
+		return cols
+	}
 	data := make([]interface{}, len(columns))
+	r.rows.Next()
 	for i := range data {
 		typ := reflect.New(types[i].ScanType()).Interface()
 		data[i] = &typ