Merge CSV2 Branch into lix-integration

lix/packages/csv-app2-server/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

lix/packages/csv-app2-server/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

lix/packages/csv-app2-server/.prettierrc.json

@@ -0,0 +1,4 @@
+{
+	"useTabs": true,
+	"trailingComma": "es5"
+}

lix/packages/csv-app2-server/README.md

@@ -0,0 +1,8 @@
+```
+npm install
+npm run dev
+```
+
+```
+open http://localhost:3000
+```

lix/packages/csv-app2-server/package.json

@@ -0,0 +1,19 @@
+{
+	"name": "csv-app2-server",
+	"type": "module",
+	"scripts": {
+		"dev": "tsx watch src/index.ts",
+		"test": "vitest run"
+	},
+	"dependencies": {
+		"@hono/node-server": "^1.13.1",
+		"@lix-js/sdk": "workspace:*",
+		"@lix-js/csv-app2": "workspace:*",
+		"hono": "^4.6.3"
+	},
+	"devDependencies": {
+		"@types/node": "^20.11.17",
+		"tsx": "^4.7.1",
+		"vitest": "^2.0.5"
+	}
+}

lix/packages/csv-app2-server/src/endpoint.test.ts

@@ -0,0 +1,135 @@
+import { describe, test, expect } from "vitest";
+import { app } from "./index";
+import fs from "node:fs/promises";
+import { newLixFile, openLixInMemory, merge } from "@lix-js/sdk";
+import { plugin } from "@lix-js/csv-app2/lix-csv-plugin";
+
+describe("hono app tes", () => {
+	test("get /lix-file/:id non existing", async () => {
+		const res = await app.request("/lix-file/i-dont-exist");
+		expect(res.status).toBe(400);
+	});
+
+	test("post /lix-file/:id", async () => {
+		const projectName = "new-one";
+		const user1newLixProjectBlob = await newLixFile();
+		const user1newProject = await openLixInMemory({
+			blob: user1newLixProjectBlob,
+			providePlugins: [plugin],
+		});
+
+		const demoCsv = await fs.readFile("./testdata/demo.csv", "utf-8");
+
+		user1newProject.currentAuthor.set("Demo User1");
+		await user1newProject.db
+			.insertInto("file")
+			.values([
+				{
+					id: "demo1",
+					path: "/data.csv",
+					data: await new Blob([demoCsv]).arrayBuffer(),
+					metadata: {
+						unique_column: "Stakeholder",
+						demo: true,
+					},
+				},
+			])
+			.execute();
+
+		const res = await app.request("/lix-file/" + projectName, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/octet-stream",
+			},
+			body: await user1newProject.toBlob(),
+		});
+
+		const content = await res.arrayBuffer();
+
+		const serverLix = await openLixInMemory({
+			blob: new Blob([content]),
+			providePlugins: [plugin],
+		});
+
+		const res2 = await app.request("/lix-file/" + projectName);
+		const content2 = await res2.arrayBuffer();
+
+		expect(new Blob([content])).toStrictEqual(new Blob([content2]));
+
+		const user2Lix = await openLixInMemory({
+			blob: new Blob([content]),
+			providePlugins: [plugin],
+		});
+		user2Lix.currentAuthor.set("Demo User2");
+
+		await user2Lix.db
+			.insertInto("file")
+			.values([
+				{
+					id: "demo2",
+					path: "/data2.csv",
+					data: await new Blob([demoCsv]).arrayBuffer(),
+					metadata: {
+						unique_column: "Stakeholder",
+						demo: true,
+					},
+				},
+			])
+			.execute();
+
+		await user2Lix.db
+			.insertInto("file")
+			.values([
+				{
+					id: "Demo User2_position",
+					path: "/Demo User2_position.json",
+					data: new TextEncoder().encode(
+						JSON.stringify({
+							position: { rowId: "test", cellName: "column-name" },
+						})
+					),
+				},
+			])
+			.execute();
+
+		await user2Lix.settled();
+		const resUser2Push = await app.request("/lix-file/" + projectName, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/octet-stream",
+			},
+			body: await user2Lix.toBlob(),
+		});
+
+		const resUser1Push = await app.request("/lix-file/" + projectName, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/octet-stream",
+			},
+			body: await user1newProject.toBlob(),
+		});
+
+		const user1PushResult = await openLixInMemory({
+			blob: new Blob([await resUser1Push.arrayBuffer()]),
+			providePlugins: [plugin],
+		});
+
+		// await merge({
+		// 	sourceLix: user1PushResult,
+		// 	targetLix: user1newProject,
+		// });
+
+		const changes = await user1newProject.db
+			.selectFrom("change")
+			.selectAll()
+			.where("change.file_id", "=", "demo1")
+			.execute();
+		// console.log(await
+		// 	user1PushResult
+		// 		.db.selectFrom("file")
+		// 		.selectAll()
+		// 		// .where("path", "like", "%.json")
+		// 		.execute()
+		// );
+	});
+});

lix/packages/csv-app2-server/src/index.ts

@@ -0,0 +1,108 @@
+import { serve } from "@hono/node-server";
+import { openLixInMemory, merge } from "@lix-js/sdk";
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+import { plugin } from "@lix-js/csv-app2/lix-csv-plugin";
+
+export const app = new Hono();
+
+// Apply CORS middleware globally
+app.use('*', cors({
+  origin: '*', 
+  allowMethods: ['GET', 'POST', 'OPTIONS'], 
+  allowHeaders: ['Content-Type', 'Authorization'], 
+}));
+
+setInterval(() => {
+	const before = process.memoryUsage().heapUsed / 1024 / 1024;
+	global.gc?.();
+	const after = process.memoryUsage().heapUsed / 1024 / 1024;
+	console.log({ before: before.toFixed(2), after: after.toFixed(2) });
+}, 10_000);
+
+const lixFiles = new Map<string, Blob>();
+
+// A Map to track locks for each ID to ensure synchronization
+const locks = new Map<string, Promise<Response>>();
+
+// Function to handle locking per ID
+async function synchronized(id: string, fn: () => Promise<Response>) {
+	const previous = locks.get(id) || Promise.resolve();
+	const current = previous.then(fn).finally(() => {
+		// Once done, remove the lock for the ID
+		if (locks.get(id) === current) {
+			locks.delete(id);
+		}
+	});
+	locks.set(id, current);
+	return current;
+}
+
+app.post("/lix-file/:id", async (c) => {
+	const { id } = c.req.param();
+
+	// Get the binary data from the request body
+	const buffer = await c.req.arrayBuffer();
+	const clientBlob = new Uint8Array(buffer);
+
+	const serverStateBlob = lixFiles.get(id);
+
+	return synchronized(id, async () => {
+		if (!serverStateBlob) {
+			const serverLix = await openLixInMemory({
+				blob: new Blob([clientBlob]),
+				providePlugins: [plugin],
+			});
+
+			lixFiles.set(id, await serverLix.toBlob());
+
+			await serverLix.close();
+			// Return the new binary data
+			return c.body(await lixFiles.get(id)!.arrayBuffer(), 200, {
+				"Content-Type": "application/octet-stream",
+			});
+		}
+
+		const clientLix = await openLixInMemory({
+			blob: new Blob([clientBlob]),
+			providePlugins: [plugin],
+		});
+
+		const serverLix = await openLixInMemory({
+			blob: serverStateBlob,
+			providePlugins: [plugin],
+		});
+
+		// console.log(clientLix.db.selectFrom())
+
+		await merge({ sourceLix: clientLix, targetLix: serverLix });
+		lixFiles.set(id, await serverLix.toBlob());
+
+		await serverLix.close();
+
+		await clientLix.close();
+		// Return the new binary data
+		return c.body(await lixFiles.get(id)!.arrayBuffer(), 200, {
+			"Content-Type": "application/octet-stream",
+		});
+	});
+});
+
+app.get("/lix-file/:id", async (c) => {
+	const { id } = c.req.param();
+
+	if (!lixFiles.get(id)) {
+		return c.text("no lix file found", 400);
+	}
+
+	return c.body(await lixFiles.get(id)!.arrayBuffer(), 200, {
+		"Content-Type": "application/octet-stream",
+	});
+});
+
+const port = process.env.PORT ? Number(process.env.PORT) : 3000;
+
+serve({
+	fetch: app.fetch,
+	port,
+});

lix/packages/csv-app2-server/testdata/demo.csv

@@ -0,0 +1,21 @@
+seq;name;first;last
+1;Pearl Griffin;Adelaide;Hopkins
+2;Gregory Perry;Dorothy;Stanley
+3;Noah Young;Johnny;Nichols
+4;John Ray;Philip;Long
+5;Eliza Lawson;Rhoda;Hawkins
+6;Fanny Pearson;Cameron;Malone
+7;Christina Haynes;Walter;Miller
+8;Cora Rowe;Tom;Cummings
+9;Carl Green;Timothy;Murray
+10;Susan Patton;Alma;Wheeler
+11;Justin Lowe;Ralph;Stephens
+12;Franklin Watkins;Lloyd;Franklin
+13;Glen West;Rachel;Joseph
+14;Frances Henderson;Susan;Luna
+15;Roy Simmons;Lillie;Goodman
+16;Mike Barber;Louise;Chavez
+17;Cecilia Hoffman;Leonard;James
+18;Estelle Pierce;Gavin;Barrett
+19;Clayton Zimmerman;Douglas;Wheeler
+20;Arthur Wells;Ann;Burns

lix/packages/csv-app2-server/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "skipLibCheck": true,
+    "types": [
+      "node"
+    ],
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx",
+  }
+}

lix/packages/csv-app2/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}