diff --git a/.gitignore b/.gitignore index 4224cb62..621c9457 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ pkg/ /client/certs /client/coverage +/client/test-report.xml /client-e2e/test/files/sizes/ dist/ node_modules/ diff --git a/client-e2e/scripts/generate-CI-test-files.sh b/client-e2e/scripts/generate-CI-test-files.sh index 0687fa2b..dffe393e 100755 --- a/client-e2e/scripts/generate-CI-test-files.sh +++ b/client-e2e/scripts/generate-CI-test-files.sh @@ -3,5 +3,5 @@ if [ ! -d "./test/files/sizes" ]; then mkdir ./test/files/sizes fi cd ./test/files/sizes -dd if=/dev/zero of=5MB bs=5000000 count=1 -dd if=/dev/zero of=20MB bs=20000000 count=1 +dd if=/dev/zero of=5MB.bin bs=5000000 count=1 +dd if=/dev/zero of=20MB.bin bs=20000000 count=1 diff --git a/client-e2e/scripts/generate-sized-test-files.sh b/client-e2e/scripts/generate-sized-test-files.sh index 2e303897..1d99065c 100755 --- a/client-e2e/scripts/generate-sized-test-files.sh +++ b/client-e2e/scripts/generate-sized-test-files.sh @@ -2,9 +2,9 @@ if [ ! -d "/usr/src/app/test/files/sizes" ]; then mkdir /usr/src/app/test/files/sizes cd /usr/src/app/test/files/sizes - dd if=/dev/zero of=5MB bs=5000000 count=1 - dd if=/dev/zero of=20MB bs=20000000 count=1 - dd if=/dev/zero of=300MB bs=300000000 count=1 - dd if=/dev/zero of=4.2GB bs=1000000 count=4200 - dd if=/dev/zero of=4.3GB bs=1000000 count=4300 + dd if=/dev/zero of=5MB.bin bs=5000000 count=1 + dd if=/dev/zero of=20MB.bin bs=20000000 count=1 + dd if=/dev/zero of=300MB.bin bs=300000000 count=1 + dd if=/dev/zero of=4.2GB.bin bs=1000000 count=4200 + dd if=/dev/zero of=4.3GB.bin bs=1000000 count=4300 fi \ No newline at end of file diff --git a/client-e2e/test/specs/cancellation.ts b/client-e2e/test/specs/cancellation.ts index 6881050d..9d6d2634 100644 --- a/client-e2e/test/specs/cancellation.ts +++ b/client-e2e/test/specs/cancellation.ts @@ -8,7 +8,7 @@ describe("Cancellation", () => { await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); await (await Page.cancelButton()).click(); @@ -27,7 +27,7 @@ describe("Cancellation", () => { it("Sends the Sender back and show cancellation message, no message for Receiver", async function () { await Page.open(); const sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); const _receiveWindow = await browser.newWindow(codeUrl); @@ -50,7 +50,7 @@ describe("Cancellation", () => { it("Sends the Receiver and Sender back. The Sender gets an error message", async function () { await Page.open(); const sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); const _receiveWindow = await browser.newWindow(codeUrl); @@ -78,7 +78,7 @@ describe("Cancellation", () => { it("(after 5 sec.) Sends the Receiver and Sender back. The Sender gets an error message", async function () { await Page.open(); const sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); const _receiveWindow = await browser.newWindow(codeUrl); @@ -112,7 +112,7 @@ describe("Cancellation", () => { it("Sends the Receiver and Sender back. The Receiver gets an error message", async function () { await Page.open(); const sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); // Receiver @@ -143,7 +143,7 @@ describe("Cancellation", () => { it("Sends the Sender back with no message", async function () { await Page.open(); const sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); const _receiveWindow = await browser.newWindow(codeUrl); diff --git a/client-e2e/test/specs/pages.ts b/client-e2e/test/specs/pages.ts index d6ff3f21..fae9e67c 100644 --- a/client-e2e/test/specs/pages.ts +++ b/client-e2e/test/specs/pages.ts @@ -81,7 +81,7 @@ describe("Check internal Pages", () => { await Page.open(); // Sender const _sendWindow = await browser.getWindowHandle(); - await Page.uploadFiles("./test/files/sizes/20MB"); + await Page.uploadFiles("./test/files/sizes/20MB.bin"); const codeUrl = await Page.getCodeUrl(); // Receiver diff --git a/client-e2e/test/specs/send-large-files.ts b/client-e2e/test/specs/send-large-files.ts index e1c7aedd..96babeca 100644 --- a/client-e2e/test/specs/send-large-files.ts +++ b/client-e2e/test/specs/send-large-files.ts @@ -45,21 +45,21 @@ async function testTransferFailure(fileName: string, timeout?: number) { describe("Send flow large files", () => { describe("when uploading a file with the size of 300MB", () => { it("will tell the user that the file is too large", async () => { - await testTransferFailure("sizes/300MB"); + await testTransferFailure("sizes/300MB.bin"); }); }); describe("when uploading a file with the size of 4.2GB", () => { it("will tell the user that the file is too large", async function () { this.timeout(120000); - await testTransferFailure("sizes/4.2GB"); + await testTransferFailure("sizes/4.2GB.bin"); }); }); describe("when uploading a file with the size of 4.3GB", () => { it("will tell the user that the file is too large", async function () { this.timeout(120000); - await testTransferFailure("sizes/4.3GB"); + await testTransferFailure("sizes/4.3GB.bin"); }); }); }); diff --git a/client-e2e/test/specs/send.ts b/client-e2e/test/specs/send.ts index e9862065..887aa2af 100644 --- a/client-e2e/test/specs/send.ts +++ b/client-e2e/test/specs/send.ts @@ -73,7 +73,7 @@ describe("Send flow", () => { describe("when uploading a file with the size of 5MB", () => { it("will transfer successfully", async () => { // 100 sec. - await testTransferSuccess("sizes/5MB", 100000); + await testTransferSuccess("sizes/5MB.bin", 100000); }); }); @@ -81,7 +81,7 @@ describe("Send flow", () => { // TODO: enable, when speed issue fixed, currently fails with Firefox it.skip("will transfer successfully", async () => { // 100 sec. - await testTransferSuccess("sizes/20MB", 100000); + await testTransferSuccess("sizes/20MB.bin", 100000); }); }); diff --git a/client/package-lock.json b/client/package-lock.json index 92081677..c94624b1 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -16,6 +16,7 @@ "classnames": "^2.3.1", "fastest-levenshtein": "^1.0.12", "formik": "^2.2.9", + "nanoid": "^4.0.2", "nosleep.js": "^0.12.0", "react": "^18.0.0", "react-dom": "^18.0.0", @@ -10029,6 +10030,24 @@ "node": ">=8" } }, + "node_modules/@storybook/telemetry/node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/@storybook/telemetry/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -14640,6 +14659,24 @@ "node": ">=10" } }, + "node_modules/css-loader/node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/css-loader/node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -23035,15 +23072,20 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", - "dev": true, + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-4.0.2.tgz", + "integrity": "sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "bin": { - "nanoid": "bin/nanoid.cjs" + "nanoid": "bin/nanoid.js" }, "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + "node": "^14 || ^16 || >=18" } }, "node_modules/nanomatch": { @@ -36884,6 +36926,12 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true + }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -40641,6 +40689,12 @@ "yallist": "^4.0.0" } }, + "nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true + }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -47086,10 +47140,9 @@ "dev": true }, "nanoid": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", - "dev": true + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-4.0.2.tgz", + "integrity": "sha512-7ZtY5KTCNheRGfEFxnedV5zFiORN1+Y1N6zvPTnHQd8ENUvfaDBeuJDZb2bN/oXwXxu3qkTXDzy57W5vAmDTBw==" }, "nanomatch": { "version": "1.2.13", diff --git a/client/package.json b/client/package.json index ea5842a3..4a29062a 100644 --- a/client/package.json +++ b/client/package.json @@ -21,6 +21,7 @@ "classnames": "^2.3.1", "fastest-levenshtein": "^1.0.12", "formik": "^2.2.9", + "nanoid": "^4.0.2", "nosleep.js": "^0.12.0", "react": "^18.0.0", "react-dom": "^18.0.0", diff --git a/client/src/app/components/CodeInput.test.tsx b/client/src/app/components/CodeInput.test.tsx index 1fb74705..229efaa9 100644 --- a/client/src/app/components/CodeInput.test.tsx +++ b/client/src/app/components/CodeInput.test.tsx @@ -11,6 +11,8 @@ const emptyLine = String.fromCharCode(8203); global.ResizeObserver = require("resize-observer-polyfill"); +jest.mock("../util/downloader/downloader.ts"); + // Sometimes tests fails on Github CI jest.retryTimes(1); diff --git a/client/src/app/sagas.ts b/client/src/app/sagas.ts index db7cf3bb..4ee451ab 100644 --- a/client/src/app/sagas.ts +++ b/client/src/app/sagas.ts @@ -4,11 +4,11 @@ import streamSaver from "streamsaver"; import { Client, ReceiveResult, SendResult } from "../../pkg"; import { setError } from "./errorSlice"; import { NoSleep } from "./NoSleep"; +import { close, createStream, write } from "./util/downloader/downloader"; import { makeProgressFunc } from "./util/makeProgressFunc"; import { completeLoading, completeTransfer, - requestCancelTransfer, reset, selectWormholeStatus, setConsenting, @@ -130,22 +130,14 @@ function* transfer(): any { "wormhole/answerConsent" ); if (consentPayload) { - const fileStream = streamSaver.createWriteStream( - receiveResult.get_file_name(), - { - size: Number(receiveResult.file_size), - } - ); - writer = fileStream.getWriter(); + const fileName = receiveResult.get_file_name(); + const stream = yield createStream(receiveResult.get_file_name()); yield pkg.download_file( receiveResult, { - write: (x: unknown) => - writer!.write(x).catch((e) => { - // If `writer.write` throws, the user cancelled the download through the browser's download manager. - console.error("Failed to write:", e); - transferChannel.put(requestCancelTransfer()); - }), + write: (x: Uint8Array) => { + write(stream, x); + }, progress: makeProgressFunc((sentBytes, totalBytes) => { transferChannel.put( setTransferProgress([sentBytes, totalBytes]) @@ -154,7 +146,7 @@ function* transfer(): any { }, cancel ); - yield writer.close(); + yield close(stream); yield put(completeTransfer()); } else { yield pkg.reject_file(receiveResult); diff --git a/client/src/app/util/downloader/__mocks__/downloader.ts b/client/src/app/util/downloader/__mocks__/downloader.ts new file mode 100644 index 00000000..5db0519d --- /dev/null +++ b/client/src/app/util/downloader/__mocks__/downloader.ts @@ -0,0 +1 @@ +// no need for a mock implementation (yet) diff --git a/client/src/app/util/downloader/cleanerWorker.ts b/client/src/app/util/downloader/cleanerWorker.ts new file mode 100644 index 00000000..9b7c2a4e --- /dev/null +++ b/client/src/app/util/downloader/cleanerWorker.ts @@ -0,0 +1,17 @@ +async function cleanup() { + const directoryHandle = await navigator.storage.getDirectory(); + const now = Date.now(); + // @ts-ignore + for await (const [filename, handle] of directoryHandle.entries()) { + const file: File = await handle.getFile(); + if (now >= file.lastModified + 60000) { + directoryHandle.removeEntry(filename); + } + } +} + +cleanup(); + +setInterval(() => { + cleanup(); +}, 30000); diff --git a/client/src/app/util/downloader/downloader.ts b/client/src/app/util/downloader/downloader.ts new file mode 100644 index 00000000..45a9f619 --- /dev/null +++ b/client/src/app/util/downloader/downloader.ts @@ -0,0 +1,62 @@ +import { nanoid } from "nanoid"; + +// @ts-ignore +const worker = new Worker(new URL("./worker.ts", import.meta.url)); +const cleanerWorker = new Worker( + // @ts-ignore + new URL("./cleanerWorker.ts", import.meta.url) +); + +type StreamData = { + ready: boolean; +}; + +const streams: Record = {}; + +function waitFor(fn: () => boolean) { + return new Promise((resolve) => { + const interval = setInterval(() => { + if (fn()) { + clearInterval(interval); + resolve(); + } + }, 100); + }); +} + +export async function createStream(filename: string): Promise { + const id = nanoid(); + worker.postMessage({ type: "createStream", id, filename }); + streams[id] = { ready: false }; + await waitFor(() => { + return streams[id].ready; + }); + return id; +} + +export function write(id: string, data: Uint8Array) { + worker.postMessage({ type: "write", id, data }); +} + +export async function close(id: string) { + worker.postMessage({ type: "closeStream", id }); + await waitFor(() => { + return !streams[id]; + }); +} + +worker.addEventListener("message", async (e) => { + if (e.data.type === "streamCreated") { + streams[e.data.id].ready = true; + } else if (e.data.type === "streamClosed") { + const directoryHandle = await navigator.storage.getDirectory(); + const fileHandle = await directoryHandle.getFileHandle(e.data.id); + const file = await fileHandle.getFile(); + + var link = document.createElement("a"); + link.download = e.data.filename; + link.href = URL.createObjectURL(file); + link.click(); + delete streams[e.data.id]; + } +}); diff --git a/client/src/app/util/downloader/worker.ts b/client/src/app/util/downloader/worker.ts new file mode 100644 index 00000000..fa2d395d --- /dev/null +++ b/client/src/app/util/downloader/worker.ts @@ -0,0 +1,59 @@ +export type MessageData = + | { + type: "createStream"; + id: string; + filename: string; + } + | { + type: "write"; + id: string; + data: Uint8Array; + } + | { + type: "closeStream"; + id: string; + }; + +type StreamData = { + filename: string; + accessHandle: any; + offset: number; +}; + +const streams: Record = {}; + +onmessage = async (e: MessageEvent) => { + if (e.data.type === "createStream") { + // Get handle to draft file in OPFS + const root = await navigator.storage.getDirectory(); + const draftHandle = await root.getFileHandle(e.data.id, { + create: true, + }); + // Get sync access handle + // @ts-ignore + const accessHandle = await draftHandle.createSyncAccessHandle(); + // save for later use + streams[e.data.id] = { + filename: e.data.filename, + accessHandle, + offset: 0, + }; + self.postMessage({ type: "streamCreated", id: e.data.id }); + } else if (e.data.type === "write") { + const stream = streams[e.data.id]; + const bytesWritten = await stream.accessHandle.write(e.data.data, { + at: stream.offset, + }); + stream.offset += bytesWritten; + } else if (e.data.type === "closeStream") { + const stream = streams[e.data.id]; + stream.accessHandle.flush(); + stream.accessHandle.close(); + self.postMessage({ + type: "streamClosed", + id: e.data.id, + filename: streams[e.data.id].filename, + }); + delete streams[e.data.id]; + } +}; diff --git a/client/wasm/src/lib.rs b/client/wasm/src/lib.rs index 9d963fa6..5eabfcd2 100644 --- a/client/wasm/src/lib.rs +++ b/client/wasm/src/lib.rs @@ -313,14 +313,12 @@ pub async fn reject_file(mut receive_result: ReceiveResult) -> Result>, } impl FileWriter { fn new(writer: JsValue) -> Self { FileWriter { writer, - f: Box::new(None), } } } @@ -345,30 +343,11 @@ impl AsyncWrite for FileWriter { panic!("writer.write is not a function") } let write_fn = js_sys::Function::from(write); - if let Some(f) = &mut *self.f { - let p = Pin::new(&mut *f); - match p.poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(_) => { - self.f = Box::new(None); - Poll::Ready(Ok(buf.len())) - } - } - } else { - let abuf = js_sys::ArrayBuffer::new(buf.len() as u32); - let uarr = js_sys::Uint8Array::new(&abuf); - uarr.copy_from(buf); - let write_call = write_fn.call1(&JsValue::UNDEFINED, &uarr.into()); - let returned_promise: js_sys::Promise = write_call.unwrap().into(); - let mut returned_future: JsFuture = returned_promise.into(); - let p = Pin::new(&mut returned_future); - match p.poll(cx) { - _ => { - self.f = Box::new(Some(returned_future)); - Poll::Pending - } - } - } + let abuf = js_sys::ArrayBuffer::new(buf.len() as u32); + let uarr = js_sys::Uint8Array::new(&abuf); + uarr.copy_from(buf); + let write_call = write_fn.call1(&JsValue::UNDEFINED, &uarr.into()); + Poll::Ready(Ok(buf.len())) } }