From 454b77b313bc04f623c4168f085d1fd7f0a2b773 Mon Sep 17 00:00:00 2001 From: Nicolet Date: Tue, 18 Feb 2025 16:28:24 -0600 Subject: [PATCH] use buffers instead of temp files, send whole zip as buffer over websocket verified zip to disk works #104 fixes #107 --- src/main/host.ts | 445 ++++------------------------------ src/main/replay.ts | 194 +++++++-------- src/renderer/CopyControls.tsx | 5 +- 3 files changed, 145 insertions(+), 499 deletions(-) diff --git a/src/main/host.ts b/src/main/host.ts index 24b6389..4b7a369 100644 --- a/src/main/host.ts +++ b/src/main/host.ts @@ -1,20 +1,12 @@ -import { app, BrowserWindow } from 'electron'; +import { BrowserWindow } from 'electron'; import { createSocket, Socket } from 'dgram'; import os from 'node:os'; import { execSync } from 'node:child_process'; import { WebSocketServer, WebSocket, MessageEvent } from 'ws'; -import { FileHandle, mkdir, open, readdir, rm, writeFile } from 'fs/promises'; +import { writeFile } from 'fs/promises'; import path from 'node:path'; -import { ZipFile } from 'yazl'; -import { createWriteStream } from 'fs'; import { IncomingMessage } from 'http'; -import { - Context, - CopyHost, - CopyClient, - Output, - WebSocketServerStatus, -} from '../common/types'; +import { CopyHost, CopyClient, WebSocketServerStatus } from '../common/types'; const PORT = 52455; @@ -37,31 +29,7 @@ type HostRequest = { name: string; } | { - request: 'initSubdir'; - subdir: string; - output: Output; - } - | { - request: 'writeContext'; - subdir: string; - context: Context; - } - | { - request: 'openReplay'; - subdir: string; - fileName: string; - } - | { - request: 'writeReplayData'; - fd: number; - data: number[]; - } - | { - request: 'closeReplay'; - fd: number; - } - | { - request: 'zipSubdir'; + request: 'writeZip'; subdir: string; } ); @@ -246,53 +214,16 @@ export function getHost(): CopyHost { } let nextRequestOrdinal = 1; -export function initSubdir(subdir: string, output: Output) { - return new Promise((resolve, reject) => { - if (!webSocket) { - reject(new Error('no webSocket')); - return; - } - - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; +export async function writeZip(subdir: string, buffer: Buffer) { + const requestOrdinal = nextRequestOrdinal; + nextRequestOrdinal += 1; - const listener = (event: MessageEvent) => { - if (typeof event.data !== 'string') { - return; - } - - const { ordinal, error } = JSON.parse(event.data) as HostResponse; - if (ordinal === requestOrdinal) { - webSocket?.removeEventListener('message', listener); - if (error) { - reject(new Error(error)); - } else { - resolve(); - } - } - }; - webSocket.addEventListener('message', listener); - - const request: HostRequest = { - ordinal: requestOrdinal, - request: 'initSubdir', - subdir, - output, - }; - webSocket.send(JSON.stringify(request)); - }); -} - -export function writeContext(subdir: string, context: Context) { - return new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { if (!webSocket) { - reject(); + reject(new Error('no webSocket')); return; } - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; - const listener = (event: MessageEvent) => { if (typeof event.data !== 'string') { return; @@ -312,138 +243,17 @@ export function writeContext(subdir: string, context: Context) { const request: HostRequest = { ordinal: requestOrdinal, - request: 'writeContext', + request: 'writeZip', subdir, - context, - }; - webSocket.send(JSON.stringify(request)); - }); -} - -export function openReplay(subdir: string, fileName: string) { - return new Promise((resolve, reject) => { - if (!webSocket) { - reject(); - return; - } - - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; - - const listener = (event: MessageEvent) => { - if (typeof event.data !== 'string') { - return; - } - - const { ordinal, error, fd } = JSON.parse(event.data) as HostResponse & { - fd: number; - }; - if (ordinal === requestOrdinal) { - webSocket?.removeEventListener('message', listener); - if (error) { - reject(new Error(error)); - } else if (Number.isInteger(fd)) { - resolve(fd); - } - } - }; - webSocket.addEventListener('message', listener); - - const request: HostRequest = { - ordinal: requestOrdinal, - request: 'openReplay', - subdir, - fileName, - }; - webSocket.send(JSON.stringify(request)); - }); -} - -export function writeReplayData(fd: number, data: Buffer) { - return new Promise((resolve, reject) => { - if (!webSocket) { - reject(); - return; - } - - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; - - const listener = (event: MessageEvent) => { - if (typeof event.data !== 'string') { - return; - } - - const { ordinal, error, bytesWritten } = JSON.parse( - event.data, - ) as HostResponse & { bytesWritten: number }; - if (ordinal === requestOrdinal) { - webSocket?.removeEventListener('message', listener); - if (error) { - reject(new Error(error)); - } else if (Number.isInteger(bytesWritten)) { - resolve(bytesWritten); - } - } - }; - webSocket.addEventListener('message', listener); - - const request: HostRequest = { - ordinal: requestOrdinal, - request: 'writeReplayData', - fd, - data: data.toJSON().data, }; webSocket.send(JSON.stringify(request)); }); -} - -export function closeReplay(fd: number) { - return new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { if (!webSocket) { - reject(); - return; - } - - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; - - const listener = (event: MessageEvent) => { - if (typeof event.data !== 'string') { - return; - } - - const { ordinal, error } = JSON.parse(event.data) as HostResponse; - if (ordinal === requestOrdinal) { - webSocket?.removeEventListener('message', listener); - if (error) { - reject(new Error(error)); - } else { - resolve(); - } - } - }; - webSocket.addEventListener('message', listener); - - const request: HostRequest = { - ordinal: requestOrdinal, - request: 'closeReplay', - fd, - }; - webSocket.send(JSON.stringify(request)); - }); -} - -export function zipSubdir(subdir: string) { - return new Promise((resolve, reject) => { - if (!webSocket) { - reject(); + reject(new Error('no webSocket')); return; } - const requestOrdinal = nextRequestOrdinal; - nextRequestOrdinal += 1; - const listener = (event: MessageEvent) => { if (typeof event.data !== 'string') { return; @@ -460,13 +270,7 @@ export function zipSubdir(subdir: string) { } }; webSocket.addEventListener('message', listener); - - const request: HostRequest = { - ordinal: requestOrdinal, - request: 'zipSubdir', - subdir, - }; - webSocket.send(JSON.stringify(request)); + webSocket.send(buffer); }); } @@ -545,8 +349,10 @@ export function setOwnFileNameFormat(fileNameFormat: string) { let broadcastSocket: Socket | null = null; let webSocketServer: WebSocketServer | null = null; -const subdirToWriteDir = new Map(); -const fdToFileHandle = new Map(); +const clientAddressToExpectedZip = new Map< + string, + { ordinal: number; subdir: string } +>(); export function startHostServer(): Promise { if (!copyDir) { throw new Error('must set copy dir'); @@ -556,6 +362,7 @@ export function startHostServer(): Promise { } clientAddressToNameAndWebSocket.clear(); + clientAddressToExpectedZip.clear(); sendCopyClients(); webSocketServer = new WebSocketServer({ port: PORT, @@ -610,203 +417,39 @@ export function startHostServer(): Promise { nameAndWebSocket.name = hostRequest.name; sendCopyClients(); } - } else { - if (!clientAddressToNameAndWebSocket.get(remoteAddress)?.name) { + } else if (hostRequest.request === 'writeZip') { + clientAddressToExpectedZip.set(remoteAddress, { + ordinal: hostRequest.ordinal, + subdir: hostRequest.subdir, + }); + const response: HostResponse = { + ordinal: hostRequest.ordinal, + error: '', + }; + newWebSocket.send(JSON.stringify(response)); + } + } else if (event.data instanceof Buffer) { + const expectedZip = clientAddressToExpectedZip.get(remoteAddress); + if (expectedZip) { + try { + await writeFile( + `${path.join(copyDir, expectedZip.subdir)}.zip`, + event.data, + ); const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'must first call computerName', + ordinal: expectedZip.ordinal, + error: '', }; newWebSocket.send(JSON.stringify(response)); - return; - } - - if (hostRequest.request === 'initSubdir') { - const writeDir = path.join( - hostRequest.output === Output.ZIP ? app.getPath('temp') : copyDir, - hostRequest.subdir, - ); - try { - await mkdir(writeDir, { recursive: true }); - subdirToWriteDir.set(hostRequest.subdir, writeDir); - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: '', - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else if (hostRequest.request === 'writeContext') { - const writeDir = subdirToWriteDir.get(hostRequest.subdir); - if (!writeDir) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'invalid subdir', - }; - newWebSocket.send(JSON.stringify(response)); - return; - } - try { - const filePath = path.join(writeDir, 'context.json'); - await writeFile(filePath, JSON.stringify(hostRequest.context)); - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: '', - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else if (hostRequest.request === 'openReplay') { - const writeDir = subdirToWriteDir.get(hostRequest.subdir); - if (!writeDir) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'invalid subdir', - }; - newWebSocket.send(JSON.stringify(response)); - return; - } - try { - const filePath = path.join(writeDir, hostRequest.fileName); - const fileHandle = await open(filePath, 'w'); - fdToFileHandle.set(fileHandle.fd, fileHandle); - const response: HostResponse & { fd: number } = { - ordinal: hostRequest.ordinal, - error: '', - fd: fileHandle.fd, - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else if (hostRequest.request === 'writeReplayData') { - const fileHandle = fdToFileHandle.get(hostRequest.fd); - if (!fileHandle) { + clientAddressToExpectedZip.delete(remoteAddress); + } catch (e: unknown) { + if (e instanceof Error) { const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'invalid fd', + ordinal: expectedZip.ordinal, + error: e.message, }; newWebSocket.send(JSON.stringify(response)); - return; } - - try { - const writeResponse = await fileHandle.write( - Buffer.from(hostRequest.data), - ); - const response: HostResponse & { bytesWritten: number } = { - ordinal: hostRequest.ordinal, - error: '', - bytesWritten: writeResponse.bytesWritten, - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else if (hostRequest.request === 'closeReplay') { - const fileHandle = fdToFileHandle.get(hostRequest.fd); - if (!fileHandle) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'invalid fd', - }; - newWebSocket.send(JSON.stringify(response)); - return; - } - - try { - await fileHandle.close(); - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: '', - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else if (hostRequest.request === 'zipSubdir') { - const writeDir = subdirToWriteDir.get(hostRequest.subdir); - if (!writeDir || !writeDir.startsWith(app.getPath('temp'))) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: 'invalid subdir', - }; - newWebSocket.send(JSON.stringify(response)); - return; - } - - try { - const fileNames = await readdir(writeDir); - const zipFile = new ZipFile(); - const zipFilePromise = new Promise((resolve) => { - zipFile.outputStream - .pipe( - createWriteStream( - `${path.join(copyDir, hostRequest.subdir)}.zip`, - ), - ) - .on('close', resolve); - }); - fileNames.forEach((fileName) => { - zipFile.addFile(path.join(writeDir, fileName), fileName); - }); - zipFile.end(); - await zipFilePromise; - await rm(writeDir, { recursive: true }); - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: '', - }; - newWebSocket.send(JSON.stringify(response)); - } catch (e: unknown) { - if (e instanceof Error) { - const response: HostResponse = { - ordinal: hostRequest.ordinal, - error: e.message, - }; - newWebSocket.send(JSON.stringify(response)); - } - } - } else { - newWebSocket.send( - JSON.stringify({ - ordinal: (hostRequest as HostRequest).ordinal, - error: `unknown request: ${ - (hostRequest as HostRequest).request - }`, - }), - ); } } } diff --git a/src/main/replay.ts b/src/main/replay.ts index 97bee11..0b8336a 100644 --- a/src/main/replay.ts +++ b/src/main/replay.ts @@ -4,23 +4,21 @@ import { open, readFile as fsReadFile, readdir, - rm, - access, FileHandle, + writeFile, } from 'fs/promises'; import { join } from 'path'; import iconv from 'iconv-lite'; import sanitize from 'sanitize-filename'; import { ZipFile } from 'yazl'; -import { createWriteStream } from 'fs'; import { ListChecks, SlippiGame, getCoordListFromGame, isSlpMinVersion, } from 'slp-enforcer'; -import { app } from 'electron'; import { parse } from 'date-fns'; +import { buffer as bufferConsumer } from 'stream/consumers'; import { Context, CopyHost, @@ -36,14 +34,7 @@ import { isValidCharacter, legalStages, } from '../common/constants'; -import { - closeReplay, - initSubdir, - openReplay, - writeContext, - writeReplayData, - zipSubdir, -} from './host'; +import { writeZip } from './host'; // https://github.com/project-slippi/slippi-launcher/blob/ae8bb69e235b6e46b24bc966aeaa80f45030c6f9/src/replays/file_system_replay_provider/load_file.ts#L91-L101 // ty vince @@ -563,6 +554,11 @@ const PLAYED_ON = Buffer.from([ 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x6f, 0x6e, 0x74, 0x7d, 0x7d, ]); +type ReplayBuffer = { + buffer: Buffer; + fileName: string; +}; + export async function writeReplays( dir: string, host: CopyHost, @@ -585,65 +581,57 @@ export async function writeReplays( ); } + const replayBuffers: ReplayBuffer[] = []; const isRemote = host.address && host.name; + const actualOutput = isRemote ? Output.ZIP : output; const sanitizedFileNames = fileNames.map((fileName) => sanitize(fileName)); const sanitizedSubdir = sanitize(subdir); - const writeDir = - output === Output.ZIP - ? join(app.getPath('temp'), sanitizedSubdir) - : join(dir, sanitizedSubdir); - if (!sanitizedSubdir && (output === Output.FOLDER || output === Output.ZIP)) { + const writeDir = join(dir, sanitizedSubdir); + if ( + !sanitizedSubdir && + (actualOutput === Output.FOLDER || actualOutput === Output.ZIP) + ) { throw new Error('subdir'); } - if (isRemote) { - await initSubdir(sanitizedSubdir, output); - } if (sanitizedSubdir) { - if (!isRemote) { - if (output === Output.ZIP) { - const tempDir = app.getPath('temp'); - await access(tempDir).catch(() => mkdir(tempDir)); - } + if (output === Output.FOLDER) { await mkdir(writeDir); } if (context) { - if (isRemote) { - await writeContext(sanitizedSubdir, context); + if (actualOutput === Output.ZIP) { + replayBuffers.push({ + buffer: Buffer.from(JSON.stringify(context)), + fileName: 'context.json', + }); } else { - const contextFilePath = join(writeDir, 'context.json'); - const contextFile = await open(contextFilePath, 'w'); - await contextFile.writeFile(JSON.stringify(context)); - await contextFile.close(); + await writeFile( + join(writeDir, 'context.json'), + JSON.stringify(context), + ); } } } const replayFilePromises = replays.map(async (replay, i) => { - const readFile = await open(replay.filePath); + const readReplay = await open(replay.filePath); - let fd = 0; - let writeFile: FileHandle | null = null; - if (isRemote) { - fd = await openReplay( - sanitizedSubdir, - sanitizedFileNames[i] ?? replay.fileName, - ); - } else { - const writeFileName = sanitizedFileNames[i] ?? replay.fileName; - const writeFilePath = join(writeDir, writeFileName); - writeFile = await open(writeFilePath, 'w'); - } + let replayBuffer = Buffer.from([]); + const fileName = sanitizedFileNames[i] ?? replay.fileName; + const replayFile = + actualOutput !== Output.ZIP + ? await open(join(writeDir, fileName), 'w') + : null; try { // raw element const rawHeader = Buffer.alloc(15); - const rawHeaderRes = await readFile.read(rawHeader, 0, 15, 0); + const rawHeaderRes = await readReplay.read(rawHeader, 0, 15, 0); if (rawHeaderRes.bytesRead !== 15) { throw new Error('raw element header read'); } - const fileSize = (await readFile.stat()).size; + const fileSize = (await readReplay.stat()).size; const rawElementLength = rawHeader.subarray(11, 15).readUInt32BE(); const rawElementReadLength = rawElementLength > 0 ? rawElementLength : fileSize - 15; @@ -653,15 +641,17 @@ export async function writeReplays( actualRawElementLength.copy(rawHeader, 11); } - let bytesWritten = isRemote - ? await writeReplayData(fd, rawHeader) - : (await writeFile!.write(rawHeader)).bytesWritten; - if (bytesWritten !== 15) { - throw new Error('raw element header write'); + if (actualOutput === Output.ZIP) { + replayBuffer = Buffer.concat([replayBuffer, rawHeader]); + } else { + const { bytesWritten } = await replayFile!.write(rawHeader); + if (bytesWritten !== 15) { + throw new Error('raw element header write'); + } } const rawElement = Buffer.alloc(rawElementReadLength); - const rawElementRes = await readFile.read( + const rawElementRes = await readReplay.read( rawElement, 0, rawElementReadLength, @@ -703,11 +693,13 @@ export async function writeReplays( Buffer.from(fixedDisplayNameArr).copy(rawElement, offset); }); } - bytesWritten = isRemote - ? await writeReplayData(fd, rawElement) - : (await writeFile!.write(rawElement)).bytesWritten; - if (bytesWritten !== rawElementReadLength) { - throw new Error('raw element write'); + if (actualOutput === Output.ZIP) { + replayBuffer = Buffer.concat([replayBuffer, rawElement]); + } else { + const { bytesWritten } = await replayFile!.write(rawElement); + if (bytesWritten !== rawElementReadLength) { + throw new Error('raw element write'); + } } // metadata @@ -715,7 +707,7 @@ export async function writeReplays( const metadataOffset = rawElementLength + 15; const metadataLength = fileSize - metadataOffset; const metadata = Buffer.alloc(metadataLength); - const metadataRes = await readFile.read( + const metadataRes = await readReplay.read( metadata, 0, metadataLength, @@ -741,16 +733,18 @@ export async function writeReplays( startAtLengthOffset + newStartAtLength + 1, startAtLengthOffset + startAtLength + 1, ); - bytesWritten = isRemote - ? await writeReplayData(fd, newMetadata) - : (await writeFile!.write(newMetadata)).bytesWritten; - if (bytesWritten !== newMetadata.length) { - throw new Error('metadata write (with start time override)'); + if (actualOutput === Output.ZIP) { + replayBuffer = Buffer.concat([replayBuffer, newMetadata]); + } else { + const { bytesWritten } = await replayFile!.write(newMetadata); + if (bytesWritten !== newMetadata.length) { + throw new Error('metadata write (with start time override)'); + } } + } else if (actualOutput === Output.ZIP) { + replayBuffer = Buffer.concat([replayBuffer, metadata]); } else { - bytesWritten = isRemote - ? await writeReplayData(fd, metadata) - : (await writeFile!.write(metadata)).bytesWritten; + const { bytesWritten } = await replayFile!.write(metadata); if (bytesWritten !== metadataLength) { throw new Error('metadata write'); } @@ -778,47 +772,53 @@ export async function writeReplays( bufs.push(PLAYED_ON); const bufsLength = bufs.reduce((acc, buf) => acc + buf.length, 0); const metadata = Buffer.concat(bufs, bufsLength); - bytesWritten = isRemote - ? await writeReplayData(fd, metadata) - : (await writeFile!.write(metadata)).bytesWritten; - if (bytesWritten !== metadata.length) { - throw new Error( - startTimes.length - ? 'metadata creation (with start time override)' - : 'metadata creation', - ); + if (actualOutput === Output.ZIP) { + replayBuffer = Buffer.concat([replayBuffer, metadata]); + } else { + const { bytesWritten } = await replayFile!.write(metadata); + if (bytesWritten !== metadata.length) { + throw new Error( + startTimes.length + ? 'metadata creation (with start time override)' + : 'metadata creation', + ); + } } } - } finally { - const promises = [readFile.close()]; - if (writeFile) { - promises.push(writeFile.close()); + if (actualOutput === Output.ZIP) { + replayBuffers.push({ buffer: replayBuffer, fileName }); } - if (fd) { - promises.push(closeReplay(fd)); + } finally { + const promises = [readReplay.close()]; + if (replayFile) { + promises.push(replayFile.close()); } await Promise.all(promises); } }); await Promise.all(replayFilePromises); - if (output === Output.ZIP && sanitizedSubdir) { + if (actualOutput === Output.ZIP) { + const zipFile = new ZipFile(); + replayBuffers.forEach(({ buffer, fileName }) => { + zipFile.addBuffer(buffer, fileName); + }); + zipFile.end(); + const zipBuffer = await bufferConsumer(zipFile.outputStream); + const promises = []; if (isRemote) { - await zipSubdir(sanitizedSubdir); - } else { - const tempDirFileNames = await readdir(writeDir); - const zipFile = new ZipFile(); - const zipFilePromise = new Promise((resolve) => { - zipFile.outputStream - .pipe(createWriteStream(`${join(dir, sanitizedSubdir)}.zip`)) - .on('close', resolve); - }); - tempDirFileNames.forEach((fileName) => { - zipFile.addFile(join(writeDir, fileName), fileName); - }); - zipFile.end(); - await zipFilePromise; - await rm(writeDir, { recursive: true }); + promises.push(writeZip(sanitizedSubdir, zipBuffer)); + } + if (dir) { + promises.push(writeFile(`${writeDir}.zip`, zipBuffer)); + } + const rejections = (await Promise.allSettled(promises)).filter( + (result) => result.status === 'rejected', + ); + if (rejections.length > 0) { + throw new Error( + rejections.map((rejection) => rejection.reason).join(', '), + ); } } } diff --git a/src/renderer/CopyControls.tsx b/src/renderer/CopyControls.tsx index 5d90fb3..3ffddc8 100644 --- a/src/renderer/CopyControls.tsx +++ b/src/renderer/CopyControls.tsx @@ -466,6 +466,7 @@ export default function CopyControls({ { const newCopySettings = { ...copySettings }; @@ -474,7 +475,9 @@ export default function CopyControls({ }} select size="small" - value={copySettings.output} + value={ + host.address && host.name ? Output.ZIP : copySettings.output + } > Separate Files Make Subfolder