From 1c5554afc8b9668f1f96fadf1d4b0eae29860d1b Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 14 Aug 2024 02:45:40 +0530 Subject: [PATCH 1/4] SDK Multipart support --- package.json | 7 +++-- src/NewPayload.ts | 60 +++++++++++++++++++++++++++++++++++++++ src/client.ts | 30 +++++++++++++++++++- src/index.ts | 2 ++ src/services/functions.ts | 10 ++++--- tsconfig.json | 2 +- 6 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 src/NewPayload.ts diff --git a/package.json b/package.json index f550d89..6ddd05d 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "node-appwrite", + "name": "node-appwrite-1", "homepage": "https://appwrite.io/support", "description": "Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API", "version": "13.0.0", @@ -42,12 +42,13 @@ }, "devDependencies": { "@types/node": "20.11.25", - "tsup": "7.2.0", "esbuild-plugin-file-path-extensions": "^2.0.0", "tslib": "2.6.2", + "tsup": "7.2.0", "typescript": "5.4.2" }, "dependencies": { - "node-fetch-native-with-agent": "1.7.2" + "node-fetch-native-with-agent": "1.7.2", + "parse-multipart-data": "^1.5.0" } } diff --git a/src/NewPayload.ts b/src/NewPayload.ts new file mode 100644 index 0000000..e3c4774 --- /dev/null +++ b/src/NewPayload.ts @@ -0,0 +1,60 @@ +import { realpathSync, readFileSync } from "fs"; +export class NewPayload { + private data: any; + + constructor(data: any) { + this.data = data; + } + + // converts JSON to binary data (Buffer) + static async fromJson(json: object): Promise { + const jsonString = JSON.stringify(json); + const encoder = new TextEncoder(); + const buffer = encoder.encode(jsonString); + return new NewPayload(buffer); + } + + // converts file to binary data (Buffer) + static fromPath(path: string): NewPayload { + const realPath = realpathSync(path); + const contents = readFileSync(realPath); + return new NewPayload(new Uint8Array(contents)); + } + + // converts text to binary data (Buffer) + static async fromPlainText(text: string): Promise { + const arrayBytes = new TextEncoder().encode(text); + return new NewPayload(arrayBytes); + } + + async fromBinary(): Promise { + return new NewPayload(this.data); + } + + // converts binary data (Buffer) to JSON + async toJson(): Promise { + const decoder = new TextDecoder(); + const jsonString = decoder.decode(this.data); + return JSON.parse(jsonString); + } + + // convert binary data (Buffer) to file + async toFile(fileName: string): Promise { + const blob = new Blob([this.data]); + return new File([blob], fileName); + } + + // converts binary data (Buffer) to text + async toPlainText(): Promise { + const decoder = new TextDecoder(); + return decoder.decode(this.data); + } + + toBinary(): Uint8Array { + return Uint8Array.from(this.data); + } + + public getData(): any { + return this.data; + } +} diff --git a/src/client.ts b/src/client.ts index 770edb4..98cf9af 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,6 +1,9 @@ import { fetch, FormData, File } from 'node-fetch-native-with-agent'; import { createAgent } from 'node-fetch-native-with-agent/agent'; import { Models } from './models'; +import { NewPayload } from './NewPayload'; +import * as multipart from 'parse-multipart-data'; +const { buffer } = require('node:stream/consumers'); type Payload = { [key: string]: any; @@ -32,6 +35,16 @@ class AppwriteException extends Error { } } +function getBoundary(str: string): string { + const lines = str.replaceAll("\r\n", "\n").split("\n").reverse(); + for (const line of lines) { + if (line !== "") { + return line.slice(0, -2).slice(2); + } + } + return ""; + } + function getUserAgent() { let ua = 'AppwriteNodeJSSDK/13.0.0'; @@ -331,7 +344,22 @@ class Client { data = await response.json(); } else if (responseType === 'arrayBuffer') { data = await response.arrayBuffer(); - } else { + } else if (response.headers.get('content-type')?.includes('multipart/form-data')) { + const body = await buffer(response.body); + const boundary = getBoundary(body.toString()); + const parts = multipart.parse(body, boundary); + const partsObject = parts.reduce<{ [key: string]: Buffer }>((acc, part) => { + if (part.name) { + acc[part.name] = part.data; + } + return acc; + }, {}); + data = { + ...partsObject, + responseBody: new NewPayload(partsObject.responseBody) + } + } + else { data = { message: await response.text() }; diff --git a/src/index.ts b/src/index.ts index 38ceb0d..cdc2c96 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,3 +33,5 @@ export { ImageGravity } from './enums/image-gravity'; export { ImageFormat } from './enums/image-format'; export { PasswordHash } from './enums/password-hash'; export { MessagingProviderType } from './enums/messaging-provider-type'; +export { NewPayload } from './NewPayload'; +export { InputFile } from './inputFile'; diff --git a/src/services/functions.ts b/src/services/functions.ts index afe10e3..6ccb0cb 100644 --- a/src/services/functions.ts +++ b/src/services/functions.ts @@ -2,6 +2,7 @@ import { AppwriteException, Client, type Payload, UploadProgress } from '../clie import type { Models } from '../models'; import { Runtime } from '../enums/runtime'; import { ExecutionMethod } from '../enums/execution-method'; +import { NewPayload } from '../NewPayload'; export class Functions { client: Client; @@ -624,7 +625,7 @@ Use the "command" param to set the entrypoint used to execute your cod * Trigger a function execution. The returned object will return you the current execution status. You can ping the `Get Execution` endpoint to get updates on the current execution status. Once this endpoint is called, your function execution process will start asynchronously. * * @param {string} functionId - * @param {string} body + * @param {NewPayload} body * @param {boolean} async * @param {string} xpath * @param {ExecutionMethod} method @@ -632,14 +633,14 @@ Use the "command" param to set the entrypoint used to execute your cod * @throws {AppwriteException} * @returns {Promise} */ - async createExecution(functionId: string, body?: string, async?: boolean, xpath?: string, method?: ExecutionMethod, headers?: object): Promise { + async createExecution(functionId: string, body?: NewPayload, async?: boolean, xpath?: string, method?: ExecutionMethod, headers?: object): Promise { if (typeof functionId === 'undefined') { throw new AppwriteException('Missing required parameter: "functionId"'); } const apiPath = '/functions/{functionId}/executions'.replace('{functionId}', functionId); const payload: Payload = {}; if (typeof body !== 'undefined') { - payload['body'] = body; + payload['body'] = body ? body.getData : body; } if (typeof async !== 'undefined') { payload['async'] = async; @@ -656,7 +657,8 @@ Use the "command" param to set the entrypoint used to execute your cod const uri = new URL(this.client.config.endpoint + apiPath); const apiHeaders: { [header: string]: string } = { - 'content-type': 'application/json', + 'content-type': 'multipart/form-data', + 'accept': 'multipart/form-data', } return await this.client.call( diff --git a/tsconfig.json b/tsconfig.json index 45afe7d..a6ce630 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,7 @@ "strict": true, "module": "es2022", "moduleResolution": "Bundler", - "lib": ["es2022"] + "lib": ["dom", "es2022"] }, "compileOnSave": false, "exclude": ["node_modules", "dist"], From c87a086049c3491267f90a9436207981df9303d5 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 14 Aug 2024 19:17:31 +0530 Subject: [PATCH 2/4] Generate boundary using library --- src/NewPayload.ts | 59 ++++++++++++++++++----------------------------- src/client.ts | 12 +--------- 2 files changed, 24 insertions(+), 47 deletions(-) diff --git a/src/NewPayload.ts b/src/NewPayload.ts index e3c4774..5fa444b 100644 --- a/src/NewPayload.ts +++ b/src/NewPayload.ts @@ -1,60 +1,47 @@ import { realpathSync, readFileSync } from "fs"; export class NewPayload { - private data: any; + private data: Buffer; - constructor(data: any) { + constructor(data: Buffer) { this.data = data; } - // converts JSON to binary data (Buffer) - static async fromJson(json: object): Promise { - const jsonString = JSON.stringify(json); - const encoder = new TextEncoder(); - const buffer = encoder.encode(jsonString); - return new NewPayload(buffer); + public getData(): Buffer { + return this.data; } - // converts file to binary data (Buffer) - static fromPath(path: string): NewPayload { - const realPath = realpathSync(path); - const contents = readFileSync(realPath); - return new NewPayload(new Uint8Array(contents)); + public static fromBuffer(buffer: Buffer): Buffer { + return Buffer.from(buffer); } - // converts text to binary data (Buffer) - static async fromPlainText(text: string): Promise { - const arrayBytes = new TextEncoder().encode(text); - return new NewPayload(arrayBytes); + public toBuffer(): Buffer { + return this.data; } - async fromBinary(): Promise { - return new NewPayload(this.data); + public static fromString(string: string): Buffer { + return Buffer.from(string); } - // converts binary data (Buffer) to JSON - async toJson(): Promise { - const decoder = new TextDecoder(); - const jsonString = decoder.decode(this.data); - return JSON.parse(jsonString); + public toString(encoding: BufferEncoding = "utf8"): string { + return this.data.toString(encoding); } - // convert binary data (Buffer) to file - async toFile(fileName: string): Promise { - const blob = new Blob([this.data]); - return new File([blob], fileName); + public static fromJSON(json: object): Buffer { + return Buffer.from(JSON.stringify(json)); } - // converts binary data (Buffer) to text - async toPlainText(): Promise { - const decoder = new TextDecoder(); - return decoder.decode(this.data); + public toJSON(): object { + return JSON.parse(this.data.toString()); } - toBinary(): Uint8Array { - return Uint8Array.from(this.data); + public static fromPath(path: string): Buffer { + const realPath = realpathSync(path); + const contents = readFileSync(realPath); + return Buffer.from(contents); } - public getData(): any { - return this.data; + public toFile(fileName: string): File { + const blob = new Blob([this.data]); + return new File([blob], fileName); } } diff --git a/src/client.ts b/src/client.ts index 98cf9af..5742e0b 100644 --- a/src/client.ts +++ b/src/client.ts @@ -35,16 +35,6 @@ class AppwriteException extends Error { } } -function getBoundary(str: string): string { - const lines = str.replaceAll("\r\n", "\n").split("\n").reverse(); - for (const line of lines) { - if (line !== "") { - return line.slice(0, -2).slice(2); - } - } - return ""; - } - function getUserAgent() { let ua = 'AppwriteNodeJSSDK/13.0.0'; @@ -346,7 +336,7 @@ class Client { data = await response.arrayBuffer(); } else if (response.headers.get('content-type')?.includes('multipart/form-data')) { const body = await buffer(response.body); - const boundary = getBoundary(body.toString()); + const boundary = multipart.getBoundary(response.headers.get('content-type') || ''); const parts = multipart.parse(body, boundary); const partsObject = parts.reduce<{ [key: string]: Buffer }>((acc, part) => { if (part.name) { From bba3b6cf15fc34dc496dcde0f6b00e2fa23831c5 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 15 Aug 2024 03:15:15 +0530 Subject: [PATCH 3/4] Address comments --- src/NewPayload.ts | 4 ++-- src/client.ts | 22 ++++++++++++---------- src/services/functions.ts | 2 +- tsconfig.json | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/NewPayload.ts b/src/NewPayload.ts index 5fa444b..73c56c7 100644 --- a/src/NewPayload.ts +++ b/src/NewPayload.ts @@ -26,11 +26,11 @@ export class NewPayload { return this.data.toString(encoding); } - public static fromJSON(json: object): Buffer { + public static fromJson(json: object): Buffer { return Buffer.from(JSON.stringify(json)); } - public toJSON(): object { + public toJson(): object { return JSON.parse(this.data.toString()); } diff --git a/src/client.ts b/src/client.ts index 5742e0b..ef66224 100644 --- a/src/client.ts +++ b/src/client.ts @@ -336,18 +336,20 @@ class Client { data = await response.arrayBuffer(); } else if (response.headers.get('content-type')?.includes('multipart/form-data')) { const body = await buffer(response.body); - const boundary = multipart.getBoundary(response.headers.get('content-type') || ''); + const boundary = multipart.getBoundary( + response.headers.get("content-type") || "" + ); const parts = multipart.parse(body, boundary); - const partsObject = parts.reduce<{ [key: string]: Buffer }>((acc, part) => { - if (part.name) { - acc[part.name] = part.data; - } - return acc; - }, {}); - data = { - ...partsObject, - responseBody: new NewPayload(partsObject.responseBody) + const partsObject: { [key: string]: Buffer } = {}; + for (const part of parts) { + if (part.name) { + partsObject[part.name] = part.data; + } } + data = { + ...partsObject, + responseBody: new NewPayload(partsObject.responseBody), + }; } else { data = { diff --git a/src/services/functions.ts b/src/services/functions.ts index 6ccb0cb..34fb834 100644 --- a/src/services/functions.ts +++ b/src/services/functions.ts @@ -640,7 +640,7 @@ Use the "command" param to set the entrypoint used to execute your cod const apiPath = '/functions/{functionId}/executions'.replace('{functionId}', functionId); const payload: Payload = {}; if (typeof body !== 'undefined') { - payload['body'] = body ? body.getData : body; + payload['body'] = body ? body.getData() : body; } if (typeof async !== 'undefined') { payload['async'] = async; diff --git a/tsconfig.json b/tsconfig.json index a6ce630..45afe7d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,7 @@ "strict": true, "module": "es2022", "moduleResolution": "Bundler", - "lib": ["dom", "es2022"] + "lib": ["es2022"] }, "compileOnSave": false, "exclude": ["node_modules", "dist"], From ab119d9cd5ce7672d8a40ff3028f1f7eb4240847 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:16:03 +0530 Subject: [PATCH 4/4] Only convert responsebody to binary --- src/BodyMultipart.ts | 156 +++++++++++++++++++++++++++++++++++++++++++ src/NewPayload.ts | 3 + src/client.ts | 10 ++- 3 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 src/BodyMultipart.ts diff --git a/src/BodyMultipart.ts b/src/BodyMultipart.ts new file mode 100644 index 0000000..7c0b4f7 --- /dev/null +++ b/src/BodyMultipart.ts @@ -0,0 +1,156 @@ +export type Part = { + contentDispositionHeader: string; + contentTypeHeader: string; + part: number[]; +}; + +type Input = { + filename?: string; + name?: string; + type: string; + data: Buffer; +}; + +enum ParsingState { + INIT, + READING_HEADERS, + READING_DATA, + READING_PART_SEPARATOR, +} + +export function parse(multipartBodyBuffer: Buffer, boundary: string): Input[] { + let lastline = ""; + let contentDispositionHeader = ""; + let contentTypeHeader = ""; + let state: ParsingState = ParsingState.INIT; + let buffer: number[] = []; + const allParts: Input[] = []; + + let currentPartHeaders: string[] = []; + + for (let i = 0; i < multipartBodyBuffer.length; i++) { + const oneByte: number = multipartBodyBuffer[i]; + const prevByte: number | null = i > 0 ? multipartBodyBuffer[i - 1] : null; + // 0x0a => \n + // 0x0d => \r + const newLineDetected: boolean = oneByte === 0x0a && prevByte === 0x0d; + const newLineChar: boolean = oneByte === 0x0a || oneByte === 0x0d; + + if (!newLineChar) lastline += String.fromCharCode(oneByte); + if (ParsingState.INIT === state && newLineDetected) { + // searching for boundary + if ("--" + boundary === lastline) { + state = ParsingState.READING_HEADERS; // found boundary. start reading headers + } + lastline = ""; + } else if (ParsingState.READING_HEADERS === state && newLineDetected) { + // parsing headers. Headers are separated by an empty line from the content. Stop reading headers when the line is empty + if (lastline.length) { + currentPartHeaders.push(lastline); + } else { + // found empty line. search for the headers we want and set the values + for (const h of currentPartHeaders) { + if (h.toLowerCase().startsWith("content-disposition:")) { + contentDispositionHeader = h; + } else if (h.toLowerCase().startsWith("content-type:")) { + contentTypeHeader = h; + } + } + state = ParsingState.READING_DATA; + buffer = []; + } + lastline = ""; + } else if (ParsingState.READING_DATA === state) { + // parsing data + if (lastline.length > boundary.length + 4) { + lastline = ""; // mem save + } + if ("--" + boundary === lastline) { + const j = buffer.length - lastline.length; + const part = buffer.slice(0, j - 1); + + allParts.push( + process({ contentDispositionHeader, contentTypeHeader, part }) + ); + buffer = []; + currentPartHeaders = []; + lastline = ""; + state = ParsingState.READING_PART_SEPARATOR; + contentDispositionHeader = ""; + contentTypeHeader = ""; + } else { + buffer.push(oneByte); + } + if (newLineDetected) { + lastline = ""; + } + } else if (ParsingState.READING_PART_SEPARATOR === state) { + if (newLineDetected) { + state = ParsingState.READING_HEADERS; + } + } + } + return allParts; +} + +export function getBoundary(str: string): string { + const lines = str.replaceAll("\r\n", "\n").split("\n").reverse(); + for (const line of lines) { + if (line !== "") { + return line.slice(0, -2).slice(2); + } + } + return ""; +} + +function process(part: Part): Input { + // will transform this object: + // { header: 'Content-Disposition: form-data; name="uploads[]"; filename="A.txt"', + // info: 'Content-Type: text/plain', + // part: 'AAAABBBB' } + // into this one: + // { filename: 'A.txt', type: 'text/plain', data: } + const obj = function (str: string) { + const k = str.split("="); + const a = k[0].trim(); + + const b = JSON.parse(k[1].trim()); + const o = {}; + Object.defineProperty(o, a, { + value: b, + writable: true, + enumerable: true, + configurable: true, + }); + return o; + }; + const header = part.contentDispositionHeader.split(";"); + + const filenameData = header[2]; + let input = {}; + if (filenameData) { + input = obj(filenameData); + const contentType = part.contentTypeHeader.split(":")[1].trim(); + Object.defineProperty(input, "type", { + value: contentType, + writable: true, + enumerable: true, + configurable: true, + }); + } + // always process the name field + Object.defineProperty(input, "name", { + value: header[1].split("=")[1].replace(/"/g, ""), + writable: true, + enumerable: true, + configurable: true, + }); + + Object.defineProperty(input, "data", { + value: Buffer.from(part.part), + writable: true, + enumerable: true, + configurable: true, + }); + return input as Input; +} diff --git a/src/NewPayload.ts b/src/NewPayload.ts index 73c56c7..a2e7c2a 100644 --- a/src/NewPayload.ts +++ b/src/NewPayload.ts @@ -41,6 +41,9 @@ export class NewPayload { } public toFile(fileName: string): File { + if (!fileName) { + fileName = "code.tar.gz"; + } const blob = new Blob([this.data]); return new File([blob], fileName); } diff --git a/src/client.ts b/src/client.ts index ef66224..c8db18c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -340,15 +340,19 @@ class Client { response.headers.get("content-type") || "" ); const parts = multipart.parse(body, boundary); - const partsObject: { [key: string]: Buffer } = {}; + const partsObject: { [key: string]: Buffer | string } = {}; for (const part of parts) { if (part.name) { - partsObject[part.name] = part.data; + if (part.name === "responseBody") { + partsObject[part.name] = part.data; + } else { + partsObject[part.name] = part.data.toString(); + } } } data = { ...partsObject, - responseBody: new NewPayload(partsObject.responseBody), + responseBody: new NewPayload(partsObject.responseBody as Buffer), }; } else {