Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Test PR] SDK Multipart support #93

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"name": "node-appwrite",
"name": "node-appwrite-1",
"homepage": "https://appwrite.io/support",
"description": "Appwrite is an open-source self-hosted backend server that abstract and simplify complex and repetitive development tasks behind a very simple REST API",
"version": "13.0.0",
Expand Down Expand Up @@ -42,12 +42,13 @@
},
"devDependencies": {
"@types/node": "20.11.25",
"tsup": "7.2.0",
"esbuild-plugin-file-path-extensions": "^2.0.0",
"tslib": "2.6.2",
"tsup": "7.2.0",
"typescript": "5.4.2"
},
"dependencies": {
"node-fetch-native-with-agent": "1.7.2"
"node-fetch-native-with-agent": "1.7.2",
Meldiron marked this conversation as resolved.
Show resolved Hide resolved
"parse-multipart-data": "^1.5.0"
}
}
156 changes: 156 additions & 0 deletions src/BodyMultipart.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
export type Part = {
contentDispositionHeader: string;
contentTypeHeader: string;
part: number[];
};

type Input = {
filename?: string;
name?: string;
type: string;
data: Buffer;
};

enum ParsingState {
INIT,
READING_HEADERS,
READING_DATA,
READING_PART_SEPARATOR,
}

export function parse(multipartBodyBuffer: Buffer, boundary: string): Input[] {
let lastline = "";
let contentDispositionHeader = "";
let contentTypeHeader = "";
let state: ParsingState = ParsingState.INIT;
let buffer: number[] = [];
const allParts: Input[] = [];

let currentPartHeaders: string[] = [];

for (let i = 0; i < multipartBodyBuffer.length; i++) {
const oneByte: number = multipartBodyBuffer[i];
const prevByte: number | null = i > 0 ? multipartBodyBuffer[i - 1] : null;
// 0x0a => \n
// 0x0d => \r
const newLineDetected: boolean = oneByte === 0x0a && prevByte === 0x0d;
const newLineChar: boolean = oneByte === 0x0a || oneByte === 0x0d;

if (!newLineChar) lastline += String.fromCharCode(oneByte);
if (ParsingState.INIT === state && newLineDetected) {
// searching for boundary
if ("--" + boundary === lastline) {
state = ParsingState.READING_HEADERS; // found boundary. start reading headers
}
lastline = "";
} else if (ParsingState.READING_HEADERS === state && newLineDetected) {
// parsing headers. Headers are separated by an empty line from the content. Stop reading headers when the line is empty
if (lastline.length) {
currentPartHeaders.push(lastline);
} else {
// found empty line. search for the headers we want and set the values
for (const h of currentPartHeaders) {
if (h.toLowerCase().startsWith("content-disposition:")) {
contentDispositionHeader = h;
} else if (h.toLowerCase().startsWith("content-type:")) {
contentTypeHeader = h;
}
}
state = ParsingState.READING_DATA;
buffer = [];
}
lastline = "";
} else if (ParsingState.READING_DATA === state) {
// parsing data
if (lastline.length > boundary.length + 4) {
lastline = ""; // mem save
}
if ("--" + boundary === lastline) {
const j = buffer.length - lastline.length;
const part = buffer.slice(0, j - 1);

allParts.push(
process({ contentDispositionHeader, contentTypeHeader, part })
);
buffer = [];
currentPartHeaders = [];
lastline = "";
state = ParsingState.READING_PART_SEPARATOR;
contentDispositionHeader = "";
contentTypeHeader = "";
} else {
buffer.push(oneByte);
}
if (newLineDetected) {
lastline = "";
}
} else if (ParsingState.READING_PART_SEPARATOR === state) {
if (newLineDetected) {
state = ParsingState.READING_HEADERS;
}
}
}
return allParts;
}

export function getBoundary(str: string): string {
const lines = str.replaceAll("\r\n", "\n").split("\n").reverse();
for (const line of lines) {
if (line !== "") {
return line.slice(0, -2).slice(2);
}
}
return "";
}

function process(part: Part): Input {
// will transform this object:
// { header: 'Content-Disposition: form-data; name="uploads[]"; filename="A.txt"',
// info: 'Content-Type: text/plain',
// part: 'AAAABBBB' }
// into this one:
// { filename: 'A.txt', type: 'text/plain', data: <Buffer 41 41 41 41 42 42 42 42> }
const obj = function (str: string) {
const k = str.split("=");
const a = k[0].trim();

const b = JSON.parse(k[1].trim());
const o = {};
Object.defineProperty(o, a, {
value: b,
writable: true,
enumerable: true,
configurable: true,
});
return o;
};
const header = part.contentDispositionHeader.split(";");

const filenameData = header[2];
let input = {};
if (filenameData) {
input = obj(filenameData);
const contentType = part.contentTypeHeader.split(":")[1].trim();
Object.defineProperty(input, "type", {
value: contentType,
writable: true,
enumerable: true,
configurable: true,
});
}
// always process the name field
Object.defineProperty(input, "name", {
value: header[1].split("=")[1].replace(/"/g, ""),
writable: true,
enumerable: true,
configurable: true,
});

Object.defineProperty(input, "data", {
value: Buffer.from(part.part),
writable: true,
enumerable: true,
configurable: true,
});
return input as Input;
}
50 changes: 50 additions & 0 deletions src/NewPayload.ts
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dont see type change.. like ExecutionModel.. How do they know about this new Payload class being used for responseBody?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have added Payload response model in appwrite. Do we need to make any other change in SDK to use it?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { realpathSync, readFileSync } from "fs";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we import fs this can't be used from Edge runtimes, such as Vercel/Cloudflare or Deno

export class NewPayload {
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
private data: Buffer;

constructor(data: Buffer) {
this.data = data;
}

public getData(): Buffer {
return this.data;
}

public static fromBuffer(buffer: Buffer): Buffer {
return Buffer.from(buffer);
}
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved

public toBuffer(): Buffer {
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
return this.data;
}

public static fromString(string: string): Buffer {
return Buffer.from(string);
}

public toString(encoding: BufferEncoding = "utf8"): string {
return this.data.toString(encoding);
}

public static fromJson(json: object): Buffer {
return Buffer.from(JSON.stringify(json));
}

public toJson(): object {
return JSON.parse(this.data.toString());
}

public static fromPath(path: string): Buffer {
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
const realPath = realpathSync(path);
const contents = readFileSync(realPath);
return Buffer.from(contents);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Manually QA fromFile and fromPath. Upload a picture to Storage service. Then see the picture in Appwrite Console. Does it know it's PNG? Can you preview it? If not, we need to fix it.

}

public toFile(fileName: string): File {
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
if (!fileName) {
fileName = "code.tar.gz";
}
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
const blob = new Blob([this.data]);
return new File([blob], fileName);
vermakhushboo marked this conversation as resolved.
Show resolved Hide resolved
}
}
26 changes: 25 additions & 1 deletion src/client.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { fetch, FormData, File } from 'node-fetch-native-with-agent';
import { createAgent } from 'node-fetch-native-with-agent/agent';
import { Models } from './models';
import { NewPayload } from './NewPayload';
import * as multipart from 'parse-multipart-data';
const { buffer } = require('node:stream/consumers');

type Payload = {
[key: string]: any;
Expand Down Expand Up @@ -331,7 +334,28 @@ class Client {
data = await response.json();
} else if (responseType === 'arrayBuffer') {
data = await response.arrayBuffer();
} else {
} else if (response.headers.get('content-type')?.includes('multipart/form-data')) {
const body = await buffer(response.body);
const boundary = multipart.getBoundary(
response.headers.get("content-type") || ""
);
const parts = multipart.parse(body, boundary);
const partsObject: { [key: string]: Buffer | string } = {};
for (const part of parts) {
if (part.name) {
if (part.name === "responseBody") {
partsObject[part.name] = part.data;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
partsObject[part.name] = part.data;
partsObject[part.name] = new Payload(part.data)

I think

} else {
partsObject[part.name] = part.data.toString();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if it's array or integer or boolean or object. Address when moving to sdk-generator

}
}
}
data = {
...partsObject,
responseBody: new NewPayload(partsObject.responseBody as Buffer),
};
}
else {
data = {
message: await response.text()
};
Expand Down
2 changes: 2 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,5 @@ export { ImageGravity } from './enums/image-gravity';
export { ImageFormat } from './enums/image-format';
export { PasswordHash } from './enums/password-hash';
export { MessagingProviderType } from './enums/messaging-provider-type';
export { NewPayload } from './NewPayload';
export { InputFile } from './inputFile';
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should remove all mention of InputFile. It should no longer exist.

With that, we also need to update existing endpoints that used to need it. they all should now use payload class

10 changes: 6 additions & 4 deletions src/services/functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { AppwriteException, Client, type Payload, UploadProgress } from '../clie
import type { Models } from '../models';
import { Runtime } from '../enums/runtime';
import { ExecutionMethod } from '../enums/execution-method';
import { NewPayload } from '../NewPayload';

export class Functions {
client: Client;
Expand Down Expand Up @@ -624,22 +625,22 @@ Use the &quot;command&quot; param to set the entrypoint used to execute your cod
* Trigger a function execution. The returned object will return you the current execution status. You can ping the `Get Execution` endpoint to get updates on the current execution status. Once this endpoint is called, your function execution process will start asynchronously.
*
* @param {string} functionId
* @param {string} body
* @param {NewPayload} body
* @param {boolean} async
* @param {string} xpath
* @param {ExecutionMethod} method
* @param {object} headers
* @throws {AppwriteException}
* @returns {Promise<Models.Execution>}
*/
async createExecution(functionId: string, body?: string, async?: boolean, xpath?: string, method?: ExecutionMethod, headers?: object): Promise<Models.Execution> {
async createExecution(functionId: string, body?: NewPayload, async?: boolean, xpath?: string, method?: ExecutionMethod, headers?: object): Promise<Models.Execution> {
if (typeof functionId === 'undefined') {
throw new AppwriteException('Missing required parameter: "functionId"');
}
const apiPath = '/functions/{functionId}/executions'.replace('{functionId}', functionId);
const payload: Payload = {};
if (typeof body !== 'undefined') {
payload['body'] = body;
payload['body'] = body ? body.getData() : body;
}
if (typeof async !== 'undefined') {
payload['async'] = async;
Expand All @@ -656,7 +657,8 @@ Use the &quot;command&quot; param to set the entrypoint used to execute your cod
const uri = new URL(this.client.config.endpoint + apiPath);

const apiHeaders: { [header: string]: string } = {
'content-type': 'application/json',
'content-type': 'multipart/form-data',
'accept': 'multipart/form-data',
}

return await this.client.call(
Expand Down