integrate yabson

gildas-lormeau · Mar 21, 2022 · 766506a · 766506a
1 parent 0226408
commit 766506a
Show file tree

Hide file tree

Showing 8 changed files with 726 additions and 56 deletions.
diff --git a/rollup.config.dev.js b/rollup.config.dev.js
@@ -80,6 +80,14 @@ export default [{
 		format: "iife",
 		plugins: []
 	}]
+}, {
+	input: ["src/extension/lib/yabson/yabson.js"],
+	output: [{
+		file: "lib/yabson.js",
+		format: "umd",
+		name: "yabson",
+		plugins: []
+	}]
 }, {
 	input: ["src/extension/lib/single-file/browser-polyfill/chrome-browser-polyfill.js"],
 	output: [{

diff --git a/rollup.config.js b/rollup.config.js
@@ -82,6 +82,14 @@ export default [{
 		format: "iife",
 		plugins: [terser()]
 	}]
+}, {
+	input: ["src/extension/lib/yabson/yabson.js"],
+	output: [{
+		file: "lib/yabson.js",
+		format: "umd",
+		name: "yabson",
+		plugins: [terser()]
+	}]
 }, {
 	input: ["src/extension/lib/single-file/browser-polyfill/chrome-browser-polyfill.js"],
 	output: [{

diff --git a/src/extension/core/bg/background.html b/src/extension/core/bg/background.html
@@ -9,6 +9,7 @@
 <body>
 	<script src="/lib/chrome-browser-polyfill.js"></script>
 	<script src="/lib/extension-background.js"></script>
+	<script src="/lib/yabson.js"></script>
 	<script src="/lib/single-file.js"></script>
 	<script src="/lib/infobar.js"></script>	
 </body>

diff --git a/src/extension/core/bg/business.js b/src/extension/core/bg/business.js
@@ -38,6 +38,7 @@ const TASK_PENDING_STATE = "pending";
 const TASK_PROCESSING_STATE = "processing";
 
 const extensionScriptFiles = [
+	"lib/yabson.js",
 	"lib/infobar.js",
 	"lib/extension.js"
 ];

diff --git a/src/extension/core/bg/downloads.js b/src/extension/core/bg/downloads.js
@@ -21,7 +21,7 @@
  *   Source.
  */
 
-/* global browser, singlefile, URL, Response */
+/* global browser, singlefile, URL, yabson, Response */
 
 import * as config from "./config.js";
 import * as bookmarks from "./bookmarks.js";
@@ -32,7 +32,7 @@ import { GDrive } from "./../../lib/gdrive/gdrive.js";
 import { pushGitHub } from "./../../lib/github/github.js";
 import { download } from "./download-util.js";
 
-const partialContents = new Map();
+const parsers = new Map();
 const MAX_CONTENT_SIZE = 32 * (1024 * 1024);
 const GDRIVE_CLIENT_ID = "207618107333-h1220p1oasj3050kr5r416661adm091a.apps.googleusercontent.com";
 const GDRIVE_CLIENT_KEY = "VQJ8Gq8Vxx72QyxPyeLtWvUt";
@@ -85,22 +85,16 @@ async function onMessage(message, sender) {
 
 async function downloadTabPage(message, tab) {
 	const tabId = tab.id;
-	let contents;
-	if (message.truncated) {
-		contents = partialContents.get(tabId);
-		if (!contents) {
-			contents = [];
-			partialContents.set(tabId, contents);
-		}
-		contents.push(message.content);
-		if (message.finished) {
-			partialContents.delete(tabId);
-		}
-	} else if (message.content) {
-		contents = [message.content];
+	let parser = parsers.get(tabId);
+	if (!parser) {
+		parser = yabson.getParser();
+		parsers.set(tabId, parser);
 	}
-	if (!message.truncated || message.finished) {
+	let result = parser.next(new Uint8Array(message.content));
+	if (result.done) {
 		let skipped;
+		message = result.value;
+		parsers.delete(tabId);
 		if (message.backgroundSave && !message.saveToGDrive) {
 			const testSkip = await testSkipSave(message.filename, message);
 			message.filenameConflictAction = testSkip.filenameConflictAction;
@@ -109,7 +103,7 @@ async function downloadTabPage(message, tab) {
 		if (skipped) {
 			ui.onEnd(tabId);
 		} else {
-			const pageData = JSON.parse(singlefile.helper.flatten(contents).join(""));
+			const pageData = message.pageData;
 			const blob = await singlefile.processors.compression.process(pageData, {
 				insertTextBody: message.insertTextBody,
 				url: tab.url,

diff --git a/src/extension/core/common/download.js b/src/extension/core/common/download.js
@@ -21,9 +21,7 @@
  *   Source.
  */
 
-/* global browser, infobar */
-
-const MAX_CONTENT_SIZE = 8 * (1024 * 1024);
+/* global browser, infobar, yabson */
 
 export {
 	downloadPage
@@ -36,42 +34,38 @@ async function downloadPage(pageData, options) {
 	if (options.includeBOM) {
 		pageData.content = "\ufeff" + pageData.content;
 	}
-	const content = JSON.stringify(pageData);
-	for (let blockIndex = 0; blockIndex * MAX_CONTENT_SIZE < content.length; blockIndex++) {
-		const message = {
+	const message = {
+		taskId: options.taskId,
+		insertTextBody: options.insertTextBody,
+		confirmFilename: options.confirmFilename,
+		filenameConflictAction: options.filenameConflictAction,
+		filename: pageData.filename,
+		saveToGDrive: options.saveToGDrive,
+		saveToGitHub: options.saveToGitHub,
+		githubToken: options.githubToken,
+		githubUser: options.githubUser,
+		githubRepository: options.githubRepository,
+		githubBranch: options.githubBranch,
+		forceWebAuthFlow: options.forceWebAuthFlow,
+		filenameReplacementCharacter: options.filenameReplacementCharacter,
+		compressHTML: options.compressHTML,
+		backgroundSave: options.backgroundSave,
+		bookmarkId: options.bookmarkId,
+		replaceBookmarkURL: options.replaceBookmarkURL,
+		includeInfobar: options.includeInfobar,
+		createRootDirectory: options.createRootDirectory,
+		selfExtractingArchive: options.selfExtractingArchive,
+		insertCanonicalLink: options.insertCanonicalLink,
+		insertMetaNoIndex: options.insertMetaNoIndex,
+		password: options.password,
+		pageData: pageData
+	};
+	const serializer = yabson.getSerializer(message);
+	for (const content of serializer) {
+		await browser.runtime.sendMessage({
 			method: "downloads.download",
-			taskId: options.taskId,
-			insertTextBody: options.insertTextBody,
-			confirmFilename: options.confirmFilename,
-			filenameConflictAction: options.filenameConflictAction,
-			filename: pageData.filename,
-			saveToGDrive: options.saveToGDrive,
-			saveToGitHub: options.saveToGitHub,
-			githubToken: options.githubToken,
-			githubUser: options.githubUser,
-			githubRepository: options.githubRepository,
-			githubBranch: options.githubBranch,
-			forceWebAuthFlow: options.forceWebAuthFlow,
-			filenameReplacementCharacter: options.filenameReplacementCharacter,
-			compressHTML: options.compressHTML,
-			backgroundSave: options.backgroundSave,
-			bookmarkId: options.bookmarkId,
-			replaceBookmarkURL: options.replaceBookmarkURL,
-			includeInfobar: options.includeInfobar,
-			createRootDirectory: options.createRootDirectory,
-			selfExtractingArchive: options.selfExtractingArchive,
-			insertCanonicalLink: options.insertCanonicalLink,
-			insertMetaNoIndex: options.insertMetaNoIndex,
-			password: options.password
-		};
-		message.truncated = content.length > MAX_CONTENT_SIZE;
-		if (message.truncated) {
-			message.finished = (blockIndex + 1) * MAX_CONTENT_SIZE > content.length;
-			message.content = content.slice(blockIndex * MAX_CONTENT_SIZE, (blockIndex + 1) * MAX_CONTENT_SIZE);
-		} else {
-			message.content = content;
-		}
-		await browser.runtime.sendMessage(message);
+			content: Array.from(content)
+		});
 	}
 	if (options.backgroundSave) {
 		await browser.runtime.sendMessage({ method: "downloads.end", taskId: options.taskId });