diff --git a/lib/gitgitgadget.ts b/lib/gitgitgadget.ts index ea4d2ab397..8e1f27feed 100644 --- a/lib/gitgitgadget.ts +++ b/lib/gitgitgadget.ts @@ -168,7 +168,7 @@ export class GitGitGadget { const email = userInfo.email; const send = async (mail: string): Promise => { - const mbox = parseMBox(mail); + const mbox = await parseMBox(mail); mbox.cc = []; mbox.to = email; console.log(mbox); diff --git a/lib/mail-archive-helper.ts b/lib/mail-archive-helper.ts index 36d9087e90..d91deb17e6 100644 --- a/lib/mail-archive-helper.ts +++ b/lib/mail-archive-helper.ts @@ -1,5 +1,4 @@ import { createHash } from "crypto"; -import * as libqp from "libqp"; import { git, revParse } from "./git"; import { GitNotes } from "./git-notes"; import { IGitGitGadgetOptions } from "./gitgitgadget"; @@ -43,20 +42,7 @@ export class MailArchiveGitHelper { } public static mbox2markdown(mbox: IParsedMBox): string { - let body = mbox.body; - - const headers = mbox.headers || []; - for (const header of headers) { - if (header.key === "Content-Transfer-Encoding") { - const value = header.value.toLowerCase(); - if (value === "base64") { - body = Buffer.from(body, "base64").toString(); - } else if (value === "quoted-printable") { - const buffer = libqp.decode(body); - body = buffer.toString("utf-8"); - } - } - } + const body = mbox.body; if (!body.length) { return ""; @@ -156,7 +142,7 @@ export class MailArchiveGitHelper { }; const mboxHandler = async (mbox: string): Promise => { - const parsedMbox = parseMBox(mbox, true); + const parsedMbox = await parseMBox(mbox, true); if (!parsedMbox.headers) { throw new Error(`Could not parse ${mbox}`); } diff --git a/lib/send-mail.ts b/lib/send-mail.ts index 648e2baf27..6ffe94c1f7 100644 --- a/lib/send-mail.ts +++ b/lib/send-mail.ts @@ -1,3 +1,4 @@ +import { simpleParser, SimpleParserOptions } from "mailparser"; import { createTransport, SendMailOptions } from "nodemailer"; import SMTPTransport = require("nodemailer/lib/smtp-transport"); import { decode } from "rfc2047"; @@ -24,12 +25,7 @@ export interface ISMTPOptions { export async function parseHeadersAndSendMail(mbox: string, smtpOptions: ISMTPOptions): Promise { - return await sendMail(parseMBox(mbox), smtpOptions); -} - -function replaceAll(input: string, pattern: string, replacement: string): - string { - return input.split(pattern).join(replacement); + return await sendMail( await parseMBox(mbox), smtpOptions); } /** @@ -41,17 +37,7 @@ function replaceAll(input: string, pattern: string, replacement: string): * @param {string} mbox The mail, in mbox format * @returns {IParsedMBox} the parsed headers/body */ -export function parseMBox(mbox: string, gentle?: boolean): - IParsedMBox { - const headerEnd = mbox.indexOf("\n\n"); - if (headerEnd < 0) { - throw new Error("Could not parse mail"); - } - const headerStart = mbox.startsWith("From ") ? mbox.indexOf("\n") + 1 : 0; - - const header = mbox.substr(headerStart, headerEnd - headerStart); - const body = mbox.substr(headerEnd + 2); - +export async function parseMBox(mbox: string, gentle?: boolean): Promise { let cc: string[] | undefined; let date: string | undefined; let from: string | undefined; @@ -60,15 +46,29 @@ export function parseMBox(mbox: string, gentle?: boolean): let subject: string | undefined; let to: string | undefined; - for (const line of header.split(/\n(?![ \t])/)) { - const colon = line.indexOf(": "); - if (colon < 0) { - throw new Error(`Failed to parse header line '${line}`); + const options: SimpleParserOptions = { + skipHtmlToText: true, + skipTextLinks : true, + skipTextToHtml: true + }; + + const parsed = await simpleParser(mbox, options); + + for (const entry of parsed.headerLines) { + const valueSet = entry.line.match(/(.*): *([^]*)$/); + if (!valueSet) { + if (entry.line[entry.line.length - 1] === ":") { + continue; + } + throw new Error(`Failed to parse header line '${entry.line}'`); } - const key = line.substr(0, colon); - const value = replaceAll(line.substr(colon + 2), "\n ", " "); - switch (key.toLowerCase()) { - case "cc": cc = (cc || []).concat(value.split(", ")); break; + const key = valueSet[1]; + const value = valueSet[2]; + + switch (entry.key) { + case "cc": cc = (cc || []).concat(value.replace(/\r?\n/g, " ").split(", ").map(item => + item.trim() + )); break; case "date": date = value; break; case "fcc": break; case "from": from = decode(value.trim()); break; @@ -81,11 +81,11 @@ export function parseMBox(mbox: string, gentle?: boolean): } if (!gentle && (!to || !subject || !from)) { - throw new Error(`Missing To, Subject and/or From header:\n${header}`); + throw new Error(`Missing To, Subject and/or From header:\n${mbox}`); } return { - body, + body: parsed.text || "", cc, date, from, diff --git a/package-lock.json b/package-lock.json index 3a7952185e..04bfa04050 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "html-to-text": "^8.1.0", "json-stable-stringify": "^1.0.1", "jsonwebtoken": "^8.5.1", - "libqp": "^1.1.0", "marked": "^4.0.12", "nodemailer": "^6.7.2", "rfc2047": "^3.0.1" @@ -27,7 +26,6 @@ "@types/jest": "^27.4.0", "@types/json-stable-stringify": "^1.0.33", "@types/jsonwebtoken": "^8.5.8", - "@types/libqp": "^1.1.1", "@types/marked": "^4.0.2", "@types/nodemailer": "^6.4.4", "@types/rfc2047": "^2.0.1", @@ -1487,15 +1485,6 @@ "@types/node": "*" } }, - "node_modules/@types/libqp": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/libqp/-/libqp-1.1.1.tgz", - "integrity": "sha512-pdSohjXQ5SxvWAXa+yv1Jrqm3izxfFHDdIybs6vYdtLWIHUxE75ftnSnA8bZQgQE3oP1RFk4kHkclLg7yD9i2A==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/lru-cache": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-5.1.0.tgz", @@ -5308,7 +5297,8 @@ "node_modules/libqp": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz", - "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=" + "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=", + "dev": true }, "node_modules/lines-and-columns": { "version": "1.2.4", @@ -8716,15 +8706,6 @@ "@types/node": "*" } }, - "@types/libqp": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/libqp/-/libqp-1.1.1.tgz", - "integrity": "sha512-pdSohjXQ5SxvWAXa+yv1Jrqm3izxfFHDdIybs6vYdtLWIHUxE75ftnSnA8bZQgQE3oP1RFk4kHkclLg7yD9i2A==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/lru-cache": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-5.1.0.tgz", @@ -11561,7 +11542,8 @@ "libqp": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz", - "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=" + "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=", + "dev": true }, "lines-and-columns": { "version": "1.2.4", diff --git a/package.json b/package.json index ff50be0ba0..cd0312b8ff 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "@types/jest": "^27.4.0", "@types/json-stable-stringify": "^1.0.33", "@types/jsonwebtoken": "^8.5.8", - "@types/libqp": "^1.1.1", "@types/marked": "^4.0.2", "@types/nodemailer": "^6.4.4", "@types/rfc2047": "^2.0.1", @@ -65,7 +64,6 @@ "html-to-text": "^8.1.0", "json-stable-stringify": "^1.0.1", "jsonwebtoken": "^8.5.1", - "libqp": "^1.1.0", "marked": "^4.0.12", "nodemailer": "^6.7.2", "rfc2047": "^3.0.1" diff --git a/tests/send-mail.test.ts b/tests/send-mail.test.ts index d967a244c3..d9dac15cfc 100644 --- a/tests/send-mail.test.ts +++ b/tests/send-mail.test.ts @@ -12,9 +12,14 @@ Fcc: Sent Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MIME-Version: 1.0 -To: reviewer@example.com -Cc: Some Body , - And Somebody Else +Header-with-no-value: +Multiline-header: + new line value +To: reviewer@example.com, + Re View , And Nobody Else +Cc: + Some Body , + And Somebody Else , And Nobody Else This Pull Request contains some really important changes that I would love to have included in git.git. @@ -42,23 +47,27 @@ base-commit: 0ae4d8d45ce43d7ad56faff2feeacf8ed5293518 2.17.0.windows.1 `; -test("parse mbox", () => { - const parsed = parseMBox(mbox0); +const to = `reviewer@example.com,\r\n Re View , And Nobody Else `; + +test("parse mbox", async () => { + const parsed = await parseMBox(mbox0); expect(parsed.from).toEqual("Ævar Arnfjörð Bjarmason "); expect(parsed.cc).toEqual([ "Some Body ", - "And Somebody Else ", + "And Somebody Else ", "And Nobody Else ", ]); expect(parsed.subject).toEqual("[PATCH 0/3] My first Pull Request!"); expect(parsed.headers).toEqual([ { key: "Content-Type", value: "text/plain; charset=UTF-8" }, { key: "Content-Transfer-Encoding", value: "8bit" }, { key: "MIME-Version", value: "1.0" }, + { key: "Header-with-no-value", value: "" }, + { key: "Multiline-header", value: "\r\n new line value" }, ]); - expect(parsed.to).toEqual("reviewer@example.com"); + expect(parsed.to).toEqual(to); }); -test("test quoted printable", () => { +test("test quoted printable", async () => { const mbox = `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001 Message-Id: @@ -80,15 +89,15 @@ three byte /=[Ee][0-9A-Fa-f]/=e1=99=ad four byte /=[Ff][0-7]/=f0=90=8d=88 `; - const parsed = parseMBox(mbox); - const body = MailArchiveGitHelper.mbox2markdown(parsed); + const parsed = await parseMBox(mbox); + const body = parsed.body; expect(body).toMatch(/1234/); expect(body).toMatch(/©/); expect(body).toMatch(/᙭/); expect(body).toMatch(/𐍈/); }); -test("test quoted printable ascii", () => { +test("test quoted printable ascii", async () => { const mbox = `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001 Message-Id: @@ -109,12 +118,12 @@ have included in git.git. 2.17.0.windows.1 `; - const parsed = parseMBox(mbox); - const body = MailArchiveGitHelper.mbox2markdown(parsed); + const parsed = await parseMBox(mbox); + const body = parsed.body; expect(body).toMatch(/1234/); }); -test("test base64", () => { +test("test base64", async () => { const mailBody = "Base 64 Data"; const mbox = `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001 @@ -132,12 +141,12 @@ Cc: Some Body , ${Buffer.from(mailBody).toString("base64")}`; - const parsed = parseMBox(mbox); - const body = MailArchiveGitHelper.mbox2markdown(parsed); + const parsed = await parseMBox(mbox); + const body = parsed.body; expect(body).toMatch(mailBody); }); -test("test empty body", () => { +test("test empty body", async () => { const mbox = `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001 Message-Id: @@ -154,7 +163,7 @@ Cc: Some Body , `; - const parsed = parseMBox(mbox); + const parsed = await parseMBox(mbox); const body = MailArchiveGitHelper.mbox2markdown(parsed); expect(body).toMatch(/^$/); });