Merge pull request gitgitgadget#889 from webstech/mailparser

sendMail: change parseMBox to use simpleParser
firoj0 · Feb 13, 2022 · 65a6be9 · 65a6be9
2 parents 7ab4019 + 05024a2
commit 65a6be9
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 86 deletions.
diff --git a/lib/gitgitgadget.ts b/lib/gitgitgadget.ts
@@ -168,7 +168,7 @@ export class GitGitGadget {
         const email = userInfo.email;
 
         const send = async (mail: string): Promise<string> => {
-            const mbox = parseMBox(mail);
+            const mbox = await parseMBox(mail);
             mbox.cc = [];
             mbox.to = email;
             console.log(mbox);

diff --git a/lib/mail-archive-helper.ts b/lib/mail-archive-helper.ts
@@ -1,5 +1,4 @@
 import { createHash } from "crypto";
-import * as libqp from "libqp";
 import { git, revParse } from "./git";
 import { GitNotes } from "./git-notes";
 import { IGitGitGadgetOptions } from "./gitgitgadget";
@@ -43,20 +42,7 @@ export class MailArchiveGitHelper {
     }
 
     public static mbox2markdown(mbox: IParsedMBox): string {
-        let body = mbox.body;
-
-        const headers = mbox.headers || [];
-        for (const header of headers) {
-            if (header.key === "Content-Transfer-Encoding") {
-                const value = header.value.toLowerCase();
-                if (value === "base64") {
-                    body = Buffer.from(body, "base64").toString();
-                } else if (value === "quoted-printable") {
-                    const buffer = libqp.decode(body);
-                    body = buffer.toString("utf-8");
-                }
-            }
-        }
+        const body = mbox.body;
 
         if (!body.length) {
             return "";
@@ -156,7 +142,7 @@ export class MailArchiveGitHelper {
         };
 
         const mboxHandler = async (mbox: string): Promise<void> => {
-                const parsedMbox = parseMBox(mbox, true);
+                const parsedMbox = await parseMBox(mbox, true);
                 if (!parsedMbox.headers) {
                     throw new Error(`Could not parse ${mbox}`);
                 }

diff --git a/lib/send-mail.ts b/lib/send-mail.ts
@@ -1,3 +1,4 @@
+import { simpleParser, SimpleParserOptions } from "mailparser";
 import { createTransport, SendMailOptions } from "nodemailer";
 import SMTPTransport = require("nodemailer/lib/smtp-transport");
 import { decode } from "rfc2047";
@@ -24,12 +25,7 @@ export interface ISMTPOptions {
 export async function parseHeadersAndSendMail(mbox: string,
                                               smtpOptions: ISMTPOptions):
     Promise<string> {
-    return await sendMail(parseMBox(mbox), smtpOptions);
-}
-
-function replaceAll(input: string, pattern: string, replacement: string):
-    string {
-    return input.split(pattern).join(replacement);
+    return await sendMail( await parseMBox(mbox), smtpOptions);
 }
 
 /**
@@ -41,17 +37,7 @@ function replaceAll(input: string, pattern: string, replacement: string):
  * @param {string} mbox The mail, in mbox format
  * @returns {IParsedMBox} the parsed headers/body
  */
-export function parseMBox(mbox: string, gentle?: boolean):
-    IParsedMBox {
-    const headerEnd = mbox.indexOf("\n\n");
-    if (headerEnd < 0) {
-        throw new Error("Could not parse mail");
-    }
-    const headerStart = mbox.startsWith("From ") ? mbox.indexOf("\n") + 1 : 0;
-
-    const header = mbox.substr(headerStart, headerEnd - headerStart);
-    const body = mbox.substr(headerEnd + 2);
-
+export async function parseMBox(mbox: string, gentle?: boolean): Promise<IParsedMBox> {
     let cc: string[] | undefined;
     let date: string | undefined;
     let from: string | undefined;
@@ -60,15 +46,29 @@ export function parseMBox(mbox: string, gentle?: boolean):
     let subject: string | undefined;
     let to: string | undefined;
 
-    for (const line of header.split(/\n(?![ \t])/)) {
-        const colon = line.indexOf(": ");
-        if (colon < 0) {
-            throw new Error(`Failed to parse header line '${line}`);
+    const options: SimpleParserOptions = {
+        skipHtmlToText: true,
+        skipTextLinks : true,
+        skipTextToHtml: true
+    };
+
+    const parsed = await simpleParser(mbox, options);
+
+    for (const entry of parsed.headerLines) {
+        const valueSet = entry.line.match(/(.*): *([^]*)$/);
+        if (!valueSet) {
+            if (entry.line[entry.line.length - 1] === ":") {
+                continue;
+            }
+            throw new Error(`Failed to parse header line '${entry.line}'`);
         }
-        const key = line.substr(0, colon);
-        const value = replaceAll(line.substr(colon + 2), "\n ", " ");
-        switch (key.toLowerCase()) {
-            case "cc": cc = (cc || []).concat(value.split(", ")); break;
+        const key = valueSet[1];
+        const value = valueSet[2];
+
+        switch (entry.key) {
+            case "cc": cc = (cc || []).concat(value.replace(/\r?\n/g, " ").split(", ").map(item =>
+                    item.trim()
+                )); break;
             case "date": date = value; break;
             case "fcc": break;
             case "from": from = decode(value.trim()); break;
@@ -81,11 +81,11 @@ export function parseMBox(mbox: string, gentle?: boolean):
     }
 
     if (!gentle && (!to || !subject || !from)) {
-        throw new Error(`Missing To, Subject and/or From header:\n${header}`);
+        throw new Error(`Missing To, Subject and/or From header:\n${mbox}`);
     }
 
     return {
-        body,
+        body: parsed.text || "",
         cc,
         date,
         from,

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -39,7 +39,6 @@
     "@types/jest": "^27.4.0",
     "@types/json-stable-stringify": "^1.0.33",
     "@types/jsonwebtoken": "^8.5.8",
-    "@types/libqp": "^1.1.1",
     "@types/marked": "^4.0.2",
     "@types/nodemailer": "^6.4.4",
     "@types/rfc2047": "^2.0.1",
@@ -65,7 +64,6 @@
     "html-to-text": "^8.1.0",
     "json-stable-stringify": "^1.0.1",
     "jsonwebtoken": "^8.5.1",
-    "libqp": "^1.1.0",
     "marked": "^4.0.12",
     "nodemailer": "^6.7.2",
     "rfc2047": "^3.0.1"

diff --git a/tests/send-mail.test.ts b/tests/send-mail.test.ts
@@ -12,9 +12,14 @@ Fcc: Sent
 Content-Type: text/plain; charset=UTF-8
 Content-Transfer-Encoding: 8bit
 MIME-Version: 1.0
-To: [email protected]
-Cc: Some Body <[email protected]>,
- And Somebody Else <[email protected]>
+Header-with-no-value:
+Multiline-header:
+ new line value
+To: [email protected],
+ Re View <[email protected]>, And Nobody Else <[email protected]>
+Cc:
+ Some Body <[email protected]>,
+ And Somebody Else <[email protected]>, And Nobody Else <[email protected]>
 
 This Pull Request contains some really important changes that I would love to
 have included in git.git.
@@ -42,23 +47,27 @@ base-commit: 0ae4d8d45ce43d7ad56faff2feeacf8ed5293518
 2.17.0.windows.1
 `;
 
-test("parse mbox", () => {
-    const parsed = parseMBox(mbox0);
+const to = `[email protected],\r\n Re View <[email protected]>, And Nobody Else <[email protected]>`;
+
+test("parse mbox", async () => {
+    const parsed = await parseMBox(mbox0);
     expect(parsed.from).toEqual("Ævar Arnfjörð Bjarmason <[email protected]>");
     expect(parsed.cc).toEqual([
         "Some Body <[email protected]>",
-        "And Somebody Else <[email protected]>",
+        "And Somebody Else <[email protected]>", "And Nobody Else <[email protected]>",
     ]);
     expect(parsed.subject).toEqual("[PATCH 0/3] My first Pull Request!");
     expect(parsed.headers).toEqual([
         { key: "Content-Type", value: "text/plain; charset=UTF-8" },
         { key: "Content-Transfer-Encoding", value: "8bit" },
         { key: "MIME-Version", value: "1.0" },
+        { key: "Header-with-no-value", value: "" },
+        { key: "Multiline-header", value: "\r\n new line value" },
     ]);
-    expect(parsed.to).toEqual("[email protected]");
+    expect(parsed.to).toEqual(to);
 });
 
-test("test quoted printable", () => {
+test("test quoted printable", async () => {
     const mbox =
     `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001
 Message-Id: <[email protected]>
@@ -80,15 +89,15 @@ three byte /=[Ee][0-9A-Fa-f]/=e1=99=ad
 four byte /=[Ff][0-7]/=f0=90=8d=88
 `;
 
-    const parsed = parseMBox(mbox);
-    const body = MailArchiveGitHelper.mbox2markdown(parsed);
+    const parsed = await parseMBox(mbox);
+    const body = parsed.body;
     expect(body).toMatch(/1234/);
     expect(body).toMatch(/©/);
     expect(body).toMatch(/᙭/);
     expect(body).toMatch(/𐍈/);
 });
 
-test("test quoted printable ascii", () => {
+test("test quoted printable ascii", async () => {
     const mbox =
     `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001
 Message-Id: <[email protected]>
@@ -109,12 +118,12 @@ have included in git.git.
 2.17.0.windows.1
 `;
 
-    const parsed = parseMBox(mbox);
-    const body = MailArchiveGitHelper.mbox2markdown(parsed);
+    const parsed = await parseMBox(mbox);
+    const body = parsed.body;
     expect(body).toMatch(/1234/);
 });
 
-test("test base64", () => {
+test("test base64", async () => {
     const mailBody = "Base 64 Data";
     const mbox =
     `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001
@@ -132,12 +141,12 @@ Cc: Some Body <[email protected]>,
 
 ${Buffer.from(mailBody).toString("base64")}`;
 
-    const parsed = parseMBox(mbox);
-    const body = MailArchiveGitHelper.mbox2markdown(parsed);
+    const parsed = await parseMBox(mbox);
+    const body = parsed.body;
     expect(body).toMatch(mailBody);
 });
 
-test("test empty body", () => {
+test("test empty body", async () => {
     const mbox =
     `From 566155e00ab72541ff0ac21eab84d087b0e882a5 Mon Sep 17 00:00:00 2001
 Message-Id: <[email protected]>
@@ -154,7 +163,7 @@ Cc: Some Body <[email protected]>,
 
 `;
 
-    const parsed = parseMBox(mbox);
+    const parsed = await parseMBox(mbox);
     const body = MailArchiveGitHelper.mbox2markdown(parsed);
     expect(body).toMatch(/^$/);
 });