diff --git a/functions/src/cron/index.ts b/functions/src/cron/index.ts index 2a3f347d7..791b8dc8f 100644 --- a/functions/src/cron/index.ts +++ b/functions/src/cron/index.ts @@ -1,7 +1,5 @@ import importExchangeRatesFunction from './exchange-rate-import'; -import importBalanceMailFunction from './postfinance-balance-import'; import importPostfinancePaymentsFilesFunction from './postfinance-payments-files-import'; -export const importBalanceMail = importBalanceMailFunction; export const importPostfinancePaymentsFiles = importPostfinancePaymentsFilesFunction; export const importExchangeRates = importExchangeRatesFunction; diff --git a/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.test.ts b/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.test.ts deleted file mode 100644 index 18adacd5f..000000000 --- a/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { describe, expect, test } from '@jest/globals'; -import functions from 'firebase-functions-test'; -import { FirestoreAdmin } from '../../../../shared/src/firebase/admin/FirestoreAdmin'; -import { getOrInitializeFirebaseAdmin } from '../../../../shared/src/firebase/admin/app'; -import { - BANK_BALANCE_FIRESTORE_PATH, - BankBalance, - getIdFromBankBalance, -} from '../../../../shared/src/types/bank-balance'; -import { PostFinanceBalanceImporter } from './PostFinanceBalanceImporter'; - -describe('importPostfinanceBalance', () => { - const projectId = 'test' + new Date().getTime(); - const testEnv = functions({ projectId: projectId }); - const firestoreAdmin = new FirestoreAdmin(getOrInitializeFirebaseAdmin({ projectId: projectId })); - const postfinanceImporter = new PostFinanceBalanceImporter(); - - beforeEach(async () => { - await testEnv.firestore.clearFirestoreData({ projectId: projectId }); - }); - - test('extract data correctly', () => { - const testMail = - '<table class=3D"body white" data-made-with-foundation=3D"" style=\n' + - '=3D"border-spacing:0;border-collapse:collapse;padding:0;vertical-align:top;=\n' + - 'text-align:left;background:#ffffff;width:100%;color:#0a0a0a;font-family:Fru=\n' + - 'tiger, Helvetica, Arial, sans-serif;font-weight:normal;margin:0;Margin:0;li=\n' + - 'ne-height:19px;font-size:14px;"><tbody><tr style=3D"padding:0;vertical-alig=\n' + - 'n:top;text-align:left;"><td class=3D"center" align=3D"center" valign=3D"top=\n' + - '" style=3D"word-wrap:break-word;-webkit-hyphens:auto;-moz-hyphens:auto;hyph=\n' + - 'ens:auto;border-collapse:collapse !important;padding:0;vertical-align:top;t=\n' + - 'ext-align:left;color:#0a0a0a;font-family:Frutiger, Helvetica, Arial, sans-s=\n' + - 'erif;font-weight:normal;margin:0;Margin:0;line-height:19px;font-size:14px;"=\n' + - '><center style=3D"width:100%;"><!--[if !mso]><!--><table class=3D"container=\n' + - '" align=3D"center" bgcolor=3D"#ffffff" cellpadding=3D"0" cellspacing=3D"0" =\n' + - 'style=3D"border-spacing:0;border-collapse:collapse;padding:0;vertical-align=\n' + - ':top;text-align:inherit;background:#fefefe;margin:0 auto;Margin:0 auto;max-=\n' + - 'width:580px;"><!--<![endif]--><!--[if mso]><table class=3D"container" align=\n' + - '=3D"center" bgcolor=3D"#ffffff" cellpadding=3D"0" cellspacing=3D"0" width=\n' + - '=3D"600"><![endif]--><tbody><tr style=3D"padding:0;vertical-align:top;text-=\n' + - 'align:left;"><td style=3D"word-wrap:break-word;-webkit-hyphens:auto;-moz-hy=\n' + - 'phens:auto;hyphens:auto;border-collapse:collapse !important;padding:0;verti=\n' + - 'cal-align:top;text-align:left;color:#0a0a0a;font-family:Frutiger, Helvetica=\n' + - ', Arial, sans-serif;font-weight:normal;margin:0;Margin:0;line-height:19px;f=\n' + - 'ont-size:14px;"><div class=3D"logo"><img alt=3D"PostFinance AG" src=3D"http=\n' + - 's://www.post.ch/static/Notifica//postfinance/postfinance-logo.png" style=3D=\n' + - '"outline:none;text-decoration:none;-ms-interpolation-mode:bicubic;width:100=\n' + - '%;max-width:100%;clear:both;display:block;" /></div></td></tr><tr style=3D"=\n' + - 'padding:0;vertical-align:top;text-align:left;"><td class=3D"first last colu=\n' + - 'mns padded" style=3D"word-wrap:break-word;-webkit-hyphens:auto;-moz-hyphens=\n' + - ':auto;hyphens:auto;border-collapse:collapse !important;padding:8px;vertical=\n' + - '-align:top;text-align:left;margin:0;Margin:0;padding-top:8px;padding-bottom=\n' + - ':8px;padding-left:8px;padding-right:8px;color:#0a0a0a;font-family:Frutiger,=\n' + - ' Helvetica, Arial, sans-serif;font-weight:normal;line-height:19px;font-size=\n' + - ':14px;"><br /><div>Dear Sir or Madam</div><br /><div>A credit of 80.00 has =\n' + - "been booked to the account Contributions. Current balance: CHF 50'993=\n" + - '97.53.</div><br /></td></tr><tr style=3D"padding:0;vertical-align:top;text-=\n' + - 'align:left;"><td class=3D"first last columns padded" style=3D"word-wrap:bre=\n' + - 'ak-word;-webkit-hyphens:auto;-moz-hyphens:auto;hyphens:auto;border-collapse=\n' + - ':collapse !important;padding:8px;vertical-align:top;text-align:left;margin:=\n' + - '0;Margin:0;padding-top:8px;padding-bottom:8px;padding-left:8px;padding-righ=\n' + - 't:8px;color:#0a0a0a;font-family:Frutiger, Helvetica, Arial, sans-serif;font=\n' + - '-weight:normal;line-height:19px;font-size:14px;"><br /><div>PostFinance</di=\n' + - 'v><br /><div>Do not reply to this automatically generated e-mail. Notificat=\n' + - 'ion settings can be modified in e-finance under =E2=80=9CSettings and profi=\n' + - 'le=E2=80=9D > =E2=80=9CNotifications=E2=80=9D.</div><br /></td></tr><tr sty=\n' + - 'le=3D"padding:0;vertical-align:top;text-align:left;"><td class=3D"first las=\n' + - 't columns padded secondary" style=3D"word-wrap:break-word;-webkit-hyphens:a=\n' + - 'uto;-moz-hyphens:auto;hyphens:auto;border-collapse:collapse !important;padd=\n' + - 'ing:8px;vertical-align:top;text-align:left;margin:0;Margin:0;padding-top:8p=\n' + - 'x;padding-bottom:8px;padding-left:8px;padding-right:8px;color:#0a0a0a;font-=\n' + - 'family:Frutiger, Helvetica, Arial, sans-serif;font-weight:normal;line-heigh=\n' + - 't:19px;font-size:smaller;background:#f5f1e8;"><div><strong>If you have any =\n' + - 'questions:</strong> 0848 888 710 (max. CHF 0.08/Min. in Switzerland)</div><=\n' + - 'div class=3D"right" style=3D"text-align:right;">Follow us on:<a href=3D"htt=\n' + - 'ps://www.postfinance.ch/socialmedia" style=3D"color:#2199e8;font-family:Fru=\n' + - 'tiger, Helvetica, Arial, sans-serif;font-weight:normal;padding:0;margin:0;M=\n' + - 'argin:0;text-align:left;line-height:1.3;text-decoration:none;"><img alt=3D"=\n' + - 'facebook" style=3D"outline:none;text-decoration:none;-ms-interpolation-mode=\n' + - ':bicubic;width:16px;max-width:100%;clear:both;display:inline;border:none;he=\n' + - 'ight:16px;" src=3D"https://www.post.ch/static/Notifica//postfinance/faceboo=\n' + - 'k.png" /></a>=C2=A0<a href=3D"https://www.postfinance.ch/socialmedia" style=\n' + - '=3D"color:#2199e8;font-family:Frutiger, Helvetica, Arial, sans-serif;font-w=\n' + - 'eight:normal;padding:0;margin:0;Margin:0;text-align:left;line-height:1.3;te=\n' + - 'xt-decoration:none;"><img alt=3D"twitter" style=3D"outline:none;text-decora=\n' + - 'tion:none;-ms-interpolation-mode:bicubic;width:16px;max-width:100%;clear:bo=\n' + - 'th;display:inline;border:none;height:16px;" src=3D"https://www.post.ch/stat=\n' + - 'ic/Notifica//postfinance/twitter.png" /></a>=C2=A0<a href=3D"https://www.po=\n' + - 'stfinance.ch/socialmedia" style=3D"color:#2199e8;font-family:Frutiger, Helv=\n' + - 'etica, Arial, sans-serif;font-weight:normal;padding:0;margin:0;Margin:0;tex=\n' + - 't-align:left;line-height:1.3;text-decoration:none;"><img alt=3D"youtube" st=\n' + - 'yle=3D"outline:none;text-decoration:none;-ms-interpolation-mode:bicubic;wid=\n' + - 'th:16px;max-width:100%;clear:both;display:inline;border:none;height:16px;" =\n' + - 'src=3D"https://www.post.ch/static/Notifica//postfinance/youtube.png" /></a>=\n' + - '</div></td></tr></tbody></table></center></td></tr></tbody></table>'; - - expect('contributions').toEqual(postfinanceImporter.extractAccount(testMail)); - expect(50).toEqual(postfinanceImporter.extractBalance(testMail)); - }); - - test('inserts balances into firestore', async () => { - const balances = [ - { - timestamp: 1663339392, - account: 'testAccount', - balance: 1000, - currency: 'CHF', - } as BankBalance, - ]; - - await postfinanceImporter.storeBalances(balances); - - const balance = balances[0]; - const snap = await firestoreAdmin - .doc<BankBalance>(BANK_BALANCE_FIRESTORE_PATH, getIdFromBankBalance(balance)) - .get(); - expect(balance).toEqual(snap.data()); - }); - jest.setTimeout(30000); -}); diff --git a/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.ts b/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.ts deleted file mode 100644 index ff4f5952a..000000000 --- a/functions/src/cron/postfinance-balance-import/PostFinanceBalanceImporter.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { logger } from 'firebase-functions'; -import imaps from 'imap-simple'; -import _ from 'lodash'; -import { Source, simpleParser } from 'mailparser'; -import { FirestoreAdmin } from '../../../../shared/src/firebase/admin/FirestoreAdmin'; -import { - BANK_BALANCE_FIRESTORE_PATH, - BankBalance, - getIdFromBankBalance, -} from '../../../../shared/src/types/bank-balance'; -import { POSTFINANCE_EMAIL_PASSWORD, POSTFINANCE_EMAIL_USER } from '../../config'; - -export class PostFinanceBalanceImporter { - private readonly accountRegex = /(?<=account\W)(?<account>.*?)(?=\W)/; // regex to retrieve the account name from the email - private readonly balanceRegex = /balance: CHF (?<balance>[0-9’.]*)/; // regex to retrieve the balance from the email - // we retrieve only unseen mails and mark them as seen once we imported the balance - private readonly searchCriteria = ['UNSEEN']; - private readonly fetchOptions = { bodies: ['HEADER', 'TEXT', ''], markSeen: true }; - private readonly firestoreAdmin: FirestoreAdmin; - - constructor() { - this.firestoreAdmin = new FirestoreAdmin(); - } - - extractBalance = (html: String) => { - return Number.parseFloat(html.match(this.balanceRegex)!.groups!['balance'].replace('’', '')); - }; - - storeBalances = async (balances: BankBalance[]): Promise<void> => { - for await (const balance of balances) { - await this.firestoreAdmin - .doc<BankBalance>(BANK_BALANCE_FIRESTORE_PATH, getIdFromBankBalance(balance)) - .set(balance); - } - }; - - extractAccount = (html: String) => { - return html.match(this.accountRegex)!.groups!['account'].toLowerCase(); - }; - - retrieveBalanceMails = async (): Promise<BankBalance[]> => { - try { - logger.info('Start checking balance inbox'); - const config = { - imap: { - user: POSTFINANCE_EMAIL_USER, - password: POSTFINANCE_EMAIL_PASSWORD, - host: 'imap.gmail.com', - port: 993, - tls: true, - tlsOptions: { rejectUnauthorized: false }, - authTimeout: 10000, - }, - }; - const connection = await imaps.connect(config); - await connection.openBox('INBOX'); - logger.info('Connected to inbox'); - const messages = await connection.search(this.searchCriteria, this.fetchOptions); - const balances = await Promise.all( - messages.map(async (item: any) => { - const all = _.find(item.parts, { which: '' }); - const id = item.attributes.uid; - const idHeader = 'Imap-Id: ' + id + '\r\n'; - const source = idHeader + all?.body; - return this.parseEmail(source); - }), - ); - connection.end(); - logger.info('Retrieved balances'); - return balances.flat(); - } catch (error) { - logger.error('Could not ingest balance mails', error); - return []; - } - }; - - parseEmail = async (source: Source): Promise<BankBalance[]> => { - const mail = await simpleParser(source); - if (!mail || !mail.html) return []; - try { - return [ - { - timestamp: mail.date!.getTime() / 1000, - account: this.extractAccount(mail.html), - balance: this.extractBalance(mail.html), - currency: 'CHF', - } as BankBalance, - ]; - } catch { - logger.info(`Could not parse email with subject ${mail.subject}`); - return []; - } - }; -} diff --git a/functions/src/cron/postfinance-balance-import/index.ts b/functions/src/cron/postfinance-balance-import/index.ts deleted file mode 100644 index 51f34bb39..000000000 --- a/functions/src/cron/postfinance-balance-import/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { onSchedule } from 'firebase-functions/v2/scheduler'; -import { PostFinanceBalanceImporter } from './PostFinanceBalanceImporter'; - -/** - * Function periodically connects to the gmail account where we send the postfinance balance statements, - * parses the emails and stores the current balances into firestore. - */ -export default onSchedule('0 * * * *', async () => { - const postFinanceImporter = new PostFinanceBalanceImporter(); - const balances = await postFinanceImporter.retrieveBalanceMails(); - await postFinanceImporter.storeBalances(balances); -}); diff --git a/shared/src/types/bank-balance.ts b/shared/src/types/bank-balance.ts deleted file mode 100644 index 8234f1b14..000000000 --- a/shared/src/types/bank-balance.ts +++ /dev/null @@ -1,12 +0,0 @@ -export const BANK_BALANCE_FIRESTORE_PATH = 'postfinance-balances'; - -export type BankBalance = { - account: string; - balance: number; - currency: string; - timestamp: number; -}; - -export const getIdFromBankBalance = (balance: BankBalance) => { - return `${balance.account}_${balance.timestamp}`; -};