From 451404680f2d15a85640db8f01d46a057078723e Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 22 Jun 2023 12:37:20 +0300 Subject: [PATCH 1/8] Started with searching from ES --- indexer.js | 2 + lib/api/messages.js | 21 +++++- lib/ensure-es-index.js | 5 ++ lib/search-query.js | 164 +++++++++++++++++++++++++---------------- package.json | 14 ++-- 5 files changed, 134 insertions(+), 72 deletions(-) diff --git a/indexer.js b/indexer.js index 5eec996f..fc4bbec8 100644 --- a/indexer.js +++ b/indexer.js @@ -335,6 +335,8 @@ function indexingJob(esclient) { uid: messageData.uid, answered: messageData.flags ? messageData.flags.includes('\\Answered') : null, + ha: (messageData.attachments && messageData.attachments.length > 0) || false, + attachments: (messageData.attachments && messageData.attachments.map(attachment => diff --git a/lib/api/messages.js b/lib/api/messages.js index 87b43fba..e29a3694 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -21,7 +21,8 @@ const { nextPageCursorSchema, previousPageCursorSchema, pageNrSchema, sessSchema const { preprocessAttachments } = require('../data-url'); const TaskHandler = require('../task-handler'); const prepareSearchFilter = require('../prepare-search-filter'); -const { getMongoDBQuery } = require('../search-query'); +const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query'); +const { getClient } = require('./lib/elasticsearch'); const BimiHandler = require('../bimi-handler'); @@ -584,6 +585,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let query; if (result.value.q) { + let hasESFeatureFlag = await db.redis.sismember(`feature:indexing`, user.toString()); + if (hasESFeatureFlag) { + // search from ElasticSearch + + let searchQuery = await getElasticSearchQuery(db, user, result.value.q); + + const esclient = getClient(); + + let searchResult = await esclient.search({ + index: config.elasticsearch.index, + query: searchQuery, + sort: { uid: 'desc' } + }); + + console.log('ES RESULTS'); + console.log(util.inspect(searchResult, false, 22, true)); + } + filter = await getMongoDBQuery(db, user, result.value.q); query = result.value.q; } else { diff --git a/lib/ensure-es-index.js b/lib/ensure-es-index.js index b96a377d..2efac611 100644 --- a/lib/ensure-es-index.js +++ b/lib/ensure-es-index.js @@ -63,6 +63,11 @@ const mappings = { type: 'boolean' }, + // has attachments + ha: { + type: 'boolean' + }, + attachments: { type: 'nested', properties: { diff --git a/lib/search-query.js b/lib/search-query.js index fa7c1bd8..5028d108 100644 --- a/lib/search-query.js +++ b/lib/search-query.js @@ -314,7 +314,7 @@ const getMongoDBQuery = async (db, user, queryStr) => { return { user: false }; }; -/* + const getElasticSearchQuery = async (db, user, queryStr) => { const parsed = parseSearchQuery(queryStr); @@ -330,9 +330,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { } }; - let curNode = searchQuery; - - let walkTree = async (node, curNode) => { + let walkTree = async node => { if (Array.isArray(node)) { let branches = []; for (let entry of node) { @@ -390,7 +388,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }; if (node.text.negated) { - // FIXME: negation support! + branch = { bool: { must_not: branch.bool.should } }; } return branch; @@ -401,23 +399,47 @@ const getElasticSearchQuery = async (db, user, queryStr) => { if (keyword) { let { value, negated } = node.keywords[keyword]; switch (keyword) { - case 'from': case 'subject': { - let regex = escapeRegexStr(value); let branch = { - headers: { - $elemMatch: { - key: keyword, - value: { - $regex: regex, - $options: 'i' - } + match: { + subject: { + query: value, + operator: 'and' } } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: branch } }; + } + branches.push(branch); + } + break; + + case 'from': + { + let branch = { + bool: { + should: [ + { + match: { + [`from.name`]: { + query: value, + operator: 'and' + } + } + }, + { + term: { + [`from.address`]: value + } + } + ], + minimum_should_match: 1 + } + }; + if (negated) { + branch = { bool: { must_not: branch } }; } branches.push(branch); } @@ -425,24 +447,35 @@ const getElasticSearchQuery = async (db, user, queryStr) => { case 'to': { - let regex = escapeRegexStr(value); + let branch = { + bool: { + should: [], + minimum_should_match: 1 + } + }; + for (let toKey of ['to', 'cc', 'bcc']) { - let branch = { - headers: { - $elemMatch: { - key: toKey, - value: { - $regex: regex, - $options: 'i' + branch.bool.should.push( + { + match: { + [`${toKey}.name`]: { + query: value, + operator: 'and' } } + }, + { + term: { + [`${toKey}.address`]: value + } } - }; - if (negated) { - branch = { $not: branch }; - } - branches.push(branch); + ); + } + + if (negated) { + branch = { bool: { must_not: branch } }; } + branches.push(branch); } break; @@ -468,9 +501,9 @@ const getElasticSearchQuery = async (db, user, queryStr) => { let mailboxEntry = await db.database.collection('mailboxes').findOne(resolveQuery, { project: { _id: -1 } }); - let branch = { mailbox: mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24)) }; + let branch = { term: { mailbox: (mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24))).toString() } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: [branch] } }; } branches.push(branch); @@ -481,9 +514,12 @@ const getElasticSearchQuery = async (db, user, queryStr) => { { value = (value || '').toString().trim(); if (/^[0-9a-f]{24}$/i.test(value)) { - let branch = { thread: new ObjectId(value) }; + let branch = { term: { thread: value } }; if (negated) { - branch = { $not: branch }; + branch = { bool: { must_not: [branch] } }; + } + if (negated) { + branch = { bool: { must_not: [branch] } }; } branches.push(branch); } @@ -493,7 +529,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { case 'has': { switch (value) { case 'attachment': { - branches.push({ ha: true }); + branches.push({ term: { ha: true } }); break; } } @@ -506,39 +542,39 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }; if (parsed && parsed.length) { - let filter = await walkTree(Array.isArray(parsed) ? { $and: parsed } : parsed); - - let extras = { user }; - if (hasTextFilter) { - extras.searchable = true; - } - - return Object.assign({ user: null }, filter, extras); + let filter = await walkTree({ $and: parsed }); + searchQuery.bool.must = searchQuery.bool.must.concat(filter); } - return { user: false }; + return searchQuery; }; -*/ - -module.exports = { parseSearchQuery, getMongoDBQuery /*, getElasticSearchQuery*/ }; -/* -const util = require('util'); - -let main = () => { - let db = require('./db'); - db.connect(() => { - let run = async () => { - let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)']; - - for (let query of queries) { - console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true)); - console.log(util.inspect({ query, parsed: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)); - } - }; +module.exports = { parseSearchQuery, getMongoDBQuery, getElasticSearchQuery }; + +if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') { + const util = require('util'); // eslint-disable-line + let main = () => { + let db = require('./db'); // eslint-disable-line + db.connect(() => { + let run = async () => { + let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)']; + + for (let query of queries) { + console.log('PARSED QUERY'); + console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true)); + console.log('MongoDB'); + console.log(util.inspect({ query, filter: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)); + console.log('ElasticSearch'); + console.log( + util.inspect({ query, filter: await getElasticSearchQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true) + ); + } + }; - run(); - }); -}; -main(); -*/ + run() + .catch(err => console.error(err)) + .finally(() => process.exit()); + }); + }; + main(); +} diff --git a/package.json b/package.json index ccf81d93..0750f205 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "ajv": "8.12.0", "chai": "4.3.7", "docsify-cli": "4.4.4", - "eslint": "8.41.0", + "eslint": "8.43.0", "eslint-config-nodemailer": "1.2.0", "eslint-config-prettier": "8.8.0", "grunt": "1.6.1", @@ -35,7 +35,7 @@ "grunt-mocha-test": "0.13.3", "grunt-shell-spawn": "0.4.0", "grunt-wait": "0.3.0", - "imapflow": "1.0.128", + "imapflow": "1.0.130", "mailparser": "3.6.4", "mocha": "10.2.0", "request": "2.88.2", @@ -53,8 +53,8 @@ "base32.js": "0.1.0", "bcryptjs": "2.4.3", "bson": "5.3.0", - "bullmq": "3.14.0", - "fido2-lib": "3.4.0", + "bullmq": "3.15.8", + "fido2-lib": "3.4.1", "gelf": "2.0.1", "generate-password": "1.7.0", "hash-wasm": "4.9.0", @@ -64,7 +64,7 @@ "iconv-lite": "0.6.3", "ioredfour": "1.2.0-ioredis-07", "ioredis": "5.3.2", - "ipaddr.js": "2.0.1", + "ipaddr.js": "2.1.0", "isemail": "3.2.0", "joi": "17.9.2", "js-yaml": "4.1.0", @@ -73,7 +73,7 @@ "libmime": "5.2.1", "libqp": "2.0.1", "logic-query-parser": "0.0.5", - "mailauth": "4.3.4", + "mailauth": "4.4.0", "mailsplit": "5.4.0", "mobileconfig": "2.4.0", "mongo-cursor-pagination": "8.1.3", @@ -82,7 +82,7 @@ "msgpack5": "6.0.2", "node-forge": "1.3.1", "node-html-parser": "6.1.5", - "nodemailer": "6.9.2", + "nodemailer": "6.9.3", "npmlog": "7.0.1", "openpgp": "5.9.0", "pem-jwk": "2.0.0", From 8c38e2e7f3f7a3fb437bcd4b0a68339183346acd Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 22 Jun 2023 12:41:24 +0300 Subject: [PATCH 2/8] fixed wrong path --- lib/api/messages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index e29a3694..99ac02e8 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -22,7 +22,7 @@ const { preprocessAttachments } = require('../data-url'); const TaskHandler = require('../task-handler'); const prepareSearchFilter = require('../prepare-search-filter'); const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query'); -const { getClient } = require('./lib/elasticsearch'); +const { getClient } = require('../elasticsearch'); const BimiHandler = require('../bimi-handler'); From 2cb80ca0ca2c3e95347e2e432a5e3264f7208a65 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Mon, 17 Jul 2023 10:37:43 +0300 Subject: [PATCH 3/8] somewhat working ES query --- indexer.js | 11 +++++++++-- lib/api/messages.js | 14 ++++++++++++-- lib/api/users.js | 2 ++ lib/search-query.js | 12 ++++++++++-- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/indexer.js b/indexer.js index fc4bbec8..bb65992a 100644 --- a/indexer.js +++ b/indexer.js @@ -56,7 +56,7 @@ class Indexer { this.running = false; log.info('Indexer', 'Stopping indexer'); try { - if (this.changeStream && !this.changeStream.isClosed()) { + if (this.changeStream && !this.changeStream.closed) { await this.changeStream.close(); } } catch (err) { @@ -188,7 +188,14 @@ class Indexer { return; } - if (this.changeStream.isClosed()) { + if (error.errorLabels && error.errorLabels.includes('NonResumableChangeStreamError')) { + // can't resume previous cursor + await db.redis.del('indexer:last'); + log.info('Indexer', 'Can not resume existing cursor'); + return; + } + + if (this.changeStream && this.changeStream.closed) { log.info('Indexer', 'The change stream is closed. Will not wait on any more changes.'); return; } else { diff --git a/lib/api/messages.js b/lib/api/messages.js index 99ac02e8..d3eaf3cb 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -592,11 +592,21 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let searchQuery = await getElasticSearchQuery(db, user, result.value.q); const esclient = getClient(); + console.log( + util.inspect( + { + index: config.elasticsearch.index, + body: { query: searchQuery, sort: { uid: 'desc' } } + }, + false, + 22, + true + ) + ); let searchResult = await esclient.search({ index: config.elasticsearch.index, - query: searchQuery, - sort: { uid: 'desc' } + body: { query: searchQuery, sort: { uid: 'desc' } } }); console.log('ES RESULTS'); diff --git a/lib/api/users.js b/lib/api/users.js index 2b599c9a..6d021dfe 100644 --- a/lib/api/users.js +++ b/lib/api/users.js @@ -859,6 +859,8 @@ module.exports = (db, server, userHandler, settingsHandler) => { fromWhitelist: userData.fromWhitelist || [], + featureFlags: userData.featureFlags || {}, + disabledScopes: userData.disabledScopes || [], hasPasswordSet: !!userData.password || !!userData.tempPassword, diff --git a/lib/search-query.js b/lib/search-query.js index 5028d108..44bbb7dd 100644 --- a/lib/search-query.js +++ b/lib/search-query.js @@ -368,7 +368,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { should: [ { match: { - 'text.plain': { + subject: { query: node.text.value, operator: 'and' } @@ -376,7 +376,15 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }, { match: { - 'text.html': { + text: { + query: node.text.value, + operator: 'and' + } + } + }, + { + match: { + html: { query: node.text.value, operator: 'and' } From 744068d330954cf1549cf40f52854eedf3b17d30 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 20 Jul 2023 13:40:01 +0300 Subject: [PATCH 4/8] Allow including specific header fields in message listing --- docs/api/openapi.yml | 15 +++++ lib/api/messages.js | 147 ++++++++++++++++++++++++++++++++----------- 2 files changed, 124 insertions(+), 38 deletions(-) diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index 832ba470..546553e5 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -1711,6 +1711,12 @@ paths: description: Ordering of the records by insert date schema: $ref: '#/components/schemas/Order' + - name: includeHeaders + in: query + description: 'Comma separated list of header keys to include in the response' + schema: + type: string + example: 'List-ID, MIME-version' - name: next in: query description: 'Cursor value for next page, retrieved from nextCursor response value' @@ -2037,6 +2043,12 @@ paths: description: Ordering of the records by insert date. If no order is supplied, results are sorted by heir mongoDB ObjectId. schema: $ref: '#/components/schemas/Order' + - name: includeHeaders + in: query + description: 'Comma separated list of header keys to include in the response' + schema: + type: string + example: 'List-ID, MIME-version' - name: page in: query description: 'Current page number. Informational only, page numbers start from 1' @@ -6918,6 +6930,9 @@ components: metaData: type: object description: Custom metadata value. Included if metaData query argument was true + headers: + type: object + description: Header object keys requested with the includeHeaders argument GetFilesResult: required: - id diff --git a/lib/api/messages.js b/lib/api/messages.js index d3eaf3cb..b197aad6 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -387,6 +387,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let sortAscending = result.value.order === 'asc'; let filterUnseen = result.value.unseen; + let includeHeaders = result.value.includeHeaders ? result.value.includeHeaders.split(',') : false; + let mailboxData; try { mailboxData = await db.database.collection('mailboxes').findOne( @@ -443,13 +445,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti hdate: true, idate: true, subject: true, - 'mimeTree.parsedHeader.from': true, - 'mimeTree.parsedHeader.sender': true, - 'mimeTree.parsedHeader.to': true, - 'mimeTree.parsedHeader.cc': true, - 'mimeTree.parsedHeader.bcc': true, - 'mimeTree.parsedHeader.content-type': true, - 'mimeTree.parsedHeader.references': true, ha: true, size: true, intro: true, @@ -466,6 +461,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti sortAscending }; + if (includeHeaders) { + // get all headers + opts.fields.projection['mimeTree.parsedHeader'] = true; + } else { + // get only required headers + for (let requiredHeader of [ + 'mimeTree.parsedHeader.from', + 'mimeTree.parsedHeader.sender', + 'mimeTree.parsedHeader.to', + 'mimeTree.parsedHeader.cc', + 'mimeTree.parsedHeader.bcc', + 'mimeTree.parsedHeader.content-type', + 'mimeTree.parsedHeader.references' + ]) { + opts.fields.projection[requiredHeader] = true; + } + } + if (pageNext) { opts.next = pageNext; } else if ((!page || page > 1) && pagePrevious) { @@ -500,7 +513,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti previousCursor: listing.hasPrevious ? listing.previous : false, nextCursor: listing.hasNext ? listing.next : false, specialUse: mailboxData.specialUse, - results: (listing.results || []).map(formatMessageListing) + results: (listing.results || []).map(entry => formatMessageListing(entry, includeHeaders)) }; return res.json(response); @@ -532,6 +545,12 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti attachments: booleanSchema, flagged: booleanSchema, unseen: booleanSchema, + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), searchable: booleanSchema, sess: sessSchema, ip: sessIPSchema @@ -546,6 +565,12 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti threadCounters: booleanSchema.default(false), limit: Joi.number().default(20).min(1).max(250), order: Joi.any().empty('').allow('asc', 'desc').optional(), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), next: nextPageCursorSchema, previous: previousPageCursorSchema, page: pageNrSchema @@ -581,6 +606,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let pagePrevious = result.value.previous; let order = result.value.order; + let includeHeaders = result.value.includeHeaders ? result.value.includeHeaders.split(',') : false; + let filter; let query; @@ -588,29 +615,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let hasESFeatureFlag = await db.redis.sismember(`feature:indexing`, user.toString()); if (hasESFeatureFlag) { // search from ElasticSearch + /* + // TODO: paging and cursors for ElasticSearch results let searchQuery = await getElasticSearchQuery(db, user, result.value.q); const esclient = getClient(); - console.log( - util.inspect( - { - index: config.elasticsearch.index, - body: { query: searchQuery, sort: { uid: 'desc' } } - }, - false, - 22, - true - ) - ); - let searchResult = await esclient.search({ + const searchOpts = { index: config.elasticsearch.index, body: { query: searchQuery, sort: { uid: 'desc' } } - }); + }; + + let searchResult = await esclient.search(searchOpts); + const searchHits = searchResult && searchResult.body && searchResult.body.hits; console.log('ES RESULTS'); console.log(util.inspect(searchResult, false, 22, true)); + */ } filter = await getMongoDBQuery(db, user, result.value.q); @@ -640,13 +662,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti hdate: true, idate: true, subject: true, - 'mimeTree.parsedHeader.from': true, - 'mimeTree.parsedHeader.sender': true, - 'mimeTree.parsedHeader.to': true, - 'mimeTree.parsedHeader.cc': true, - 'mimeTree.parsedHeader.bcc': true, - 'mimeTree.parsedHeader.content-type': true, - 'mimeTree.parsedHeader.references': true, ha: true, intro: true, size: true, @@ -663,6 +678,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti sortAscending: order === 'asc' ? true : undefined }; + if (includeHeaders) { + // get all headers + opts.fields.projection['mimeTree.parsedHeader'] = true; + } else { + // get only required headers + for (let requiredHeader of [ + 'mimeTree.parsedHeader.from', + 'mimeTree.parsedHeader.sender', + 'mimeTree.parsedHeader.to', + 'mimeTree.parsedHeader.cc', + 'mimeTree.parsedHeader.bcc', + 'mimeTree.parsedHeader.content-type', + 'mimeTree.parsedHeader.references' + ]) { + opts.fields.projection[requiredHeader] = true; + } + } + if (pageNext) { opts.next = pageNext; } else if ((!page || page > 1) && pagePrevious) { @@ -697,7 +730,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti page, previousCursor: listing.hasPrevious ? listing.previous : false, nextCursor: listing.hasNext ? listing.next : false, - results: (listing.results || []).map(formatMessageListing) + results: (listing.results || []).map(entry => formatMessageListing(entry, includeHeaders)) }; return res.json(response); @@ -2499,6 +2532,12 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti next: nextPageCursorSchema, previous: previousPageCursorSchema, order: Joi.any().empty('').allow('asc', 'desc').default('desc'), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), page: pageNrSchema, sess: sessSchema, ip: sessIPSchema @@ -2533,6 +2572,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti let pagePrevious = result.value.previous; let sortAscending = result.value.order === 'asc'; + let includeHeaders = result.value.includeHeaders ? result.value.includeHeaders.split(',') : false; + let total = await db.database.collection('archived').countDocuments({ user }); let opts = { @@ -2551,13 +2592,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti hdate: true, idate: true, subject: true, - 'mimeTree.parsedHeader.from': true, - 'mimeTree.parsedHeader.sender': true, - 'mimeTree.parsedHeader.to': true, - 'mimeTree.parsedHeader.cc': true, - 'mimeTree.parsedHeader.bcc': true, - 'mimeTree.parsedHeader.content-type': true, - 'mimeTree.parsedHeader.references': true, ha: true, intro: true, size: true, @@ -2573,6 +2607,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti sortAscending }; + if (includeHeaders) { + // get all headers + opts.fields.projection['mimeTree.parsedHeader'] = true; + } else { + // get only required headers + for (let requiredHeader of [ + 'mimeTree.parsedHeader.from', + 'mimeTree.parsedHeader.sender', + 'mimeTree.parsedHeader.to', + 'mimeTree.parsedHeader.cc', + 'mimeTree.parsedHeader.bcc', + 'mimeTree.parsedHeader.content-type', + 'mimeTree.parsedHeader.references' + ]) { + opts.fields.projection[requiredHeader] = true; + } + } + if (pageNext) { opts.next = pageNext; } else if ((!page || page > 1) && pagePrevious) { @@ -2606,7 +2658,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti m.uid = m._id; return m; }) - .map(formatMessageListing) + .map(entry => formatMessageListing(entry, includeHeaders)) }; return res.json(response); @@ -3127,7 +3179,17 @@ function leftPad(val, chr, len) { return chr.repeat(len - val.toString().length) + val; } -function formatMessageListing(messageData) { +function formatMessageListing(messageData, includeHeaders) { + includeHeaders = [] + .concat(includeHeaders || []) + .map(entry => { + if (typeof entry !== 'string') { + return false; + } + return entry.toLowerCase().trim(); + }) + .filter(entry => entry); + let parsedHeader = (messageData.mimeTree && messageData.mimeTree.parsedHeader) || {}; let from = parsedHeader.from || @@ -3176,6 +3238,15 @@ function formatMessageListing(messageData) { bimi: messageData.bimi }; + if (includeHeaders.length) { + response.headers = {}; + for (let headerKey of includeHeaders) { + if (parsedHeader[headerKey]) { + response.headers[headerKey] = parsedHeader[headerKey]; + } + } + } + if (messageData.meta && 'custom' in messageData.meta) { response.metaData = tools.formatMetaData(messageData.meta.custom); } From 8acdc031534caba87d0a56e4edf2bd349c05f5a7 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 20 Jul 2023 13:43:06 +0300 Subject: [PATCH 5/8] fixed tests --- lib/api/messages.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index b197aad6..0b58e5b5 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -21,8 +21,8 @@ const { nextPageCursorSchema, previousPageCursorSchema, pageNrSchema, sessSchema const { preprocessAttachments } = require('../data-url'); const TaskHandler = require('../task-handler'); const prepareSearchFilter = require('../prepare-search-filter'); -const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query'); -const { getClient } = require('../elasticsearch'); +const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query'); +//const { getClient } = require('../elasticsearch'); const BimiHandler = require('../bimi-handler'); From c96174e8eb21cdb290ea9a25f93c36403bcf3ed3 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 20 Jul 2023 13:43:42 +0300 Subject: [PATCH 6/8] fixed tests --- lib/search-query.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/search-query.js b/lib/search-query.js index 44bbb7dd..8e1bcd3d 100644 --- a/lib/search-query.js +++ b/lib/search-query.js @@ -558,7 +558,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => { }; module.exports = { parseSearchQuery, getMongoDBQuery, getElasticSearchQuery }; - +/* if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') { const util = require('util'); // eslint-disable-line let main = () => { @@ -586,3 +586,4 @@ if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') { }; main(); } +*/ From f711ec73ae0a5d2fed249a6c84cee4c956a92588 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 20 Jul 2023 13:47:56 +0300 Subject: [PATCH 7/8] Bumped deps --- package.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 0750f205..63e39dec 100644 --- a/package.json +++ b/package.json @@ -26,24 +26,24 @@ "ajv": "8.12.0", "chai": "4.3.7", "docsify-cli": "4.4.4", - "eslint": "8.43.0", + "eslint": "8.45.0", "eslint-config-nodemailer": "1.2.0", "eslint-config-prettier": "8.8.0", "grunt": "1.6.1", "grunt-cli": "1.4.3", - "grunt-eslint": "24.1.0", + "grunt-eslint": "24.3.0", "grunt-mocha-test": "0.13.3", "grunt-shell-spawn": "0.4.0", "grunt-wait": "0.3.0", - "imapflow": "1.0.130", - "mailparser": "3.6.4", + "imapflow": "1.0.134", + "mailparser": "3.6.5", "mocha": "10.2.0", "request": "2.88.2", "supertest": "6.3.3" }, "dependencies": { "@fidm/x509": "1.2.1", - "@opensearch-project/opensearch": "2.2.0", + "@opensearch-project/opensearch": "2.3.1", "@phc/pbkdf2": "1.1.14", "@postalsys/vmc": "1.0.6", "@root/acme": "3.1.0", @@ -52,7 +52,7 @@ "axios": "1.4.0", "base32.js": "0.1.0", "bcryptjs": "2.4.3", - "bson": "5.3.0", + "bson": "5.4.0", "bullmq": "3.15.8", "fido2-lib": "3.4.1", "gelf": "2.0.1", @@ -82,7 +82,7 @@ "msgpack5": "6.0.2", "node-forge": "1.3.1", "node-html-parser": "6.1.5", - "nodemailer": "6.9.3", + "nodemailer": "6.9.4", "npmlog": "7.0.1", "openpgp": "5.9.0", "pem-jwk": "2.0.0", From 9ac5b61ea62f602d9a8bd5fdd2fcfcfed67f06a1 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 20 Jul 2023 13:55:45 +0300 Subject: [PATCH 8/8] Bumped deps --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 63e39dec..75adab26 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "base32.js": "0.1.0", "bcryptjs": "2.4.3", "bson": "5.4.0", - "bullmq": "3.15.8", + "bullmq": "4.6.0", "fido2-lib": "3.4.1", "gelf": "2.0.1", "generate-password": "1.7.0", @@ -73,7 +73,7 @@ "libmime": "5.2.1", "libqp": "2.0.1", "logic-query-parser": "0.0.5", - "mailauth": "4.4.0", + "mailauth": "4.4.1", "mailsplit": "5.4.0", "mobileconfig": "2.4.0", "mongo-cursor-pagination": "8.1.3",