From b7b4e7cd7251ba5418edb25ae91654c99be6d7dd Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Mon, 20 Jan 2025 15:37:33 -0800 Subject: [PATCH] connection.ini: replaces ... haproxy_hosts, ehlo_hello_message, connection_close_message, banner_includes_uuid, deny_includes_uuid, databytes, max_mime_parts, max_line_length, max_data_line_length, and smtpgreeting. To upgrade, apply any localized settings from the old config files to the new connection.ini file and for tidiness, delete the old config files. - moved the following settings from smtp.ini to connection.ini: - headers.* - main.smtp_utf8 - main.strict_rfc1869 - early_talker.pause, removed support, use earlytalker.ini --- Changes.md | 15 ++++ config/connection.ini | 63 +++++++++++++++++ config/connection_close_message | 1 - config/databytes | 1 - config/max_unrecognized_commands | 1 - config/smtp.ini | 18 ----- connection.js | 113 ++++++++++++++----------------- docs/CoreConfig.md | 95 +++++--------------------- docs/HAProxy.md | 38 +++-------- plugins/early_talker.js | 5 -- server.js | 14 ---- test/connection.js | 2 +- 12 files changed, 157 insertions(+), 209 deletions(-) create mode 100644 config/connection.ini delete mode 100644 config/connection_close_message delete mode 100644 config/databytes delete mode 100644 config/max_unrecognized_commands diff --git a/Changes.md b/Changes.md index ad975c176..18b5c3968 100644 --- a/Changes.md +++ b/Changes.md @@ -4,9 +4,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ### Unreleased +#### BREAKING, ACTION REQUIRED + +- connection.ini: new config file, replaces haproxy_hosts, ehlo_hello_message, connection_close_message, banner_includes_uuid, deny_includes_uuid, databytes, max_mime_parts, max_line_length, max_data_line_length, and smtpgreeting. To upgrade, apply any localized settings to the new connection.ini file. +- moved the following settings from smtp.ini to connection.ini: + - headers.* + - main.smtp_utf8 + - main.strict_rfc1869 +- early_talker.pause, removed support, use earlytalker.ini + +#### Changes + - deps(eslint): update to v9 - docs(plugins/\*.md): use \# to indicate heading levels - deps(various): bump to latest versions +- docs(CoreConfig): removed incorrect early_talker.delay reference (hasn't worked in years). + +#### Fixes + - fix(outbound): in outbound hook_delivered, when mx.exchange contains an IP, use mx.from_dns - fix(bin/haraka): fix for finding path to config/docs/Plugins.md diff --git a/config/connection.ini b/config/connection.ini new file mode 100644 index 000000000..cde880cac --- /dev/null +++ b/config/connection.ini @@ -0,0 +1,63 @@ +; +[main] + +; Require senders to conform to RFC 1869 and RFC 821 when sending the MAIL FROM and RCPT TO commands. In particular, the inclusion of spurious spaces or missing angle brackets will be rejected. +; strict_rfc1869 = false + +; Advertise support for SMTPUTF8 (RFC-6531) +; smtputf8=true + + +[haproxy] +; Array: hosts or CIDRs that Haraka should enable the PROXY protocol from. See docs/HAProxy for format +hosts[] = +; hosts[] = 192.0.2.4 +; hosts[] = 192.0.2.5 +; hosts[] = [2001:db8::1] +; hosts[] = [2001:db8::2] + + +[headers] +; add_received=true +; clean_auth_results=true + +; show_version=true + +max_lines=1000 + +max_received=100 + + +[max] +; Integer. The maximum SIZE of an email +bytes=26214400 + +; Integer. Limit a potential denial of service in potentially hostile emails. +mime_parts=1000 + +; Integer. The maximum length of lines in SMTP session commands (e.g. RCPT, HELO etc). Defaults to 512 (bytes) as mandated by RFC 5321 §4.5.3.1.4. Clients exceeding this limit will be immediately disconnected with a "521 Command line too long" error. +line_length=512 + +; Integer. The maximum length of lines in the DATA section of emails. Defaults to 992 (bytes), the limit set by Sendmail. When this limit is exceeded the three bytes "\r\n " (0x0d 0x0a 0x20) are inserted into the stream to "fix" it. This has the potential to "break" some email, but makes it more likely to be accepted by upstream/downstream services, and is the same behaviour as Sendmail. Also when the data line length limit is exceeded `transaction.notes.data_line_length_exceeded` is set to `true`. +data_line_length=992 + + +[message] +; Array. The greeting used when a client connects. +; greeting[]=My Custom +; greeting[]=Greeting Message + +helo=Haraka is at your service. + +; String. Override the default connection close message. +close=closing connection. Have a jolly good day. + + +[uuid] +; integer, how many UUID chars to show. +; 0 = none, 6 is enough to be unique per day, 40 will include the +; full connection and transaction UUID +banner_chars=6 + +; include N characters of the uuid (in brackets) at the start of each line of the deny message +deny_chars=0 diff --git a/config/connection_close_message b/config/connection_close_message deleted file mode 100644 index 204851277..000000000 --- a/config/connection_close_message +++ /dev/null @@ -1 +0,0 @@ -closing connection. Have a jolly good day. \ No newline at end of file diff --git a/config/databytes b/config/databytes deleted file mode 100644 index 56e1e76b5..000000000 --- a/config/databytes +++ /dev/null @@ -1 +0,0 @@ -26214400 diff --git a/config/max_unrecognized_commands b/config/max_unrecognized_commands deleted file mode 100644 index f599e28b8..000000000 --- a/config/max_unrecognized_commands +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/config/smtp.ini b/config/smtp.ini index 142bab332..26a4b37c0 100644 --- a/config/smtp.ini +++ b/config/smtp.ini @@ -44,21 +44,3 @@ ; after this time it will hard close. 30s is usually long enough to ; wait for outbound connections to finish. ;force_shutdown_timeout=30 - -; SMTP service extensions: https://tools.ietf.org/html/rfc1869 -; strict_rfc1869 = false - -; Advertise support for SMTPUTF8 (RFC-6531) -;smtputf8=true - -[headers] -;add_received=true -;clean_auth_results=true - -;show_version=true - -; replace max_header_lines -max_lines=1000 - -; replace max_received_count -max_received=100 diff --git a/connection.js b/connection.js index c185bd741..c5fe59996 100644 --- a/connection.js +++ b/connection.js @@ -24,33 +24,33 @@ const outbound = require('./outbound'); const states = constants.connection.state; -// Load HAProxy hosts into an object for fast lookups -// as this list is checked on every new connection. -let haproxy_hosts_ipv4 = []; -let haproxy_hosts_ipv6 = []; -function loadHAProxyHosts () { - const hosts = config.get('haproxy_hosts', 'list', loadHAProxyHosts); - const new_ipv4_hosts = []; - const new_ipv6_hosts = []; - for (let i=0; i { return ipaddr.parse(self.remote.ip).match(element[0], element[1]); })) { @@ -401,10 +392,10 @@ class Connection { let maxlength; if (this.state === states.PAUSE_DATA || this.state === states.DATA) { - maxlength = this.max_data_line_length; + maxlength = cfg.max.data_line_length; } else { - maxlength = this.max_line_length; + maxlength = cfg.max.line_length; } let offset; @@ -535,10 +526,10 @@ class Connection { if (code >= 400) { this.last_reject = `${code} ${messages.join(' ')}`; - if (this.deny_includes_uuid) { + if (cfg.uuid.deny_chars) { uuid = (this.transaction || this).uuid; - if (this.deny_includes_uuid > 1) { - uuid = uuid.substr(0, this.deny_includes_uuid); + if (cfg.uuid.deny_chars > 1) { + uuid = uuid.substr(0, cfg.uuid.deny_chars); } } } @@ -649,7 +640,7 @@ class Connection { } init_transaction (cb) { this.reset_transaction(() => { - this.transaction = trans.createTransaction(this.tran_uuid(), this.cfg); + this.transaction = trans.createTransaction(this.tran_uuid(), cfg); // Catch any errors from the message_stream this.transaction.message_stream.on('error', (err) => { this.logcrit(`message_stream error: ${err.message}`); @@ -792,19 +783,19 @@ class Connection { }); break; default: { - let greeting = config.get('smtpgreeting', 'list'); - if (greeting.length) { + let greeting = cfg.message.greeting; + if (greeting?.length) { // RFC5321 section 4.2 // Hostname/domain should appear after the 220 greeting[0] = `${this.local.host} ESMTP ${greeting[0]}`; - if (this.banner_includes_uuid) { - greeting[0] += ` (${this.uuid})`; + if (cfg.uuid.banner_chars) { + greeting[0] += ` (${this.uuid.substr(0, cfg.uuid.banner_chars)})`; } } else { greeting = `${this.local.host} ESMTP ${this.local.info} ready`; - if (this.banner_includes_uuid) { - greeting += ` (${this.uuid})`; + if (cfg.uuid.banner_chars) { + greeting += ` (${this.uuid.substr(0, cfg.uuid.banner_chars)})`; } } this.respond(220, msg || greeting); @@ -850,7 +841,7 @@ class Connection { default: // RFC5321 section 4.1.1.1 // Hostname/domain should appear after 250 - this.respond(250, `${this.local.host} Hello ${this.get_remote('host')}, ${this.ehlo_hello_message}`); + this.respond(250, `${this.local.host} Hello ${this.get_remote('host')}, ${cfg.message.helo}`); } } ehlo_respond (retval, msg) { @@ -883,16 +874,14 @@ class Connection { // Hostname/domain should appear after 250 const response = [ - `${this.local.host} Hello ${this.get_remote('host')}, ${this.ehlo_hello_message}`, + `${this.local.host} Hello ${this.get_remote('host')}, ${cfg.message.helo}`, "PIPELINING", "8BITMIME", ]; - if (this.cfg.main.smtputf8) { - response.push("SMTPUTF8"); - } + if (cfg.main.smtputf8) response.push("SMTPUTF8"); - response.push(`SIZE ${this.max_bytes}`); + response.push(`SIZE ${cfg.max.bytes}`); this.capabilities = response; @@ -905,7 +894,7 @@ class Connection { this.respond(250, this.capabilities); } quit_respond (retval, msg) { - this.respond(221, msg || `${this.local.host} ${this.connection_close_message}`, () => { + this.respond(221, msg || `${this.local.host} ${cfg.message.close}`, () => { this.disconnect(); }); } @@ -1314,7 +1303,7 @@ class Connection { let results; try { - results = rfc1869.parse('mail', line, this.cfg.main.strict_rfc1869 && !this.relaying); + results = rfc1869.parse('mail', line, (!this.relaying && cfg.main.strict_rfc1869)); } catch (err) { this.errors++; @@ -1356,7 +1345,7 @@ class Connection { // Handle SIZE extension if (params?.SIZE && params.SIZE > 0) { - if (this.max_bytes > 0 && params.SIZE > this.max_bytes) { + if (cfg.max.bytes > 0 && params.SIZE > cfg.max.bytes) { return this.respond(550, 'Message too big!'); } } @@ -1378,7 +1367,7 @@ class Connection { let results; try { - results = rfc1869.parse('rcpt', line, this.cfg.main.strict_rfc1869 && !this.relaying); + results = rfc1869.parse('rcpt', line, cfg.main.strict_rfc1869 && !this.relaying); } catch (err) { this.errors++; @@ -1512,7 +1501,7 @@ class Connection { return this.respond(503, "RCPT required first"); } - if (this.cfg.headers.add_received) { + if (cfg.headers.add_received) { this.accumulate_data(`Received: ${this.received_line()}\r\n`); } plugins.run_hooks('data', this); @@ -1577,11 +1566,11 @@ class Connection { } // Stop accumulating data as we're going to reject at dot. - if (this.max_bytes && this.transaction.data_bytes > this.max_bytes) { + if (cfg.max.bytes && this.transaction.data_bytes > cfg.max.bytes) { return; } - if (this.transaction.mime_part_count >= this.max_mime_parts) { + if (this.transaction.mime_part_count >= cfg.max.mime_parts) { this.logcrit("Possible DoS attempt - too many MIME parts"); this.respond(554, "Transaction failed due to too many MIME parts", () => { this.disconnect(); @@ -1596,13 +1585,13 @@ class Connection { this.totalbytes += this.transaction.data_bytes; // Check message size limit - if (this.max_bytes && this.transaction.data_bytes > this.max_bytes) { - this.lognotice(`Incoming message exceeded databytes size of ${this.max_bytes}`); + if (cfg.max.bytes && this.transaction.data_bytes > cfg.max.bytes) { + this.lognotice(`Incoming message exceeded max size of ${cfg.max.bytes}`); return plugins.run_hooks('max_data_exceeded', this); } // Check max received headers count - if (this.transaction.header.get_all('received').length > this.cfg.headers.max_received) { + if (this.transaction.header.get_all('received').length > cfg.headers.max_received) { this.logerror("Incoming message had too many Received headers"); this.respond(550, "Too many received headers - possible mail loop", () => { this.reset_transaction(); @@ -1611,11 +1600,11 @@ class Connection { } // Warn if we hit the maximum parsed header lines limit - if (this.transaction.header_lines.length >= this.cfg.headers.max_lines) { - this.logwarn(`Incoming message reached maximum parsing limit of ${this.cfg.headers.max_lines} header lines`); + if (this.transaction.header_lines.length >= cfg.headers.max_lines) { + this.logwarn(`Incoming message reached maximum parsing limit of ${cfg.headers.max_lines} header lines`); } - if (this.cfg.headers.clean_auth_results) { + if (cfg.headers.clean_auth_results) { this.auth_results_clean(); // rename old A-R headers } const ar_field = this.auth_results(); // assemble new one diff --git a/docs/CoreConfig.md b/docs/CoreConfig.md index b09356004..81552ede4 100644 --- a/docs/CoreConfig.md +++ b/docs/CoreConfig.md @@ -11,10 +11,6 @@ If either of these files exist then they are loaded first. This file is designed to use the JSON/YAML file overrides documented in [haraka-config](https://github.com/haraka/haraka-config) to optionally provide the entire configuration in a single file. -* databytes - -Contains the maximum SIZE of an email that Haraka will receive. - * plugins The list of plugins to load @@ -40,35 +36,29 @@ The list of plugins to load specify -1 to disable spooling completely or 0 to force all messages to be spooled to disk. * graceful\_shutdown - (default: false) enable this to wait for sockets on shutdown instead of closing them quickly * force_shutdown_timeout - (default: 30) number of seconds to wait for a graceful shutdown - * smtputf8 - (default: true) advertise support for SMTPUTF8 - * strict\_rfc1869 - (default: false) Requires senders to conform to RFC 1869 and RFC 821 when sending the MAIL FROM and RCPT TO commands. In particular, - the inclusion of spurious spaces or missing angle brackets will be rejected. * me A name to use for this server. Used in received lines and elsewhere. Setup by default to be your hostname. -* deny\_includes\_uuid - - Each connection and mail in Haraka includes a UUID which is also in most log - messages. If you put a `1` in this file then every denied mail (either via - DENY/5xx or DENYSOFT/4xx return codes) will include the uuid at the start - of each line of the deny message in brackets, making it easy to track - problems back to the logs. - - Because UUIDs are long, if you put a number greater than 1 in the config - file, it will be truncated to that length. We recommend a 6 as a good - balance of finding in the logs and not making lines too long. - -* banner\_includes\_uuid - - This will add the full UUID to the first line of the SMTP greeting banner. +* connection.ini -* early\_talker\_delay + See inline comments in connection.ini for the following settings: - If clients talk early we *punish* them with a delay of this many milliseconds - default: 1000. + * haproxy.hosts\_ipv4 + * haproxy.hosts\_ipv6 + * headers.\* + * max.bytes + * max.line\_length + * max.data\_line\_length + * max.mime\_parts + * message.greeting + * message.close + * smtputf8 + * strict\_rfc1869 + * uuid.deny\_chars + * uuid.banner\_bytes * plugin\_timeout @@ -81,59 +71,8 @@ The list of plugins to load If the plugin is in a sub-directory of plugins, then you must create this file in the equivalent path e.g. the queue/smtp_forward would need a timeout file in `config/queue/smtp_forward.timeout` -* smtpgreeting - - The greeting line used when a client connects. This can be multiple lines - if required (this may cause some connecting machines to fail - though - usually only spam-bots). - -* max\_received\_count - - The maximum number of "Received" headers allowed in an email. This is a - simple protection against mail loops. Defaults to 100. - -* max\_line\_length - - The maximum length of lines in SMTP session commands (e.g. RCPT, HELO etc). - Defaults to 512 (bytes) which is mandated by RFC 5321 §4.5.3.1.4. Clients - exceeding this limit will be immediately disconnected with a "521 Command - line too long" error. - -* max\_data\_line\_length - - The maximum length of lines in the DATA section of emails. Defaults to 992 - (bytes) which is the limit set by Sendmail. When this limit is exceeded the - three bytes "\r\n " (0x0d 0x0a 0x20) are inserted into the stream to "fix" - it. This has the potential to "break" some email, but makes it more likely - to be accepted by upstream/downstream services, and is the same behaviour - as Sendmail. Also when the data line length limit is exceeded - `transaction.notes.data_line_length_exceeded` is set to `true`. - -* outbound.concurrency\_max - - Maximum concurrency to use when delivering mails outbound. Defaults to 100. - -* outbound.disabled - - Put a `1` in this file to temporarily disable outbound delivery. Useful - while figuring out network issues or testing. +* outbound.ini * outbound.bounce\_message - The bounce message if delivery of the message fails. The default is normally fine. Bounce messages contain a number of template - replacement values which are best discovered by looking at the source code. - -* haproxy\_hosts - - A list of HAProxy hosts that Haraka should enable the PROXY protocol from. - See [HAProxy.md](HAProxy.md) - -* max_mime_parts - - Defaults to 1000. There's a potential denial of service in large numbers of - MIME parts in carefully crafted emails. If this limit is too low for some - reason you can increase it by setting a value in this file. - -* connection\_close\_message - - Defaults to `closing connection. Have a jolly good day.` can be overrridden with custom text + The bounce message if delivery of the message fails. The default is normally fine. Bounce messages contain a number of template replacement values which are best discovered by looking at the source code. diff --git a/docs/HAProxy.md b/docs/HAProxy.md index 6f932c3e4..eb214b927 100644 --- a/docs/HAProxy.md +++ b/docs/HAProxy.md @@ -1,34 +1,20 @@ -HAProxy PROXY protocol extension support -======================================== +# HAProxy PROXY protocol extension support -Haraka natively supports the PROXY protocol [1]. +Haraka supports PROXY protocol [1]. -This allows an upstream proxy to pass IP address and port of the client which -Haraka will use instead of the socket IP address (which is of the proxy). -This allows DNSBLs and access control lists to operate on the proxied address. +This allows an upstream proxy to pass the IP address and port of the remote client. Haraka will use the remote IP instead of the socket IP address (which is the proxy). This allows DNSBLs and access control lists to use the correct source address. -Support is disabled by default and if HAProxy or other attempts to send a -PROXY command then Haraka will return a DENYSOFTDISCONNECT error. -DENYSOFT is used to prevent configuration errors from rejecting valid mail. +Support is disabled by default. Attempts to send a PROXY command will return a DENYSOFTDISCONNECT error. DENYSOFT is used to prevent configuration errors from rejecting valid mail. -To enable support for PROXY you must create a `haproxy_hosts` configuration -file which should contain a list of IP addresses of the HAProxy hosts -that should be allowed to send the PROXY command. A range of IP -addresses can be specified by it's CIDR network address. +To enable support for PROXY you must populate connection.ini[haproxy]hosts[] with the IP addresses of the HAProxy hosts that MUST send the PROXY command. Ranges can be specified with CIDR notation. -When a host connects to Haraka that matches an IP address present in the -`haproxy_hosts` file - a banner is not sent, instead Haraka waits for the -PROXY command to be sent before proceeding. The connection will timeout -with `421 PROXY timed out` if the command is not sent within 30 seconds. +When a proxy host connects to Haraka, a banner is not sent. Instead Haraka awaits the PROXY command. The connection will timeout with `421 PROXY timed out` if the command is not sent within 30 seconds. -NOTE: because Haraka does not send a banner when a listed HAProxy host -connects you must set check-send-proxy to ensure that the service checks -send a PROXY command before they run. +NOTE: because Haraka does not send a banner when a listed HAProxy host connects you must set check-send-proxy to ensure that the service checks send a PROXY command before they run. [1] http://haproxy.1wt.eu/download/1.5/doc/proxy-protocol.txt -HAProxy supports the PROXY protocol in version 1.5 or later however there -are patches available to add support for 1.4. +HAProxy supports the PROXY protocol in version 1.5 or later. Here is an example listener section for haproxy.cfg: @@ -45,13 +31,9 @@ listen smtp :25 server smtp5 ip.of.haraka.server5:25 check-send-proxy check inter 10s send-proxy ``` -The important part is `send-proxy` which causes HAProxy to send the PROXY -extension on connection. +The important part is `send-proxy` which causes HAProxy to send the PROXY extension on connection. -When using `option smtpchk` you will see CONNRESET errors reported in the Haraka logs as -smtpchk drops the connection before the HELO response is still being written. -You can use the `option tcp-check` instead to provide a better service check by having -the check wait for the banner, send QUIT and then check the response: +When using `option smtpchk` you will see CONNRESET errors reported in the Haraka logs as smtpchk drops the connection before the HELO response is still being written. You can use the `option tcp-check` instead to provide a better service check by having the check wait for the banner, send QUIT and then check the response: ``` option tcp-check diff --git a/plugins/early_talker.js b/plugins/early_talker.js index 4ccfd5e36..9f668106f 100644 --- a/plugins/early_talker.js +++ b/plugins/early_talker.js @@ -28,11 +28,6 @@ exports.load_config = function () { this.pause = this.cfg.main.pause * 1000; return; } - - // config/early_talker.pause is in milliseconds - this.pause = this.config.get('early_talker.pause', () => { - this.load_config(); - }); } exports.early_talker = function (next, connection) { diff --git a/server.js b/server.js index 9870795fa..e5d910ba9 100644 --- a/server.js +++ b/server.js @@ -32,12 +32,7 @@ Server.load_smtp_ini = () => { Server.cfg = Server.config.get('smtp.ini', { booleans: [ '-main.daemonize', - '-main.strict_rfc1869', - '+main.smtputf8', '-main.graceful_shutdown', - '+headers.add_received', - '+headers.show_version', - '+headers.clean_auth_results', ], }, () => { Server.load_smtp_ini(); @@ -56,15 +51,6 @@ Server.load_smtp_ini = () => { nodes: 1, }; - Server.cfg.headers.max_received = parseInt(Server.cfg.headers.max_received) || parseInt(Server.config.get('max_received_count')) || 100; - Server.cfg.headers.max_lines = parseInt(Server.cfg.headers.max_lines) || parseInt(Server.config.get('max_header_lines')) || 1000; - - const strict_ext = Server.config.get('strict_rfc1869'); - if (Server.cfg.main.strict_rfc1869 === false && strict_ext) { - Server.logwarn(`legacy config config/strict_rfc1869 is overriding smtp.ini`) - Server.cfg.main.strict_rfc1869 = strict_ext; - } - for (const key in defaults) { if (Server.cfg.main[key] !== undefined) continue; Server.cfg.main[key] = defaults[key]; diff --git a/test/connection.js b/test/connection.js index 9d4efe5ee..0dc86776b 100644 --- a/test/connection.js +++ b/test/connection.js @@ -7,7 +7,7 @@ const DSN = require('haraka-dsn') const connection = require('../connection'); const Server = require('../server'); -// huge hack here, but plugin tests need constants +// hack alert, but plugin tests need constants constants.import(global); const _set_up = (done) => {