From ccfa48ee3c9d17fa4997e19df2f9c790755a22ea Mon Sep 17 00:00:00 2001
From: ellie timoney <ellie@fastmail.com>
Date: Fri, 27 Aug 2021 16:05:30 +1000
Subject: [PATCH] Config: add understanding of imapd.conf BITFIELDs

This will let us treat 'httpmodules' et al as the set of flags it is,
rather than an opaque string
---
 Cassandane/Config.pm      | 163 +++++++++++++++++++++++++--
 Cassandane/Instance.pm    |  15 +--
 Cassandane/Test/Config.pm | 231 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 384 insertions(+), 25 deletions(-)

diff --git a/Cassandane/Config.pm b/Cassandane/Config.pm
index 6ccfde23f..6de770ea3 100644
--- a/Cassandane/Config.pm
+++ b/Cassandane/Config.pm
@@ -47,16 +47,45 @@ use Cassandane::Util::Log;
 
 my $default;
 
+# XXX Manually entered from lib/imapoptions in cyrus-imapd repo.
+# XXX Once these repositories are merged, we'll be able to automate keeping
+# XXX this synchronised...
+my %bitfields = (
+    'calendar_component_set' => 'VEVENT VTODO VJOURNAL VFREEBUSY VAVAILABILITY VPOLL',
+    'event_extra_params' => 'bodyStructure clientAddress diskUsed flagNames messageContent messageSize messages modseq service timestamp uidnext vnd.cmu.midset vnd.cmu.unseenMessages vnd.cmu.envelope vnd.cmu.sessionId vnd.cmu.mailboxACL vnd.cmu.mbtype vnd.cmu.davFilename vnd.cmu.davUid vnd.fastmail.clientId vnd.fastmail.sessionId vnd.fastmail.convExists vnd.fastmail.convUnseen vnd.fastmail.cid vnd.fastmail.counters vnd.fastmail.jmapEmail vnd.fastmail.jmapStates vnd.cmu.emailid vnd.cmu.threadid',
+    'event_groups' => 'message quota flags access mailbox subscription calendar applepushservice',
+    'httpmodules' => 'admin caldav carddav cgi domainkey freebusy ischedule jmap prometheus rss tzdist webdav',
+    'metapartition_files' => 'header index cache expunge squat annotations lock dav archivecache',
+    'newsaddheaders' => 'to replyto',
+    'sieve_extensions' => 'fileinto reject vacation vacation-seconds notify include envelope environment body relational regex subaddress copy date index imap4flags imapflags mailbox mboxmetadata servermetadata variables editheader extlists duplicate ihave fcc special-use redirect-dsn redirect-deliverby mailboxid vnd.cyrus.log x-cyrus-log vnd.cyrus.jmapquery x-cyrus-jmapquery snooze vnd.cyrus.snooze x-cyrus-snooze',
+);
+my $bitfields_fixed = 0;
+
 sub new
 {
     my $class = shift;
+
+    if (!$bitfields_fixed) {
+        while (my ($key, $allvalues) = each %bitfields) {
+            $bitfields{$key} = {};
+            foreach my $v (split /\s/, $allvalues) {
+                $bitfields{$key}->{$v} = 1;
+            }
+        }
+        $bitfields_fixed = 1;
+    }
+
     my $self = {
         parent => undef,
         variables => {},
-        params => { @_ },
+        params => {},
     };
 
     bless $self, $class;
+
+    # any arguments are initial params, process them properly
+    $self->set(@_);
+
     return $self;
 }
 
@@ -105,22 +134,121 @@ sub clone
     return $child;
 }
 
+sub _explode_bit_string
+{
+    my ($s) = @_;
+    return split /[_ ]/, $s;
+}
+
 sub set
 {
     my ($self, %nv) = @_;
     while (my ($n, $v) = each %nv)
     {
-        $self->{params}->{$n} = $v;
+        if (exists $bitfields{$n}) {
+            # it's a bitfield, set exactly what's given (clearing others)
+            if (ref $v eq 'ARRAY') {
+                $self->clear_all_bits($n);
+                $self->set_bits($n, @{$v});
+            }
+            elsif (ref $v eq q{}) {
+                $self->clear_all_bits($n);
+                $self->set_bits($n, _explode_bit_string($v));
+            }
+            else {
+                die "don't know what to do with value '$v'";
+            }
+        }
+        else {
+            $self->{params}->{$n} = $v;
+        }
+    }
+}
+
+sub set_bits
+{
+    my ($self, $name, @bits) = @_;
+
+    die "$name is not a bitfield option" if not exists $bitfields{$name};
+
+    # explode space- or underscore-delimited list as only bit
+    if (scalar @bits == 1 && $bits[0] =~ m/[_ ]/) {
+        @bits = _explode_bit_string($bits[0]);
+    }
+
+    foreach my $bit (@bits) {
+        die "$bit is not a $name value"
+            if not exists $bitfields{$name}->{$bit};
+
+        $self->{params}->{$name}->{$bit} = 1;
+    }
+}
+
+sub clear_bits
+{
+    my ($self, $name, @bits) = @_;
+
+    die "$name is not a bitfield option" if not exists $bitfields{$name};
+
+    # explode space- or underscore-delimited list as only bit
+    if (scalar @bits == 1 && $bits[0] =~ m/[_ ]/) {
+        @bits = _explode_bit_string($bits[0]);
+    }
+
+    foreach my $bit (@bits) {
+        die "$bit is not a $name value"
+            if not exists $bitfields{$name}->{$bit};
+
+        $self->{params}->{$name}->{$bit} = 0;
     }
 }
 
+sub clear_all_bits
+{
+    my ($self, $name) = @_;
+
+    die "$name is not a bitfield option" if not exists $bitfields{$name};
+
+    $self->{params}->{$name}->{$_} = 0 for keys %{$bitfields{$name}};
+}
+
 sub get
 {
     my ($self, $n) = @_;
-    while (defined $self)
-    {
-        return $self->{params}->{$n}
-            if exists $self->{params}->{$n};
+    if (exists $bitfields{$n}) {
+        my %bits;
+        while (defined $self) {
+            if (exists $self->{params}->{$n}) {
+                while (my ($bit, $val) = each %{$self->{params}->{$n}}) {
+                    $bits{$bit} //= $val;
+                }
+            }
+            $self = $self->{parent};
+        }
+        my @v = grep { $bits{$_} } sort keys %bits;
+        return wantarray ? @v : join q{ }, @v;
+    }
+    else {
+        while (defined $self)
+        {
+            return $self->{params}->{$n}
+                if exists $self->{params}->{$n};
+            $self = $self->{parent};
+        }
+    }
+    return undef;
+}
+
+sub get_bit
+{
+    my ($self, $name, $bit) = @_;
+
+    die "$name is not a bitfield option" if not exists $bitfields{$name};
+    die "$bit is not a $name value" if not exists $bitfields{$name}->{$bit};
+
+    while (defined $self) {
+        return $self->{params}->{$name}->{$bit}
+            if exists $self->{params}->{$name}->{$bit};
         $self = $self->{parent};
     }
     return undef;
@@ -129,6 +257,9 @@ sub get
 sub get_bool
 {
     my ($self, $n, $def) = @_;
+
+    die "bitfield $n cannot be boolean" if exists $bitfields{$n};
+
     $def = 'no' if !defined $def;
     my $v = $self->get($n);
     $v = $def if !defined $v;
@@ -199,8 +330,16 @@ sub _flatten
     {
         foreach my $n (keys %{$conf->{params}})
         {
-            $nv{$n} = $self->substitute($conf->{params}->{$n})
-                unless exists $nv{$n};
+            if (exists $bitfields{$n}) {
+                # no variable substitution on bitfields
+                while (my ($bit, $val) = each %{$conf->{params}->{$n}}) {
+                    $nv{$n}->{$bit} //= $val;
+                }
+            }
+            else {
+                $nv{$n} = $self->substitute($conf->{params}->{$n})
+                    unless exists $nv{$n};
+            }
         }
     }
     return \%nv;
@@ -216,7 +355,13 @@ sub generate
     while (my ($n, $v) = each %$nv)
     {
         next unless defined $v;
-        print CONF "$n: $v\n";
+        if (exists $bitfields{$n}) {
+            my @bits = grep { $nv->{$n}->{$_} } sort keys %{$nv->{$n}};
+            print CONF "$n: " . join(q{ }, @bits) . "\n";
+        }
+        else {
+            print CONF "$n: $v\n";
+        }
     }
     close CONF;
 }
diff --git a/Cassandane/Instance.pm b/Cassandane/Instance.pm
index 34addea72..9e087be1c 100644
--- a/Cassandane/Instance.pm
+++ b/Cassandane/Instance.pm
@@ -651,12 +651,9 @@ sub _generate_imapd_conf
         event_notifier => 'pusher',
     );
     if ($cyrus_major_version >= 3) {
-        my $event_groups = $self->{config}->get('event_groups') || '';
-        $event_groups .= ' mailbox message flags calendar';
-        $self->{config}->set(
-            imipnotifier => 'imip',
-            event_groups => $event_groups,
-        );
+        $self->{config}->set(imipnotifier => 'imip');
+        $self->{config}->set_bits('event_groups',
+                                  'mailbox message flags calendar');
 
         if ($cyrus_major_version > 3 || $cyrus_minor_version >= 1) {
             $self->{config}->set(
@@ -666,11 +663,7 @@ sub _generate_imapd_conf
         }
     }
     else {
-        my $event_groups = $self->{config}->get('event_groups') || '';
-        $event_groups .= ' mailbox message flags';
-        $self->{config}->set(
-            event_groups => 'mailbox message flags',
-        );
+        $self->{config}->set_bits('event_groups', 'mailbox message flags');
     }
     if ($self->{buildinfo}->get('search', 'xapian')) {
         my %xapian_defaults = (
diff --git a/Cassandane/Test/Config.pm b/Cassandane/Test/Config.pm
index c99838163..2fff79b32 100644
--- a/Cassandane/Test/Config.pm
+++ b/Cassandane/Test/Config.pm
@@ -40,11 +40,13 @@
 package Cassandane::Test::Config;
 use strict;
 use warnings;
+use Data::Dumper;
 use File::Temp qw(tempfile);
 
 use lib '.';
 use base qw(Cassandane::Unit::TestCase);
 use Cassandane::Config;
+use Cassandane::Util::Log;
 
 sub new
 {
@@ -142,7 +144,7 @@ sub _generate_and_read
     while (<$fh>)
     {
         chomp;
-        my ($n, $v) = m/^([^:\s]+):\s*(\S+)$/;
+        my ($n, $v) = m/^([^:\s]+):\s*(.+)*$/;
         $self->assert(defined $v);
         $nv{$n} = $v;
     }
@@ -160,16 +162,22 @@ sub test_generate
     my $c = Cassandane::Config->new();
     $c->set(foo => 'bar');
     $c->set(quux => 'foonly');
+    $c->set('httpmodules', 'caldav jmap');
+    $c->set('event_groups', [qw(quota)]);
+
     my $c2 = $c->clone();
     $c2->set(hello => 'world');
     $c2->set(foo => 'baz');
+    $c2->set('event_groups', [qw(flags quota)]);
 
     my $nv = $self->_generate_and_read($c2);
 
-    $self->assert(scalar(keys(%$nv)) == 3);
-    $self->assert($nv->{foo} eq 'baz');
-    $self->assert($nv->{hello} eq 'world');
-    $self->assert($nv->{quux} eq 'foonly');
+    $self->assert_num_equals(5, scalar(keys(%$nv)));
+    $self->assert_str_equals('baz', $nv->{foo});
+    $self->assert_str_equals('world', $nv->{hello});
+    $self->assert_str_equals('foonly', $nv->{quux});
+    $self->assert_str_equals('caldav jmap', $nv->{httpmodules});
+    $self->assert_str_equals('flags quota', $nv->{event_groups});
 }
 
 sub test_variables
@@ -208,5 +216,218 @@ sub test_variables
     $self->assert_str_equals('foAnly', $nv->{quux});
 }
 
+sub test_bitfields
+{
+    my ($self) = @_;
+
+    my $c = Cassandane::Config->new();
+
+    # can set bitfields as space separated strings
+    $c->set('httpmodules' => 'caldav jmap');
+    # get in scalar context returns space separated string
+    $self->assert_str_equals('caldav jmap', scalar $c->get('httpmodules'));
+    # get in list context returns list
+    $self->assert_deep_equals([qw(caldav jmap)], [$c->get('httpmodules')]);
+
+    # can clear a whole bitfield
+    $c->clear_all_bits('httpmodules');
+    $self->assert_null($c->get('httpmodules'));
+
+    # can set bitfields as array reference
+    $c->set('httpmodules' => [qw(caldav jmap)]);
+    # get in scalar context returns space separated string
+    $self->assert_str_equals('caldav jmap', scalar $c->get('httpmodules'));
+    # get in list context returns list
+    $self->assert_deep_equals([qw(caldav jmap)], [$c->get('httpmodules')]);
+
+    # can clear one bit
+    $c->clear_bits('httpmodules', 'caldav');
+    $self->assert_str_equals('jmap', $c->get('httpmodules'));
+
+    # can set one bit
+    $c->set_bits('httpmodules', 'prometheus');
+    $self->assert_str_equals('jmap prometheus', scalar $c->get('httpmodules'));
+
+    # can get one bit
+    $self->assert($c->get_bit('httpmodules', 'prometheus'));
+    $self->assert($c->get_bit('httpmodules', 'jmap'));
+    # valid bits that aren't set are false
+    $self->assert(not $c->get_bit('httpmodules', 'caldav'));
+    $self->assert(not $c->get_bit('httpmodules', 'freebusy'));
+
+    # can set a few bits
+    $c->set_bits('httpmodules', 'caldav', 'carddav');
+    $self->assert_str_equals('caldav carddav jmap prometheus',
+                             scalar $c->get('httpmodules'));
+    $c->set_bits('httpmodules', 'ischedule rss');
+    $self->assert_str_equals('caldav carddav ischedule jmap prometheus rss',
+                             scalar $c->get('httpmodules'));
+    $c->set_bits('httpmodules', 'cgi_webdav');
+    $self->assert_str_equals('caldav carddav cgi ischedule jmap prometheus rss webdav',
+                             scalar $c->get('httpmodules'));
+
+    # can clear a few bits
+    $c->clear_bits('httpmodules', 'caldav', 'carddav');
+    $self->assert_str_equals('cgi ischedule jmap prometheus rss webdav',
+                             scalar $c->get('httpmodules'));
+    $c->clear_bits('httpmodules', 'ischedule rss');
+    $self->assert_str_equals('cgi jmap prometheus webdav',
+                             scalar $c->get('httpmodules'));
+    $c->clear_bits('httpmodules', 'cgi_webdav');
+    $self->assert_str_equals('jmap prometheus',
+                             scalar $c->get('httpmodules'));
+
+
+    # setting with set() should replace previous bit set
+    $c->set('httpmodules' => [qw(admin tzdist)]);
+    $self->assert(not $c->get_bit('httpmodules', 'prometheus'));
+    $self->assert(not $c->get_bit('httpmodules', 'jmap'));
+    $self->assert($c->get_bit('httpmodules', 'admin'));
+    $self->assert($c->get_bit('httpmodules', 'tzdist'));
+    $self->assert_str_equals('admin tzdist', scalar $c->get('httpmodules'));
+
+    # cannot set bits on non-bitfield options
+    eval {
+        $c->set_bits('conversations', 'irrelevant');
+    };
+    my $e = $@;
+    $self->assert_matches(qr{conversations is not a bitfield option}, $e);
+
+    # cannot set invalid bits on bitfield options
+    eval {
+        $c->set_bits('httpmodules', 'bogus');
+    };
+    $e = $@;
+    $self->assert_matches(qr{bogus is not a httpmodules value}, $e);
+
+    # cannot mix and match bits from other bitfields
+    eval {
+        $c->set_bits('httpmodules', 'VEVENT');
+    };
+    $e = $@;
+    $self->assert_matches(qr{VEVENT is not a httpmodules value}, $e);
+
+    # should be able to set valid bitfields in constructor
+    my $c2 = Cassandane::Config->new('foo' => 'bar',
+                                     'httpmodules' => 'caldav jmap',
+                                     'event_groups' => [qw(message quota)]);
+    $self->assert_not_null($c2);
+
+    # expectations should still hold for bitfields set via constructor
+    $self->assert_str_equals('bar', $c2->get('foo'));
+    $self->assert_str_equals('caldav jmap', scalar $c2->get('httpmodules'));
+    $self->assert_deep_equals([qw(caldav jmap)], [$c2->get('httpmodules')]);
+    $self->assert_str_equals('message quota', scalar $c2->get('event_groups'));
+    $self->assert_deep_equals([qw(message quota)], [$c2->get('event_groups')]);
+}
+
+sub test_clone_bitfields
+{
+    my ($self) = @_;
+
+    my $c = Cassandane::Config->new();
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_null($c->get('event_groups'));
+
+    my $c2 = $c->clone();
+    $self->assert($c2 ne $c);
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_null($c2->get('httpmodules'));
+    $self->assert_null($c->get('event_groups'));
+    $self->assert_null($c2->get('event_groups'));
+
+    # set bit in clone doesn't affect parent
+    $c2->set_bits('httpmodules', 'caldav');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_null($c->get('event_groups'));
+    $self->assert_null($c2->get('event_groups'));
+
+    # set bit in parent is inherited by child
+    $c->set_bits('event_groups', 'access', 'mailbox');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('access mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('access mailbox', scalar $c2->get('event_groups'));
+
+    # set bit in child supplements parent
+    $c2->set_bits('event_groups', 'quota');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('access mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('access mailbox quota', scalar $c2->get('event_groups'));
+
+    # clear bit in child overrides parent
+    $c2->clear_bits('event_groups', 'mailbox');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('access mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('access quota', scalar $c2->get('event_groups'));
+
+    # clear bit in parent updates inheriting child
+    $c->clear_bits('event_groups', 'access');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('quota', scalar $c2->get('event_groups'));
+
+    # clear bit in child updates child
+    $c2->clear_bits('event_groups', 'quota');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('caldav', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_null($c2->get('event_groups'));
+
+    # set explicit list in parent updates child
+    $c->set('httpmodules', 'jmap prometheus carddav');
+    $self->assert_str_equals('carddav jmap prometheus',
+                             scalar $c->get('httpmodules'));
+    $self->assert_str_equals('caldav carddav jmap prometheus',
+                             scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_null($c2->get('event_groups'));
+
+    # clear all in child overrides parent
+    $c2->clear_all_bits('httpmodules');
+    $self->assert_str_equals('carddav jmap prometheus',
+                             scalar $c->get('httpmodules'));
+    $self->assert_null($c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_null($c2->get('event_groups'));
+
+    # discard clone and recreate, clone should be the same as parent again
+    undef $c2;
+    $c2 = $c->clone();
+    $self->assert_not_equals($c, $c2);
+    $self->assert_equals(scalar $c->get('httpmodules'),
+                         scalar $c2->get('httpmodules'));
+    $self->assert_equals(scalar $c->get('event_groups'),
+                         scalar $c2->get('event_groups'));
+
+    # bit set in both parent and child is only listed once
+    $c2->set_bits('httpmodules', 'jmap');
+    $self->assert_str_equals('carddav jmap prometheus',
+                             scalar $c->get('httpmodules'));
+    $self->assert_str_equals('carddav jmap prometheus',
+                             scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('mailbox', scalar $c2->get('event_groups'));
+
+    # clearing bit in parent doesn't affect child who has it explicitly set
+    $c->clear_bits('httpmodules', 'jmap');
+    $self->assert_str_equals('carddav prometheus',
+                             scalar $c->get('httpmodules'));
+    $self->assert_str_equals('carddav jmap prometheus',
+                             scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('mailbox', scalar $c2->get('event_groups'));
+
+    # clearing all in parent doesn't affect child's explicit bits
+    $c->clear_all_bits('httpmodules');
+    $self->assert_null($c->get('httpmodules'));
+    $self->assert_str_equals('jmap', scalar $c2->get('httpmodules'));
+    $self->assert_str_equals('mailbox', scalar $c->get('event_groups'));
+    $self->assert_str_equals('mailbox', scalar $c2->get('event_groups'));
+}
 
 1;