Remove TODO comment; turned into a gitlab issue

multischleuder/api.py

@@ -57,31 +57,37 @@ class SchleuderApi:
 
     # List Management
 
-    def get_lists(self) -> List['SchleuderList']:
+    def get_lists(self) -> List[SchleuderList]:
         lists = self.__request('lists.json')
         return [SchleuderList.from_api(**s) for s in lists]
 
     # Subscriber Management
 
-    def get_subscribers(self, schleuder: 'SchleuderList') -> List['SchleuderSubscriber']:
+    def get_subscribers(self, schleuder: SchleuderList) -> List[SchleuderSubscriber]:
         response = self.__request('subscriptions.json', schleuder.id)
         subs: List[SchleuderSubscriber] = []
         for r in response:
+            key: Optional[SchleuderKey] = None
+            if r['fingerprint']:
                 key = self.get_key(r['fingerprint'], schleuder)
             sub = SchleuderSubscriber.from_api(key, **r)
             subs.append(sub)
         return subs
 
-    def get_subscriber(self, email: str, schleuder: 'SchleuderList') -> 'SchleuderSubscriber':
+    def get_subscriber(self, email: str, schleuder: SchleuderList) -> SchleuderSubscriber:
         response = self.__request('subscriptions.json', list_id=schleuder.id)
         for r in response:
             if r['email'] != email:
                 continue
+            key: Optional[SchleuderKey] = None
+            if r['fingerprint']:
                 key = self.get_key(r['fingerprint'], schleuder)
             return SchleuderSubscriber.from_api(key, **r)
         raise KeyError(f'{email} is not subscribed to {schleuder.name}')
 
-    def subscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'):
+    def subscribe(self, sub: SchleuderSubscriber, schleuder: SchleuderList):
+        if sub.key is None:
+            raise ValueError('Cannot subscribe without key')
         if self._dry_run:
             return
         data = json.dumps({
@@ -90,13 +96,15 @@ class SchleuderApi:
         })
         self.__request('subscriptions.json', list_id=schleuder.id, data=data, method='POST')
 
-    def unsubscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'):
+    def unsubscribe(self, sub: SchleuderSubscriber, schleuder: SchleuderList):
         listsub = self.get_subscriber(sub.email, schleuder)
         if self._dry_run:
             return
         self.__request('subscriptions/{}.json', fmt=[listsub.id], method='DELETE')
 
-    def update_fingerprint(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'):
+    def update_fingerprint(self, sub: SchleuderSubscriber, schleuder: SchleuderList):
+        if sub.key is None:
+            raise ValueError('Cannot update fingerprint without key')
         listsub = self.get_subscriber(sub.email, schleuder)
         if self._dry_run:
             return
@@ -105,13 +113,11 @@ class SchleuderApi:
 
     # Key Management
 
-    def get_key(self, fpr: str, schleuder: 'SchleuderList'):
+    def get_key(self, fpr: str, schleuder: SchleuderList) -> SchleuderKey:
-        if self._dry_run:
-            return
         key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
         return SchleuderKey.from_api(schleuder.id, **key)
 
-    def post_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
+    def post_key(self, key: SchleuderKey, schleuder: SchleuderList):
         if self._dry_run:
             return
         data = json.dumps({
@@ -119,7 +125,7 @@ class SchleuderApi:
         })
         self.__request('keys.json', list_id=schleuder.id, data=data, method='POST')
 
-    def delete_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
+    def delete_key(self, key: SchleuderKey, schleuder: SchleuderList):
         if self._dry_run:
             return
         self.__request('keys/{}.json', list_id=schleuder.id, fmt=[key.fingerprint], method='DELETE')

multischleuder/conflict.py

@@ -1,5 +1,5 @@
 
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import email.mime.application
 import email.mime.multipart
@@ -45,9 +45,10 @@ class ConflictMessage:
                              self._chosen.schleuder))
         # Include all subscriptions, including the FULL key
         for s in subs:
-            h.update(struct.pack('!ds',
+            key = b'N/A'
-                                 s.schleuder,
+            if s.key is not None:
-                                 s.key.blob.encode()))
+                key = s.key.blob.encode()
+            h.update(struct.pack('!ds', s.schleuder, key))
         return h.hexdigest()
 
     @property
@@ -57,10 +58,12 @@ class ConflictMessage:
     @property
     def mime(self) -> email.mime.multipart.MIMEMultipart:
         # Render the message body
-        _chosen = f'{self._chosen.key.fingerprint}  {self._chosen.email}'
+        fpr = 'N/A' if self._chosen.key is None else self._chosen.key.fingerprint
+        _chosen = f'{fpr}  {self._chosen.email}'
         _affected = ''
         for affected in self._affected:
-            _affected += f'{affected.key.fingerprint}  {affected.schleuder}\n'
+            fpr = 'N/A' if affected.key is None else affected.key.fingerprint
+            _affected += f'{fpr}  {affected.schleuder}\n'
         msg: str = self._template.format(
             subscriber=self._chosen.email,
             schleuder=self._schleuder,
@@ -73,6 +76,7 @@ class ConflictMessage:
         sessionkey = cipher.gen_key()
         try:
             for affected in self._affected:
+                if affected.key is not None:
                     key, _ = pgpy.PGPKey.from_blob(affected.key.blob)
                     key._require_usage_flags = False
                     pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
@@ -123,22 +127,26 @@ class KeyConflictResolution:
         for s in subscriptions:
             subs.setdefault(s.email, []).append(s)
         # Perform conflict resolution for each set of subscribers with the same email
-        return [self._resolve(target, mail_from, s) for s in subs.values()]
+        resolved = [self._resolve(target, mail_from, s) for s in subs.values()]
+        return [r for r in resolved if r is not None]
 
     def _resolve(self,
                  target: str,
                  mail_from: str,
-                 subscriptions: List[SchleuderSubscriber]) -> SchleuderSubscriber:
+                 subscriptions: List[SchleuderSubscriber]) -> Optional[SchleuderSubscriber]:
-        if len(subscriptions) == 1:
+        notnull = [s for s in subscriptions if s.key is not None]
-            return subscriptions[0]
+        if len(notnull) == 0:
-        if len({s.key.blob for s in subscriptions}) == 1:
+            return None
+        if len({s.key.blob for s in subscriptions if s.key is not None}) == 1:
             # No conflict if all keys are the same
-            return subscriptions[0]
+            return notnull[0]
-        # Conflict Resolution: Choose the OLDEST subscriptions, but notify using ALL keys
+        # Conflict Resolution: Choose the OLDEST subscription with a key, but notify using ALL keys
-        earliest: SchleuderSubscriber = min(subscriptions, key=lambda x: x.created_at)
+        earliest: SchleuderSubscriber = min(notnull, key=lambda x: x.created_at)
-        self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder}:')
+        assert earliest.key is not None  # Make mypy happy; it can't know that earliest.key can't be None
+        self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.key.fingerprint}:')
         for s in subscriptions:
-            self._logger.debug(f'  - {s.schleuder}: {s.key.fingerprint}')
+            fpr = 'N/A' if s.key is None else s.key.fingerprint
+            self._logger.debug(f'  - {s.schleuder}: {fpr}')
         # At this point, the messages are only queued locally, they will be sent afterwards
         msg = ConflictMessage(
             target,

multischleuder/processor.py

@@ -36,7 +36,7 @@ class MultiList:
             for s in self._api.get_subscribers(target_list)
             if s.email not in self._unmanaged
         }
-        current_keys: Set[SchleuderKey] = {s.key for s in current_subs}
+        current_keys: Set[SchleuderKey] = {s.key for s in current_subs if s.key is not None}
         all_subs: List[SchleuderSubscriber] = []
         # This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists ...
         for source in sources:
@@ -46,9 +46,9 @@ class MultiList:
                     continue
                 all_subs.append(s)
         # ... which is taken care of by the key conflict resolution routine
-        self._kcr.resolve(self._target, self._mail_from, all_subs)
+        resolved = self._kcr.resolve(self._target, self._mail_from, all_subs)
-        intended_subs: Set[SchleuderSubscriber] = set(all_subs)
+        intended_subs: Set[SchleuderSubscriber] = set(resolved)
-        intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs}
+        intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None}
         # Determine the change set
         to_subscribe = intended_subs.difference(current_subs)
         to_unsubscribe = current_subs.difference(intended_subs)

multischleuder/test/test_api.py

@@ -79,6 +79,21 @@ _SUBSCRIBER_RESPONSE = '''
 ]
 '''
 
+_SUBSCRIBER_RESPONSE_NOKEY = '''
+[
+  {
+    "id": 24,
+    "list_id": 42,
+    "email": "foo@example.org",
+    "fingerprint": "",
+    "admin": false,
+    "delivery_enabled": true,
+    "created_at": "2022-04-15T01:11:12.123Z",
+    "updated_at": "2022-04-15T01:11:12.123Z"
+  }
+]
+'''
+
 _KEY_RESPONSE = '''
 {
   "fingerprint": "2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6",
@@ -96,7 +111,7 @@ _KEY_RESPONSE = '''
 
 class TestSchleuderApi(unittest.TestCase):
 
-    def _mock_api(self, mock):
+    def _mock_api(self, mock, nokey=False):
         m = MagicMock()
         m.getcode.return_value = 200
 
@@ -105,6 +120,8 @@ class TestSchleuderApi(unittest.TestCase):
             if '/lists' in url:
                 return _LIST_RESPONSE.encode()
             if '/subscriptions' in url:
+                if nokey:
+                    return _SUBSCRIBER_RESPONSE_NOKEY.encode()
                 return _SUBSCRIBER_RESPONSE.encode()
             if '/keys' in url:
                 return _KEY_RESPONSE.encode()
@@ -147,6 +164,21 @@ class TestSchleuderApi(unittest.TestCase):
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', subs[0].key.blob)
         self.assertEqual(42, subs[0].key.schleuder)
 
+    @patch('urllib.request.urlopen')
+    def test_get_subscribers_nokey(self, mock):
+        api = self._mock_api(mock, nokey=True)
+        subs = api.get_subscribers(SchleuderList(42, '', ''))
+        # Test request data
+        self.assertEqual('https://localhost:4443/subscriptions.json?list_id=42',
+                         mock.call_args_list[0][0][0].get_full_url())
+        self.assertEqual(1, len(subs))
+        self.assertEqual(24, subs[0].id)
+        self.assertEqual('foo@example.org', subs[0].email)
+        self.assertEqual(42, subs[0].schleuder)
+        self.assertEqual(datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
+                         subs[0].created_at)
+        self.assertIsNone(subs[0].key)
+
     @patch('urllib.request.urlopen')
     def test_get_subscriber(self, mock):
         api = self._mock_api(mock)
@@ -159,6 +191,14 @@ class TestSchleuderApi(unittest.TestCase):
         with self.assertRaises(KeyError):
             api.get_subscriber('bar@example.org', SchleuderList(42, '', ''))
 
+    @patch('urllib.request.urlopen')
+    def test_get_subscriber_nokey(self, mock):
+        api = self._mock_api(mock, nokey=True)
+        sub = api.get_subscriber('foo@example.org', SchleuderList(42, '', ''))
+        self.assertEqual(24, sub.id)
+        self.assertEqual('foo@example.org', sub.email)
+        self.assertIsNone(sub.key)
+
     @patch('urllib.request.urlopen')
     def test_subscribe(self, mock):
         api = self._mock_api(mock)
@@ -171,6 +211,14 @@ class TestSchleuderApi(unittest.TestCase):
         self.assertEqual('POST', mock.call_args_list[-1][0][0].method)
         # todo assert request payload
 
+    @patch('urllib.request.urlopen')
+    def test_subscribe_nokey(self, mock):
+        api = self._mock_api(mock)
+        now = datetime.utcnow()
+        sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now)
+        with self.assertRaises(ValueError):
+            api.subscribe(sub, SchleuderList(42, '', ''))
+
     @patch('urllib.request.urlopen')
     def test_unsubscribe(self, mock):
         api = self._mock_api(mock)
@@ -195,6 +243,14 @@ class TestSchleuderApi(unittest.TestCase):
         self.assertEqual('PATCH', mock.call_args_list[-1][0][0].method)
         # todo assert request payload
 
+    @patch('urllib.request.urlopen')
+    def test_update_fingerprint_nokey(self, mock):
+        api = self._mock_api(mock)
+        now = datetime.utcnow()
+        sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now)
+        with self.assertRaises(ValueError):
+            api.update_fingerprint(sub, SchleuderList(42, '', ''))
+
     @patch('urllib.request.urlopen')
     def test_get_key(self, mock):
         api = self._mock_api(mock)
@@ -234,13 +290,13 @@ class TestSchleuderApi(unittest.TestCase):
         key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
         sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now)
         sch = SchleuderList(42, '', '')
-        # create, update, delete should be no-ops; 2 requests for retrieving the subscriptions in unsub & update
+        # create, update, delete should be no-ops; 5 requests for retrieving the keys & subscriptions in unsub & update
         api.subscribe(sub, sch)
         api.unsubscribe(sub, sch)
         api.update_fingerprint(sub, sch)
         api.post_key(key, SchleuderList(42, '', ''))
         api.delete_key(key, SchleuderList(42, '', ''))
-        self.assertGreater(3, len(mock.call_args_list))
+        self.assertGreater(5, len(mock.call_args_list))
         # only reads should execute
         api.get_lists()
         api.get_subscribers(sch)

multischleuder/test/test_conflict.py

@@ -182,6 +182,40 @@ class TestKeyConflictResolution(unittest.TestCase):
         self.assertEqual('test@schleuder.example.org', payload['schleuder'])
         self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4  foo@example.org', payload['chosen'])
 
+    def test_conflict_one_nullkey(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+
+        # This subscription has no key, it it will be discarded
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', None, sch2.id, date2)
+
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+
+        self.assertEqual(1, len(resolved))
+        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
+        # Null key should not trigger a confict
+        self.assertEqual(0, len(kcr._messages))
+
+    def test_conflict_only_nullkeys(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', None, sch1.id, date1)
+
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1])
+        self.assertEqual(0, len(resolved))
+
     def test_send_messages_nostate(self):
         sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
         key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
@@ -326,3 +360,34 @@ class TestKeyConflictResolution(unittest.TestCase):
         mock_smtp.send_message.assert_not_called()
         contents.seek(0)
         self.assertEqual(_CONFLICT_STATE_STALE, contents.read())
+
+    def test_send_messages_nullkey(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+        # This subscription does not have a key, so it will not be considered
+        sch3 = SchleuderList(7, 'test-east@schleuder.example.org', '36C545D9C88AD0E7EFEF18DE06ACC53D5820EBC0')
+        date3 = datetime(2022, 4, 11, 5, 23, 42, 0, tzinfo=tzutc())
+        sub3 = SchleuderSubscriber(7, 'foo@example.org', None, sch3.id, date3)
+
+        mock_smtp = MagicMock()
+        mock_smtp.__enter__.return_value = mock_smtp
+        kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2, sub3])
+        self.assertEqual(1, len(kcr._messages))
+        msg = kcr._messages[0].mime
+        pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))
+        # Verify that the message is encrypted with both keys
+        self.assertEqual(2, len(pgp.encrypters))
+        dec1 = _PRIVKEY_1.decrypt(pgp)
+        dec2 = _PRIVKEY_2.decrypt(pgp)
+        self.assertEqual(dec1.message, dec2.message)

multischleuder/test/test_multilist.py

@@ -0,0 +1,210 @@
+
+from typing import Dict, List
+
+import unittest
+from datetime import datetime
+from unittest.mock import MagicMock
+
+from dateutil.tz import tzutc
+
+from multischleuder.processor import MultiList
+from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
+
+
+def _resolve(target: str,
+             mail_from: str,
+             subscriptions: List[SchleuderSubscriber]) -> List[SchleuderSubscriber]:
+    # Mock conflict resolution that does not send or prepare messages
+    subs: Dict[str, List[SchleuderSubscriber]] = {}
+    for s in subscriptions:
+        subs.setdefault(s.email, []).append(s)
+    return [min(s, key=lambda x: x.created_at) for s in subs.values()]
+
+
+def _list_lists():
+    return [
+        SchleuderList(1, 'test@schleuder.example.org', '0CD581127A178F1931909B14B7CC98DA082D2A64'),
+        SchleuderList(2, 'test-global@schleuder.example.org', 'F60B441065D485224E0F53EBBEAD4E67512CB645'),
+        SchleuderList(3, 'test-north@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'),
+        SchleuderList(4, 'test-south@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'),
+        SchleuderList(5, 'test-east@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'),
+        SchleuderList(6, 'test-west@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'),
+        SchleuderList(7, 'test2-global@schleuder.example.org', 'A821095FB503836134D54138D3306C629C0DDFB5')
+    ]
+
+
+def _get_key(fpr: str, schleuder: SchleuderList):
+    key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
+                        'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
+    key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
+                        'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
+    key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',
+                        'alex.example@example.org', 'BEGIN PGP 716DEC79', schleuder.id)
+    key4 = SchleuderKey('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6',
+                        'aspammer@example.org', 'BEGIN PGP 045AE2D6', schleuder.id)
+    key5 = SchleuderKey('3699B7549B2A78F27AC93420FB6AD038C95558E5',
+                        'anna.example@example.org', 'BEGIN PGP C95558E5', schleuder.id)
+    key6 = SchleuderKey('698118749D1490C9822FA1D9600E77BC529681F9',
+                        'anna.example@example.org', 'BEGIN PGP 529681F9', schleuder.id)
+    key7 = SchleuderKey('BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3',
+                        'anotherspammer@example.org', 'BEGIN PGP 9233D3E3', schleuder.id)
+    key8 = SchleuderKey('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9',
+                        'andy.example@example.org', 'BEGIN PGP 8C89A2B9', schleuder.id)
+    key9 = SchleuderKey('C8C77172C31A796C563920C9358EBEA0F7662478',
+                        'aaron.example@example.org', 'BEGIN PGP F7662478', schleuder.id)
+    return {
+        '966842467B3254143F994D5E5C408C012D216471': key1,
+        '6449FFB6EE68187962FA013B5CA2F4F51791BAF6': key2,
+        '414D3960D34730F63C74D5190EBC5A16716DEC79': key3,
+        '8258FAF8B161B3DD8F784874F73E2DDF045AE2D6': key4,
+        '3699B7549B2A78F27AC93420FB6AD038C95558E5': key5,
+        '698118749D1490C9822FA1D9600E77BC529681F9': key6,
+        'BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3': key7,
+        '73E6C1170DA158D0139EF1F2349F17BB8C89A2B9': key8,
+        'C8C77172C31A796C563920C9358EBEA0F7662478': key9,
+    }[fpr]
+
+
+def _get_subs(schleuder: SchleuderList):
+    key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
+                        'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
+    key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
+                        'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
+    key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',
+                        'alex.example@example.org', 'BEGIN PGP 716DEC79', schleuder.id)
+    key4 = SchleuderKey('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6',
+                        'aspammer@example.org', 'BEGIN PGP 045AE2D6', schleuder.id)
+    key5 = SchleuderKey('3699B7549B2A78F27AC93420FB6AD038C95558E5',
+                        'anna.example@example.org', 'BEGIN PGP C95558E5', schleuder.id)
+    key6 = SchleuderKey('698118749D1490C9822FA1D9600E77BC529681F9',
+                        'anna.example@example.org', 'BEGIN PGP 529681F9', schleuder.id)
+    key7 = SchleuderKey('BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3',
+                        'anotherspammer@example.org', 'BEGIN PGP 9233D3E3', schleuder.id)
+    key8 = SchleuderKey('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9',
+                        'andy.example@example.org', 'BEGIN PGP 8C89A2B9', schleuder.id)
+    key9 = SchleuderKey('C8C77172C31A796C563920C9358EBEA0F7662478',
+                        'aaron.example@example.org', 'BEGIN PGP F7662478', schleuder.id)
+    date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+    date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+    if schleuder.id == 2:
+        return [
+            # admin should be ignored (unmanaged)
+            SchleuderSubscriber(0, 'admin@example.org', key1, schleuder.id, date1),
+            # ada.lovelace should be unsubscribed
+            SchleuderSubscriber(1, 'ada.lovelace@example.org', key2, schleuder.id, date1),
+            # alex.example already subscribed - no change
+            SchleuderSubscriber(2, 'alex.example@example.org', key3, schleuder.id, date1),
+            # aspammer should be unsubscribed (banned)
+            SchleuderSubscriber(3, 'aspammer@example.org', key4, schleuder.id, date1),
+            # anna.example's key should be updated, but the subscrption should stay unchanged
+            SchleuderSubscriber(4, 'anna.example@example.org', key5, schleuder.id, date1),
+        ]
+    elif schleuder.id == 3:
+        return [
+            SchleuderSubscriber(5, 'alex.example@example.org', key3, schleuder.id, date1),
+            SchleuderSubscriber(6, 'aspammer@example.org', key4, schleuder.id, date1),
+        ]
+    elif schleuder.id == 4:
+        return [
+            SchleuderSubscriber(7, 'anna.example@example.org', key6, schleuder.id, date1),
+            SchleuderSubscriber(8, 'anotherspammer@example.org', key7, schleuder.id, date1),
+        ]
+    elif schleuder.id == 5:
+        return [
+            SchleuderSubscriber(9, 'andy.example@example.org', key8, schleuder.id, date1),
+        ]
+    elif schleuder.id == 6:
+        return [
+            SchleuderSubscriber(10, 'aaron.example@example.org', key9, schleuder.id, date2),
+        ]
+    else:
+        return []
+
+
+def _get_sub(email: str, schleuder: SchleuderList):
+    subs = _get_subs(schleuder)
+    return [s for s in subs if s.email == email][0]
+
+
+class TestMultiList(unittest.TestCase):
+
+    def _kcr_mock(self):
+        kcr = MagicMock()
+        kcr.resolve = _resolve
+        return kcr
+
+    def _api_mock(self):
+        api = MagicMock()
+        api.get_lists = _list_lists
+        api.get_subscribers = _get_subs
+        api.get_subscriber = _get_sub
+        api.get_key = _get_key
+        return api
+
+    def test_create(self):
+        sources = [
+            'test-north@schleuder.example.org',
+            'test-east@schleuder.example.org',
+            'test-south@schleuder.example.org',
+            'test-west@schleuder.example.org'
+        ]
+        api = self._api_mock()
+        ml = MultiList(sources=sources,
+                       target='test-global@schleuder.example.org',
+                       unmanaged=['admin@example.org'],
+                       banned=['aspammer@example.org', 'anotherspammer@example.org'],
+                       mail_from='test-global-owner@schleuder.example.org',
+                       api=api,
+                       kcr=self._kcr_mock())
+        ml.process()
+
+        # Key uploads
+        self.assertEqual(3, len(api.post_key.call_args_list))
+        a, b, c = sorted([x[0] for x in api.post_key.call_args_list], key=lambda x: x[0].fingerprint)
+        self.assertEqual(2, a[1].id)
+        self.assertEqual('anna.example@example.org', a[0].email)
+        self.assertEqual('698118749D1490C9822FA1D9600E77BC529681F9', a[0].fingerprint)
+        self.assertEqual(2, b[1].id)
+        self.assertEqual('andy.example@example.org', b[0].email)
+        self.assertEqual('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', b[0].fingerprint)
+        self.assertEqual(2, c[1].id)
+        self.assertEqual('aaron.example@example.org', c[0].email)
+        self.assertEqual('C8C77172C31A796C563920C9358EBEA0F7662478', c[0].fingerprint)
+
+        # Subscriptions
+        self.assertEqual(2, len(api.subscribe.call_args_list))
+        a, b = sorted([x[0] for x in api.subscribe.call_args_list], key=lambda x: x[0].email)
+        self.assertEqual(2, a[1].id)
+        self.assertEqual('aaron.example@example.org', a[0].email)
+        self.assertEqual('C8C77172C31A796C563920C9358EBEA0F7662478', a[0].key.fingerprint)
+        self.assertEqual(2, b[1].id)
+        self.assertEqual('andy.example@example.org', b[0].email)
+        self.assertEqual('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', b[0].key.fingerprint)
+
+        # Key updates
+        self.assertEqual(1, len(api.update_fingerprint.call_args_list))
+        a = api.update_fingerprint.call_args_list[0][0]
+        self.assertEqual(2, a[1].id)
+        self.assertEqual('anna.example@example.org', a[0].email)
+        self.assertEqual('698118749D1490C9822FA1D9600E77BC529681F9', a[0].key.fingerprint)
+
+        # Unsubscribes
+        self.assertEqual(2, len(api.unsubscribe.call_args_list))
+        a, b = sorted([x[0] for x in api.unsubscribe.call_args_list], key=lambda x: x[0].email)
+        self.assertEqual(2, a[1].id)
+        self.assertEqual('ada.lovelace@example.org', a[0].email)
+        self.assertEqual(2, b[1].id)
+        self.assertEqual('aspammer@example.org', b[0].email)
+
+        # Key deletions
+        self.assertEqual(3, len(api.delete_key.call_args_list))
+        a, b, c = sorted([x[0] for x in api.delete_key.call_args_list], key=lambda x: x[0].fingerprint)
+        self.assertEqual(2, a[1].id)
+        self.assertEqual('anna.example@example.org', a[0].email)
+        self.assertEqual('3699B7549B2A78F27AC93420FB6AD038C95558E5', a[0].fingerprint)
+        self.assertEqual(2, b[1].id)
+        self.assertEqual('ada.lovelace@example.org', b[0].email)
+        self.assertEqual('6449FFB6EE68187962FA013B5CA2F4F51791BAF6', b[0].fingerprint)
+        self.assertEqual(2, c[1].id)
+        self.assertEqual('aspammer@example.org', c[0].email)
+        self.assertEqual('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', c[0].fingerprint)

multischleuder/test/test_types.py

@@ -35,3 +35,14 @@ class TestSchleuderTypes(unittest.TestCase):
         self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
                          s.created_at)
         self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', s.key.fingerprint)
+        self.assertIn('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6 (foo@example.org)', str(k))
+
+    def test_parse_subscriber_nokey(self):
+        s = SchleuderSubscriber.from_api(None, **json.loads(_SUBSCRIBER_RESPONSE)[0])
+        self.assertEqual(23, s.id)
+        self.assertEqual('foo@example.org', str(s))
+        self.assertEqual('foo@example.org', s.email)
+        self.assertEqual(42, s.schleuder)
+        self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
+                         s.created_at)
+        self.assertIsNone(s.key)

multischleuder/types.py

@@ -1,10 +1,12 @@
 
+from typing import Optional
+
 from dataclasses import dataclass, field, Field
 from datetime import datetime
 from dateutil.parser import isoparse
 
 
-@dataclass
+@dataclass(frozen=True)
 class SchleuderList:
     id: int = field(compare=True)
     name: str = field(compare=False)
@@ -18,16 +20,16 @@ class SchleuderList:
         return SchleuderList(id, email, fingerprint)
 
 
-@dataclass
+@dataclass(frozen=True)
 class SchleuderSubscriber:
     id: int = field(compare=False)
     email: str = field(compare=True)
-    key: 'SchleuderKey' = field(compare=False)
+    key: Optional['SchleuderKey'] = field(compare=False)
     schleuder: int = field(compare=False)
     created_at: datetime = field(compare=False)
 
     @staticmethod
-    def from_api(key: 'SchleuderKey',
+    def from_api(key: Optional['SchleuderKey'],
                  id: int,
                  list_id: int,
                  email: str,
@@ -40,7 +42,7 @@ class SchleuderSubscriber:
         return f'{self.email}'
 
 
-@dataclass
+@dataclass(frozen=True)
 class SchleuderKey:
     fingerprint: str = field(compare=True)
     email: str = field(compare=True)