From a0fb9a503b6e9e1b2a881706342ddac8f9da8832 Mon Sep 17 00:00:00 2001 From: s3lph <1375407-s3lph@users.noreply.gitlab.com> Date: Sat, 16 Apr 2022 05:37:38 +0200 Subject: [PATCH] Remove TODO comment; turned into a gitlab issue --- multischleuder/api.py | 32 ++-- multischleuder/conflict.py | 46 +++--- multischleuder/processor.py | 16 +- multischleuder/test/test_api.py | 62 +++++++- multischleuder/test/test_conflict.py | 65 ++++++++ multischleuder/test/test_multilist.py | 210 ++++++++++++++++++++++++++ multischleuder/test/test_types.py | 11 ++ multischleuder/types.py | 12 +- 8 files changed, 406 insertions(+), 48 deletions(-) create mode 100644 multischleuder/test/test_multilist.py diff --git a/multischleuder/api.py b/multischleuder/api.py index 9ab9f3d..6eecf9d 100644 --- a/multischleuder/api.py +++ b/multischleuder/api.py @@ -57,31 +57,37 @@ class SchleuderApi: # List Management - def get_lists(self) -> List['SchleuderList']: + def get_lists(self) -> List[SchleuderList]: lists = self.__request('lists.json') return [SchleuderList.from_api(**s) for s in lists] # Subscriber Management - def get_subscribers(self, schleuder: 'SchleuderList') -> List['SchleuderSubscriber']: + def get_subscribers(self, schleuder: SchleuderList) -> List[SchleuderSubscriber]: response = self.__request('subscriptions.json', schleuder.id) subs: List[SchleuderSubscriber] = [] for r in response: - key = self.get_key(r['fingerprint'], schleuder) + key: Optional[SchleuderKey] = None + if r['fingerprint']: + key = self.get_key(r['fingerprint'], schleuder) sub = SchleuderSubscriber.from_api(key, **r) subs.append(sub) return subs - def get_subscriber(self, email: str, schleuder: 'SchleuderList') -> 'SchleuderSubscriber': + def get_subscriber(self, email: str, schleuder: SchleuderList) -> SchleuderSubscriber: response = self.__request('subscriptions.json', list_id=schleuder.id) for r in response: if r['email'] != email: continue - key = self.get_key(r['fingerprint'], schleuder) + key: Optional[SchleuderKey] = None + if r['fingerprint']: + key = self.get_key(r['fingerprint'], schleuder) return SchleuderSubscriber.from_api(key, **r) raise KeyError(f'{email} is not subscribed to {schleuder.name}') - def subscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'): + def subscribe(self, sub: SchleuderSubscriber, schleuder: SchleuderList): + if sub.key is None: + raise ValueError('Cannot subscribe without key') if self._dry_run: return data = json.dumps({ @@ -90,13 +96,15 @@ class SchleuderApi: }) self.__request('subscriptions.json', list_id=schleuder.id, data=data, method='POST') - def unsubscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'): + def unsubscribe(self, sub: SchleuderSubscriber, schleuder: SchleuderList): listsub = self.get_subscriber(sub.email, schleuder) if self._dry_run: return self.__request('subscriptions/{}.json', fmt=[listsub.id], method='DELETE') - def update_fingerprint(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'): + def update_fingerprint(self, sub: SchleuderSubscriber, schleuder: SchleuderList): + if sub.key is None: + raise ValueError('Cannot update fingerprint without key') listsub = self.get_subscriber(sub.email, schleuder) if self._dry_run: return @@ -105,13 +113,11 @@ class SchleuderApi: # Key Management - def get_key(self, fpr: str, schleuder: 'SchleuderList'): - if self._dry_run: - return + def get_key(self, fpr: str, schleuder: SchleuderList) -> SchleuderKey: key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr]) return SchleuderKey.from_api(schleuder.id, **key) - def post_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'): + def post_key(self, key: SchleuderKey, schleuder: SchleuderList): if self._dry_run: return data = json.dumps({ @@ -119,7 +125,7 @@ class SchleuderApi: }) self.__request('keys.json', list_id=schleuder.id, data=data, method='POST') - def delete_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'): + def delete_key(self, key: SchleuderKey, schleuder: SchleuderList): if self._dry_run: return self.__request('keys/{}.json', list_id=schleuder.id, fmt=[key.fingerprint], method='DELETE') diff --git a/multischleuder/conflict.py b/multischleuder/conflict.py index 44a2bd0..ef1c6ac 100644 --- a/multischleuder/conflict.py +++ b/multischleuder/conflict.py @@ -1,5 +1,5 @@ -from typing import Dict, List +from typing import Dict, List, Optional import email.mime.application import email.mime.multipart @@ -45,9 +45,10 @@ class ConflictMessage: self._chosen.schleuder)) # Include all subscriptions, including the FULL key for s in subs: - h.update(struct.pack('!ds', - s.schleuder, - s.key.blob.encode())) + key = b'N/A' + if s.key is not None: + key = s.key.blob.encode() + h.update(struct.pack('!ds', s.schleuder, key)) return h.hexdigest() @property @@ -57,10 +58,12 @@ class ConflictMessage: @property def mime(self) -> email.mime.multipart.MIMEMultipart: # Render the message body - _chosen = f'{self._chosen.key.fingerprint} {self._chosen.email}' + fpr = 'N/A' if self._chosen.key is None else self._chosen.key.fingerprint + _chosen = f'{fpr} {self._chosen.email}' _affected = '' for affected in self._affected: - _affected += f'{affected.key.fingerprint} {affected.schleuder}\n' + fpr = 'N/A' if affected.key is None else affected.key.fingerprint + _affected += f'{fpr} {affected.schleuder}\n' msg: str = self._template.format( subscriber=self._chosen.email, schleuder=self._schleuder, @@ -73,9 +76,10 @@ class ConflictMessage: sessionkey = cipher.gen_key() try: for affected in self._affected: - key, _ = pgpy.PGPKey.from_blob(affected.key.blob) - key._require_usage_flags = False - pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey) + if affected.key is not None: + key, _ = pgpy.PGPKey.from_blob(affected.key.blob) + key._require_usage_flags = False + pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey) finally: del sessionkey # Build the MIME message @@ -123,22 +127,26 @@ class KeyConflictResolution: for s in subscriptions: subs.setdefault(s.email, []).append(s) # Perform conflict resolution for each set of subscribers with the same email - return [self._resolve(target, mail_from, s) for s in subs.values()] + resolved = [self._resolve(target, mail_from, s) for s in subs.values()] + return [r for r in resolved if r is not None] def _resolve(self, target: str, mail_from: str, - subscriptions: List[SchleuderSubscriber]) -> SchleuderSubscriber: - if len(subscriptions) == 1: - return subscriptions[0] - if len({s.key.blob for s in subscriptions}) == 1: + subscriptions: List[SchleuderSubscriber]) -> Optional[SchleuderSubscriber]: + notnull = [s for s in subscriptions if s.key is not None] + if len(notnull) == 0: + return None + if len({s.key.blob for s in subscriptions if s.key is not None}) == 1: # No conflict if all keys are the same - return subscriptions[0] - # Conflict Resolution: Choose the OLDEST subscriptions, but notify using ALL keys - earliest: SchleuderSubscriber = min(subscriptions, key=lambda x: x.created_at) - self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder}:') + return notnull[0] + # Conflict Resolution: Choose the OLDEST subscription with a key, but notify using ALL keys + earliest: SchleuderSubscriber = min(notnull, key=lambda x: x.created_at) + assert earliest.key is not None # Make mypy happy; it can't know that earliest.key can't be None + self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.key.fingerprint}:') for s in subscriptions: - self._logger.debug(f' - {s.schleuder}: {s.key.fingerprint}') + fpr = 'N/A' if s.key is None else s.key.fingerprint + self._logger.debug(f' - {s.schleuder}: {fpr}') # At this point, the messages are only queued locally, they will be sent afterwards msg = ConflictMessage( target, diff --git a/multischleuder/processor.py b/multischleuder/processor.py index 5ab7711..496c5d3 100644 --- a/multischleuder/processor.py +++ b/multischleuder/processor.py @@ -36,7 +36,7 @@ class MultiList: for s in self._api.get_subscribers(target_list) if s.email not in self._unmanaged } - current_keys: Set[SchleuderKey] = {s.key for s in current_subs} + current_keys: Set[SchleuderKey] = {s.key for s in current_subs if s.key is not None} all_subs: List[SchleuderSubscriber] = [] # This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists ... for source in sources: @@ -46,9 +46,9 @@ class MultiList: continue all_subs.append(s) # ... which is taken care of by the key conflict resolution routine - self._kcr.resolve(self._target, self._mail_from, all_subs) - intended_subs: Set[SchleuderSubscriber] = set(all_subs) - intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs} + resolved = self._kcr.resolve(self._target, self._mail_from, all_subs) + intended_subs: Set[SchleuderSubscriber] = set(resolved) + intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None} # Determine the change set to_subscribe = intended_subs.difference(current_subs) to_unsubscribe = current_subs.difference(intended_subs) @@ -58,19 +58,19 @@ class MultiList: # Perform the actual list modifications in an order which avoids race conditions for key in to_add: self._api.post_key(key, target_list) - self._logger.info(f'Added key: {key}') + self._logger.info(f'Added key: {key}') for sub in to_subscribe: self._api.subscribe(sub, target_list) - self._logger.info(f'Subscribed user: {sub}') + self._logger.info(f'Subscribed user: {sub}') for sub in to_update: self._api.update_fingerprint(sub, target_list) - self._logger.info(f'Updated key: {sub}') + self._logger.info(f'Updated key: {sub}') for sub in to_unsubscribe: self._api.unsubscribe(sub, target_list) self._logger.info(f'Unsubscribed user: {sub}') for key in to_remove: self._api.delete_key(key, target_list) - self._logger.info(f'Removed key: {key}') + self._logger.info(f'Removed key: {key}') # Finally, send any queued key conflict messages. self._kcr.send_messages() if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0: diff --git a/multischleuder/test/test_api.py b/multischleuder/test/test_api.py index 04ca5e8..b30221a 100644 --- a/multischleuder/test/test_api.py +++ b/multischleuder/test/test_api.py @@ -79,6 +79,21 @@ _SUBSCRIBER_RESPONSE = ''' ] ''' +_SUBSCRIBER_RESPONSE_NOKEY = ''' +[ + { + "id": 24, + "list_id": 42, + "email": "foo@example.org", + "fingerprint": "", + "admin": false, + "delivery_enabled": true, + "created_at": "2022-04-15T01:11:12.123Z", + "updated_at": "2022-04-15T01:11:12.123Z" + } +] +''' + _KEY_RESPONSE = ''' { "fingerprint": "2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6", @@ -96,7 +111,7 @@ _KEY_RESPONSE = ''' class TestSchleuderApi(unittest.TestCase): - def _mock_api(self, mock): + def _mock_api(self, mock, nokey=False): m = MagicMock() m.getcode.return_value = 200 @@ -105,6 +120,8 @@ class TestSchleuderApi(unittest.TestCase): if '/lists' in url: return _LIST_RESPONSE.encode() if '/subscriptions' in url: + if nokey: + return _SUBSCRIBER_RESPONSE_NOKEY.encode() return _SUBSCRIBER_RESPONSE.encode() if '/keys' in url: return _KEY_RESPONSE.encode() @@ -147,6 +164,21 @@ class TestSchleuderApi(unittest.TestCase): self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', subs[0].key.blob) self.assertEqual(42, subs[0].key.schleuder) + @patch('urllib.request.urlopen') + def test_get_subscribers_nokey(self, mock): + api = self._mock_api(mock, nokey=True) + subs = api.get_subscribers(SchleuderList(42, '', '')) + # Test request data + self.assertEqual('https://localhost:4443/subscriptions.json?list_id=42', + mock.call_args_list[0][0][0].get_full_url()) + self.assertEqual(1, len(subs)) + self.assertEqual(24, subs[0].id) + self.assertEqual('foo@example.org', subs[0].email) + self.assertEqual(42, subs[0].schleuder) + self.assertEqual(datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()), + subs[0].created_at) + self.assertIsNone(subs[0].key) + @patch('urllib.request.urlopen') def test_get_subscriber(self, mock): api = self._mock_api(mock) @@ -159,6 +191,14 @@ class TestSchleuderApi(unittest.TestCase): with self.assertRaises(KeyError): api.get_subscriber('bar@example.org', SchleuderList(42, '', '')) + @patch('urllib.request.urlopen') + def test_get_subscriber_nokey(self, mock): + api = self._mock_api(mock, nokey=True) + sub = api.get_subscriber('foo@example.org', SchleuderList(42, '', '')) + self.assertEqual(24, sub.id) + self.assertEqual('foo@example.org', sub.email) + self.assertIsNone(sub.key) + @patch('urllib.request.urlopen') def test_subscribe(self, mock): api = self._mock_api(mock) @@ -171,6 +211,14 @@ class TestSchleuderApi(unittest.TestCase): self.assertEqual('POST', mock.call_args_list[-1][0][0].method) # todo assert request payload + @patch('urllib.request.urlopen') + def test_subscribe_nokey(self, mock): + api = self._mock_api(mock) + now = datetime.utcnow() + sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now) + with self.assertRaises(ValueError): + api.subscribe(sub, SchleuderList(42, '', '')) + @patch('urllib.request.urlopen') def test_unsubscribe(self, mock): api = self._mock_api(mock) @@ -195,6 +243,14 @@ class TestSchleuderApi(unittest.TestCase): self.assertEqual('PATCH', mock.call_args_list[-1][0][0].method) # todo assert request payload + @patch('urllib.request.urlopen') + def test_update_fingerprint_nokey(self, mock): + api = self._mock_api(mock) + now = datetime.utcnow() + sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now) + with self.assertRaises(ValueError): + api.update_fingerprint(sub, SchleuderList(42, '', '')) + @patch('urllib.request.urlopen') def test_get_key(self, mock): api = self._mock_api(mock) @@ -234,13 +290,13 @@ class TestSchleuderApi(unittest.TestCase): key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42) sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now) sch = SchleuderList(42, '', '') - # create, update, delete should be no-ops; 2 requests for retrieving the subscriptions in unsub & update + # create, update, delete should be no-ops; 5 requests for retrieving the keys & subscriptions in unsub & update api.subscribe(sub, sch) api.unsubscribe(sub, sch) api.update_fingerprint(sub, sch) api.post_key(key, SchleuderList(42, '', '')) api.delete_key(key, SchleuderList(42, '', '')) - self.assertGreater(3, len(mock.call_args_list)) + self.assertGreater(5, len(mock.call_args_list)) # only reads should execute api.get_lists() api.get_subscribers(sch) diff --git a/multischleuder/test/test_conflict.py b/multischleuder/test/test_conflict.py index 892f177..83392d2 100644 --- a/multischleuder/test/test_conflict.py +++ b/multischleuder/test/test_conflict.py @@ -182,6 +182,40 @@ class TestKeyConflictResolution(unittest.TestCase): self.assertEqual('test@schleuder.example.org', payload['schleuder']) self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4 foo@example.org', payload['chosen']) + def test_conflict_one_nullkey(self): + sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') + key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id) + date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) + sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1) + + # This subscription has no key, it it will be discarded + sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3') + date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) + sub2 = SchleuderSubscriber(7, 'foo@example.org', None, sch2.id, date2) + + kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) + resolved = kcr.resolve( + target='test@schleuder.example.org', + mail_from='test-owner@schleuder.example.org', + subscriptions=[sub1, sub2]) + + self.assertEqual(1, len(resolved)) + self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint) + # Null key should not trigger a confict + self.assertEqual(0, len(kcr._messages)) + + def test_conflict_only_nullkeys(self): + sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') + date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) + sub1 = SchleuderSubscriber(3, 'foo@example.org', None, sch1.id, date1) + + kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) + resolved = kcr.resolve( + target='test@schleuder.example.org', + mail_from='test-owner@schleuder.example.org', + subscriptions=[sub1]) + self.assertEqual(0, len(resolved)) + def test_send_messages_nostate(self): sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id) @@ -326,3 +360,34 @@ class TestKeyConflictResolution(unittest.TestCase): mock_smtp.send_message.assert_not_called() contents.seek(0) self.assertEqual(_CONFLICT_STATE_STALE, contents.read()) + + def test_send_messages_nullkey(self): + sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') + key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id) + date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) + sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1) + # This subscription is older, so its key will be preferred + sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3') + key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id) + date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) + sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) + # This subscription does not have a key, so it will not be considered + sch3 = SchleuderList(7, 'test-east@schleuder.example.org', '36C545D9C88AD0E7EFEF18DE06ACC53D5820EBC0') + date3 = datetime(2022, 4, 11, 5, 23, 42, 0, tzinfo=tzutc()) + sub3 = SchleuderSubscriber(7, 'foo@example.org', None, sch3.id, date3) + + mock_smtp = MagicMock() + mock_smtp.__enter__.return_value = mock_smtp + kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE) + resolved = kcr.resolve( + target='test@schleuder.example.org', + mail_from='test-owner@schleuder.example.org', + subscriptions=[sub1, sub2, sub3]) + self.assertEqual(1, len(kcr._messages)) + msg = kcr._messages[0].mime + pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True)) + # Verify that the message is encrypted with both keys + self.assertEqual(2, len(pgp.encrypters)) + dec1 = _PRIVKEY_1.decrypt(pgp) + dec2 = _PRIVKEY_2.decrypt(pgp) + self.assertEqual(dec1.message, dec2.message) diff --git a/multischleuder/test/test_multilist.py b/multischleuder/test/test_multilist.py new file mode 100644 index 0000000..c9b8066 --- /dev/null +++ b/multischleuder/test/test_multilist.py @@ -0,0 +1,210 @@ + +from typing import Dict, List + +import unittest +from datetime import datetime +from unittest.mock import MagicMock + +from dateutil.tz import tzutc + +from multischleuder.processor import MultiList +from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber + + +def _resolve(target: str, + mail_from: str, + subscriptions: List[SchleuderSubscriber]) -> List[SchleuderSubscriber]: + # Mock conflict resolution that does not send or prepare messages + subs: Dict[str, List[SchleuderSubscriber]] = {} + for s in subscriptions: + subs.setdefault(s.email, []).append(s) + return [min(s, key=lambda x: x.created_at) for s in subs.values()] + + +def _list_lists(): + return [ + SchleuderList(1, 'test@schleuder.example.org', '0CD581127A178F1931909B14B7CC98DA082D2A64'), + SchleuderList(2, 'test-global@schleuder.example.org', 'F60B441065D485224E0F53EBBEAD4E67512CB645'), + SchleuderList(3, 'test-north@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'), + SchleuderList(4, 'test-south@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'), + SchleuderList(5, 'test-east@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'), + SchleuderList(6, 'test-west@schleuder.example.org', '552BB62D10424FF169498B3D183C27F29DAAFDFB'), + SchleuderList(7, 'test2-global@schleuder.example.org', 'A821095FB503836134D54138D3306C629C0DDFB5') + ] + + +def _get_key(fpr: str, schleuder: SchleuderList): + key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471', + 'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id) + key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6', + 'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id) + key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79', + 'alex.example@example.org', 'BEGIN PGP 716DEC79', schleuder.id) + key4 = SchleuderKey('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', + 'aspammer@example.org', 'BEGIN PGP 045AE2D6', schleuder.id) + key5 = SchleuderKey('3699B7549B2A78F27AC93420FB6AD038C95558E5', + 'anna.example@example.org', 'BEGIN PGP C95558E5', schleuder.id) + key6 = SchleuderKey('698118749D1490C9822FA1D9600E77BC529681F9', + 'anna.example@example.org', 'BEGIN PGP 529681F9', schleuder.id) + key7 = SchleuderKey('BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3', + 'anotherspammer@example.org', 'BEGIN PGP 9233D3E3', schleuder.id) + key8 = SchleuderKey('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', + 'andy.example@example.org', 'BEGIN PGP 8C89A2B9', schleuder.id) + key9 = SchleuderKey('C8C77172C31A796C563920C9358EBEA0F7662478', + 'aaron.example@example.org', 'BEGIN PGP F7662478', schleuder.id) + return { + '966842467B3254143F994D5E5C408C012D216471': key1, + '6449FFB6EE68187962FA013B5CA2F4F51791BAF6': key2, + '414D3960D34730F63C74D5190EBC5A16716DEC79': key3, + '8258FAF8B161B3DD8F784874F73E2DDF045AE2D6': key4, + '3699B7549B2A78F27AC93420FB6AD038C95558E5': key5, + '698118749D1490C9822FA1D9600E77BC529681F9': key6, + 'BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3': key7, + '73E6C1170DA158D0139EF1F2349F17BB8C89A2B9': key8, + 'C8C77172C31A796C563920C9358EBEA0F7662478': key9, + }[fpr] + + +def _get_subs(schleuder: SchleuderList): + key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471', + 'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id) + key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6', + 'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id) + key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79', + 'alex.example@example.org', 'BEGIN PGP 716DEC79', schleuder.id) + key4 = SchleuderKey('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', + 'aspammer@example.org', 'BEGIN PGP 045AE2D6', schleuder.id) + key5 = SchleuderKey('3699B7549B2A78F27AC93420FB6AD038C95558E5', + 'anna.example@example.org', 'BEGIN PGP C95558E5', schleuder.id) + key6 = SchleuderKey('698118749D1490C9822FA1D9600E77BC529681F9', + 'anna.example@example.org', 'BEGIN PGP 529681F9', schleuder.id) + key7 = SchleuderKey('BE11EE506C39B496A18A1FF56B3D3C0C9233D3E3', + 'anotherspammer@example.org', 'BEGIN PGP 9233D3E3', schleuder.id) + key8 = SchleuderKey('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', + 'andy.example@example.org', 'BEGIN PGP 8C89A2B9', schleuder.id) + key9 = SchleuderKey('C8C77172C31A796C563920C9358EBEA0F7662478', + 'aaron.example@example.org', 'BEGIN PGP F7662478', schleuder.id) + date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) + date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) + if schleuder.id == 2: + return [ + # admin should be ignored (unmanaged) + SchleuderSubscriber(0, 'admin@example.org', key1, schleuder.id, date1), + # ada.lovelace should be unsubscribed + SchleuderSubscriber(1, 'ada.lovelace@example.org', key2, schleuder.id, date1), + # alex.example already subscribed - no change + SchleuderSubscriber(2, 'alex.example@example.org', key3, schleuder.id, date1), + # aspammer should be unsubscribed (banned) + SchleuderSubscriber(3, 'aspammer@example.org', key4, schleuder.id, date1), + # anna.example's key should be updated, but the subscrption should stay unchanged + SchleuderSubscriber(4, 'anna.example@example.org', key5, schleuder.id, date1), + ] + elif schleuder.id == 3: + return [ + SchleuderSubscriber(5, 'alex.example@example.org', key3, schleuder.id, date1), + SchleuderSubscriber(6, 'aspammer@example.org', key4, schleuder.id, date1), + ] + elif schleuder.id == 4: + return [ + SchleuderSubscriber(7, 'anna.example@example.org', key6, schleuder.id, date1), + SchleuderSubscriber(8, 'anotherspammer@example.org', key7, schleuder.id, date1), + ] + elif schleuder.id == 5: + return [ + SchleuderSubscriber(9, 'andy.example@example.org', key8, schleuder.id, date1), + ] + elif schleuder.id == 6: + return [ + SchleuderSubscriber(10, 'aaron.example@example.org', key9, schleuder.id, date2), + ] + else: + return [] + + +def _get_sub(email: str, schleuder: SchleuderList): + subs = _get_subs(schleuder) + return [s for s in subs if s.email == email][0] + + +class TestMultiList(unittest.TestCase): + + def _kcr_mock(self): + kcr = MagicMock() + kcr.resolve = _resolve + return kcr + + def _api_mock(self): + api = MagicMock() + api.get_lists = _list_lists + api.get_subscribers = _get_subs + api.get_subscriber = _get_sub + api.get_key = _get_key + return api + + def test_create(self): + sources = [ + 'test-north@schleuder.example.org', + 'test-east@schleuder.example.org', + 'test-south@schleuder.example.org', + 'test-west@schleuder.example.org' + ] + api = self._api_mock() + ml = MultiList(sources=sources, + target='test-global@schleuder.example.org', + unmanaged=['admin@example.org'], + banned=['aspammer@example.org', 'anotherspammer@example.org'], + mail_from='test-global-owner@schleuder.example.org', + api=api, + kcr=self._kcr_mock()) + ml.process() + + # Key uploads + self.assertEqual(3, len(api.post_key.call_args_list)) + a, b, c = sorted([x[0] for x in api.post_key.call_args_list], key=lambda x: x[0].fingerprint) + self.assertEqual(2, a[1].id) + self.assertEqual('anna.example@example.org', a[0].email) + self.assertEqual('698118749D1490C9822FA1D9600E77BC529681F9', a[0].fingerprint) + self.assertEqual(2, b[1].id) + self.assertEqual('andy.example@example.org', b[0].email) + self.assertEqual('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', b[0].fingerprint) + self.assertEqual(2, c[1].id) + self.assertEqual('aaron.example@example.org', c[0].email) + self.assertEqual('C8C77172C31A796C563920C9358EBEA0F7662478', c[0].fingerprint) + + # Subscriptions + self.assertEqual(2, len(api.subscribe.call_args_list)) + a, b = sorted([x[0] for x in api.subscribe.call_args_list], key=lambda x: x[0].email) + self.assertEqual(2, a[1].id) + self.assertEqual('aaron.example@example.org', a[0].email) + self.assertEqual('C8C77172C31A796C563920C9358EBEA0F7662478', a[0].key.fingerprint) + self.assertEqual(2, b[1].id) + self.assertEqual('andy.example@example.org', b[0].email) + self.assertEqual('73E6C1170DA158D0139EF1F2349F17BB8C89A2B9', b[0].key.fingerprint) + + # Key updates + self.assertEqual(1, len(api.update_fingerprint.call_args_list)) + a = api.update_fingerprint.call_args_list[0][0] + self.assertEqual(2, a[1].id) + self.assertEqual('anna.example@example.org', a[0].email) + self.assertEqual('698118749D1490C9822FA1D9600E77BC529681F9', a[0].key.fingerprint) + + # Unsubscribes + self.assertEqual(2, len(api.unsubscribe.call_args_list)) + a, b = sorted([x[0] for x in api.unsubscribe.call_args_list], key=lambda x: x[0].email) + self.assertEqual(2, a[1].id) + self.assertEqual('ada.lovelace@example.org', a[0].email) + self.assertEqual(2, b[1].id) + self.assertEqual('aspammer@example.org', b[0].email) + + # Key deletions + self.assertEqual(3, len(api.delete_key.call_args_list)) + a, b, c = sorted([x[0] for x in api.delete_key.call_args_list], key=lambda x: x[0].fingerprint) + self.assertEqual(2, a[1].id) + self.assertEqual('anna.example@example.org', a[0].email) + self.assertEqual('3699B7549B2A78F27AC93420FB6AD038C95558E5', a[0].fingerprint) + self.assertEqual(2, b[1].id) + self.assertEqual('ada.lovelace@example.org', b[0].email) + self.assertEqual('6449FFB6EE68187962FA013B5CA2F4F51791BAF6', b[0].fingerprint) + self.assertEqual(2, c[1].id) + self.assertEqual('aspammer@example.org', c[0].email) + self.assertEqual('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', c[0].fingerprint) diff --git a/multischleuder/test/test_types.py b/multischleuder/test/test_types.py index 24767ff..f86d8cc 100644 --- a/multischleuder/test/test_types.py +++ b/multischleuder/test/test_types.py @@ -35,3 +35,14 @@ class TestSchleuderTypes(unittest.TestCase): self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()), s.created_at) self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', s.key.fingerprint) + self.assertIn('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6 (foo@example.org)', str(k)) + + def test_parse_subscriber_nokey(self): + s = SchleuderSubscriber.from_api(None, **json.loads(_SUBSCRIBER_RESPONSE)[0]) + self.assertEqual(23, s.id) + self.assertEqual('foo@example.org', str(s)) + self.assertEqual('foo@example.org', s.email) + self.assertEqual(42, s.schleuder) + self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()), + s.created_at) + self.assertIsNone(s.key) diff --git a/multischleuder/types.py b/multischleuder/types.py index 4f10cbe..7c5bcb8 100644 --- a/multischleuder/types.py +++ b/multischleuder/types.py @@ -1,10 +1,12 @@ +from typing import Optional + from dataclasses import dataclass, field, Field from datetime import datetime from dateutil.parser import isoparse -@dataclass +@dataclass(frozen=True) class SchleuderList: id: int = field(compare=True) name: str = field(compare=False) @@ -18,16 +20,16 @@ class SchleuderList: return SchleuderList(id, email, fingerprint) -@dataclass +@dataclass(frozen=True) class SchleuderSubscriber: id: int = field(compare=False) email: str = field(compare=True) - key: 'SchleuderKey' = field(compare=False) + key: Optional['SchleuderKey'] = field(compare=False) schleuder: int = field(compare=False) created_at: datetime = field(compare=False) @staticmethod - def from_api(key: 'SchleuderKey', + def from_api(key: Optional['SchleuderKey'], id: int, list_id: int, email: str, @@ -40,7 +42,7 @@ class SchleuderSubscriber: return f'{self.email}' -@dataclass +@dataclass(frozen=True) class SchleuderKey: fingerprint: str = field(compare=True) email: str = field(compare=True)