Add conflict resolution unit tests, fix bugs in conflict resolution

multischleuder/conflict.py

@@ -109,9 +109,9 @@ class KeyConflictResolution:
 
     def __init__(self, smtp: 'SmtpClient', interval: int, statefile: str, template: str):
         self._smtp = smtp
-        self._interval: int
+        self._interval: int = interval
         self._state_file: str = statefile
-        self._template: str = statefile
+        self._template: str = template
         self._messages: List[ConflictMessage] = []
         self._logger: logging.Logger = logging.getLogger()
 
@@ -129,10 +129,11 @@ class KeyConflictResolution:
                  target: str,
                  mail_from: str,
                  subscriptions: List['SchleuderSubscriber']) -> 'SchleuderSubscriber':
-        if len({s.email for s in subscriptions}) != 1:
-            raise ValueError('Number of unique subscriptions must be 1')
         if len(subscriptions) == 1:
             return subscriptions[0]
+        if len({s.key.blob for s in subscriptions}) == 1:
+            # No conflict if all keys are the same
+            return subscriptions[0]
         # Conflict Resolution: Choose the OLDEST subscriptions, but notify using ALL keys
         earliest: SchleuderSubscriber = min(subscriptions, key=lambda x: x.created_at)
         self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder}:')
@@ -143,8 +144,8 @@ class KeyConflictResolution:
             target,
             earliest,
             subscriptions,
-            self._template,
+            mail_from,
-            mail_from
+            self._template
         )
         self._messages.append(msg)
         # Return the result of conflict resolution
@@ -168,11 +169,11 @@ class KeyConflictResolution:
                 f.truncate()
                 json.dump(state, f)
         # Finally send the mails
-        with self._smtp as smtp:
+        if len(msgs) > 0 and not dry_run:
-            for m in msgs:
+            with self._smtp as smtp:
-                msg = m.mime
+                for m in msgs:
-                self._logger.debug(f'MIME Message:\n{str(m)}')
+                    msg = m.mime
-                if not dry_run:
+                    self._logger.debug(f'MIME Message:\n{str(m)}')
                     self._logger.info(f'Sending key conflict message to {msg["To"]}')
                     smtp.send_message(msg)
         # Clear conflict messages

multischleuder/test/test_conflict.py

@@ -0,0 +1,328 @@
+
+import io
+import json
+import unittest
+from datetime import datetime, timedelta
+from unittest.mock import patch, mock_open, MagicMock
+
+import pgpy  # type: ignore
+from dateutil.tz import tzutc
+
+from multischleuder.conflict import ConflictMessage, KeyConflictResolution
+from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
+
+
+# 2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6
+_PRIVKEY_1, _ = pgpy.PGPKey.from_blob('''
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lFgEYlirsBYJKwYBBAHaRw8BAQdAGAHsSb3b3x+V6d7XouOXJryqW4mcjn1nDT2z
+Fgf5lEwAAPoCqlVJWb79nANzKDdH8/mJCl5UT0CEoWyuAWtr89ofEw7ltD1NdWx0
+aXNjaGxldWRlciBUZXN0IEtleSAoVEVTVCAtIERPIE5PVCBVU0UpIDxmb29AZXhh
+bXBsZS5vcmc+iJAEExYIADgWIQQvu8Dfl/2/HktwTu3jnvT6xCC+tgUCYlirsAIb
+AwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRDjnvT6xCC+tlGVAP0a0Yqoc/nm
+OTwV1rzczdhYy9Gk7K2z7I0N380NJU4UJAEA/rVYy38ePeTl5/2NjcJ2WirTPNBT
+wbMtgtLtKxHergCcXQRiWKuwEgorBgEEAZdVAQUBAQdAOcauX1G9YtgMr27fmRYM
+cfji9yk9dgJEpC3GgHkYynEDAQgHAAD/XhxqpdVzZHl/Rce4VCSAq1b1LWRMYyYH
+MveBRrkMuMgPgIh4BBgWCAAgFiEEL7vA35f9vx5LcE7t4570+sQgvrYFAmJYq7AC
+GwwACgkQ4570+sQgvrbiUQD+LKfQo1THwAqtIwunslrCaxP64PalzDW4fepk8cyN
+reAA/3X/W8pDxfm0RjOUT069Wq1/0RJAEuPPExowR25vmqIF
+=b9XF
+-----END PGP PRIVATE KEY BLOCK-----
+''')
+
+
+# 135AFA0FB3FF584828911208B7913308392972A4
+_PRIVKEY_2, _ = pgpy.PGPKey.from_blob('''
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lFgEYljycBYJKwYBBAHaRw8BAQdA8PMUGJJ4oiYo6wwYviH798WOKKQMQJyIqhyu
+URqE/b0AAP4wpWZo8GPyB7+I8qQbOzwwb+gdKTmp0WvE1P2QhxIpYRCPtEhNdWx0
+aXNjaGxldWRlciBDb25mbGljdCBUZXN0IEtleSAoVEVTVCAyIC0gRE8gTk9UIFVT
+RSkgPGZvb0BleGFtcGxlLm9yZz6IkAQTFggAOBYhBBNa+g+z/1hIKJESCLeRMwg5
+KXKkBQJiWPJwAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJELeRMwg5KXKk
+xasA/1Pdb8eiLXTdqAMOI8H8BwEvLebiOFYL8eJx1AyjZy/vAQCNxlK4Z5cA6KzW
+Zwe51YuCF69QBRatAGLhx8PoWB0DApxdBGJY8nASCisGAQQBl1UBBQEBB0DuWMns
+ibefPCLJvR/LCfwRDhI3IC5W7S1506lZli3MSwMBCAcAAP9Ax8BOzTa4ewZLvO+z
+2l5NBEddpKZ6q3NFKbmhmtQ3OBCtiHgEGBYIACAWIQQTWvoPs/9YSCiREgi3kTMI
+OSlypAUCYljycAIbDAAKCRC3kTMIOSlypBVMAP9Uu0Hr4bJyl35WA5I7hrC666Hr
+QBzu2Swgk6MkU45SLQD+LagpBVJxHcbvmK+n8MFvTSrusF8H78P4TrMLP4Onvw4=
+=UiF7
+-----END PGP PRIVATE KEY BLOCK-----
+''')
+
+
+_TEMPLATE = '''{{
+  "subscriber": "{subscriber}",
+  "schleuder": "{schleuder}",
+  "chosen": "{chosen}"
+}}'''
+
+
+_CONFLICT_STATE_NONE = '{}'
+_CONFLICT_STATE_STALE = '''{
+  "53e707b460c062a2e705f59750f9297dae002a17": 123456
+}'''
+_CONFLICT_STATE_RECENT = f'''{{
+  "53e707b460c062a2e705f59750f9297dae002a17": {int((datetime.utcnow() - timedelta(hours=6)).timestamp())}
+}}'''
+
+
+class TestKeyConflictResolution(unittest.TestCase):
+
+    def test_order_resistent_hash(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        msg1 = ConflictMessage(
+            schleuder='',
+            chosen=sub2,
+            affected=[sub1, sub2],
+            mail_from='',
+            template='')
+        msg2 = ConflictMessage(
+            schleuder='',
+            chosen=sub2,
+            affected=[sub2, sub1],
+            mail_from='',
+            template='')
+        msg3 = ConflictMessage(
+            schleuder='',
+            chosen=sub1,
+            affected=[sub2, sub1],
+            mail_from='',
+            template='')
+        msg4 = ConflictMessage(
+            schleuder='foo',
+            chosen=sub1,
+            affected=[sub2, sub1],
+            mail_from='bar',
+            template='baz')
+
+        self.assertEqual(msg1.digest, msg2.digest)
+        self.assertNotEqual(msg1.digest, msg3.digest)
+        self.assertEqual(msg3.digest, msg4.digest)
+
+    def test_empty(self):
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        self.assertEqual(0, len(kcr.resolve('', '', [])))
+        self.assertEqual(0, len(kcr._messages))
+
+    def test_one(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve('', '', [sub1])
+        self.assertEqual(1, len(resolved))
+        self.assertEqual(sub1, resolved[0])
+        self.assertEqual(0, len(kcr._messages))
+
+    def test_same_keys_conflict(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+
+        # Same key
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+
+        self.assertEqual(1, len(resolved))
+        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
+        # Same keys, no conflict message
+        self.assertEqual(0, len(kcr._messages))
+
+    def test_different_keys_conflict(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+
+        self.assertEqual(1, len(resolved))
+        self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint)
+        # Different keys should trigger a conflict message
+        self.assertEqual(1, len(kcr._messages))
+        msg = kcr._messages[0].mime
+        pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))
+        # Verify that the message is encrypted with both keys
+        dec1 = _PRIVKEY_1.decrypt(pgp)
+        dec2 = _PRIVKEY_2.decrypt(pgp)
+        self.assertEqual(dec1.message, dec2.message)
+
+        payload = json.loads(dec1.message)
+        self.assertEqual('foo@example.org', payload['subscriber'])
+        self.assertEqual('test@schleuder.example.org', payload['schleuder'])
+        self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4  foo@example.org', payload['chosen'])
+
+    def test_send_messages_nostate(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        sub2 = SchleuderSubscriber(4, 'bar@example.org', key1, sch1.id, date2)
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        sub3 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+        sub4 = SchleuderSubscriber(8, 'bar@example.org', key2, sch2.id, date1)
+
+        mock_smtp = MagicMock()
+        mock_smtp.__enter__.return_value = mock_smtp
+        kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2, sub3, sub4])
+        self.assertEqual(2, len(kcr._messages))
+        msgs = kcr._messages
+
+        now = datetime.utcnow().timestamp()
+        contents = io.StringIO(_CONFLICT_STATE_NONE)
+        with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
+            mock_statefile().__enter__.return_value = contents
+            kcr.send_messages()
+        mock_statefile.assert_called_with('/tmp/state.json', 'a+')
+        mock_smtp.__enter__.assert_called_once()
+        self.assertEqual(2, mock_smtp.send_message.call_count)
+        contents.seek(0)
+        state = json.loads(contents.read())
+        self.assertEqual(2, len(state))
+        self.assertIn(msgs[0].digest, state)
+        self.assertIn(msgs[1].digest, state)
+        self.assertLess(now - state[msgs[0].digest], 60)
+        self.assertLess(now - state[msgs[1].digest], 60)
+
+    def test_send_messages_stalestate(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        mock_smtp = MagicMock()
+        mock_smtp.__enter__.return_value = mock_smtp
+        kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+        self.assertEqual(1, len(kcr._messages))
+        msg = kcr._messages[0]
+
+        now = datetime.utcnow().timestamp()
+        contents = io.StringIO(_CONFLICT_STATE_STALE)
+        with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile:
+            mock_statefile().__enter__.return_value = contents
+            kcr.send_messages()
+        mock_statefile.assert_called_with('/tmp/state.json', 'a+')
+        mock_smtp.__enter__.assert_called_once()
+        self.assertEqual(1, mock_smtp.send_message.call_count)
+        contents.seek(0)
+        state = json.loads(contents.read())
+        self.assertEqual(1, len(state))
+        self.assertIn(msg.digest, state)
+        self.assertLess(now - state[msg.digest], 60)
+
+    def test_send_messages_recentstate(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        mock_smtp = MagicMock()
+        mock_smtp.__enter__.return_value = mock_smtp
+        kcr = KeyConflictResolution(mock_smtp, 86400, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+        self.assertEqual(1, len(kcr._messages))
+        msg = kcr._messages[0]
+
+        now = datetime.utcnow().timestamp()
+        contents = io.StringIO(_CONFLICT_STATE_RECENT)
+        with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_RECENT)) as mock_statefile:
+            mock_statefile().__enter__.return_value = contents
+            kcr.send_messages()
+        mock_statefile.assert_called_with('/tmp/state.json', 'a+')
+        # No message should be sent
+        mock_smtp.__enter__.assert_not_called()
+        mock_smtp.send_message.assert_not_called()
+        # Statefile should not have been updated
+        contents.seek(0)
+        state = json.loads(contents.read())
+        self.assertEqual(1, len(state))
+        self.assertIn(msg.digest, state)
+        self.assertLess(now - state[msg.digest], 86460)
+        self.assertGreater(now - state[msg.digest], 60)
+
+    def test_send_messages_dryrun(self):
+        sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
+        key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
+        date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
+        sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
+        # This subscription is older, so its key will be preferred
+        sch2 = SchleuderList(23, 'test-south@schleuder.example.org', 'AF586C0625CF77BBB659747515D41C5D84BF99D3')
+        key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
+        date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
+        sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
+
+        mock_smtp = MagicMock()
+        mock_smtp.__enter__.return_value = mock_smtp
+        kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
+        resolved = kcr.resolve(
+            target='test@schleuder.example.org',
+            mail_from='test-owner@schleuder.example.org',
+            subscriptions=[sub1, sub2])
+        self.assertEqual(1, len(kcr._messages))
+        msg = kcr._messages[0]
+
+        now = datetime.utcnow().timestamp()
+        contents = io.StringIO(_CONFLICT_STATE_STALE)
+        with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile:
+            mock_statefile().__enter__.return_value = contents
+            kcr.send_messages(dry_run=True)
+        mock_statefile.assert_called_with('/tmp/state.json', 'a+')
+        mock_statefile().write.assert_not_called()
+        mock_smtp.__enter__.assert_not_called()
+        mock_smtp.send_message.assert_not_called()
+        contents.seek(0)
+        self.assertEqual(_CONFLICT_STATE_STALE, contents.read())