Add test case with real schleuder

.gitignore

@@ -8,4 +8,4 @@
 **/.mypy_cache/
 
 ca.pem
-multischleuder.yml
+./multischleuder.yml

.gitlab-ci.yml

@@ -3,6 +3,7 @@ image: python:3.9-bullseye
 
 stages:
 - test
+- coverage
 - build
 - deploy
 
@@ -19,8 +20,8 @@ test:
   script:
   - pip3 install -e .
   - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
-  - python3 -m coverage combine
+  artifacts:
-  - python3 -m coverage report --rcfile=setup.cfg
+    - ".coverage*"
 
 codestyle:
   stage: test
@@ -35,6 +36,47 @@ mypy:
   - mypy --install-types --non-interactive multischleuder
   - mypy multischleuder
 
+schleuder:
+  stage: test
+  script:
+    - debconf-set-selections <<<"postfix postfix/mailname string example.org"
+    - debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'"
+    - apt update; apt install --yes schleuder schleuder-cli postfix
+    - /usr/lib/postfix/configure-instance.sh -
+    - echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
+    - /usr/sbin/postmulti -i - -p start
+    - schleuder-cli lists list || true
+    - export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
+    - echo "  - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
+    - |
+      cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
+      host: localhost
+      port: 4443
+      tls_fingerprint: ${CERT_FPR}
+      api_key: '00000000000000000000000000000000'
+      EOF
+    - /usr/bin/schleuder-api-daemon &
+    - sleep 5  # wait for daemons to start
+    - export API_DAEMON_PID=$!
+    - test/prepare-schleuder.sh
+    - pip3 install -e .
+    - python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
+    - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
+    - test/report.sh
+    - kill -9 ${API_DAEMON_PID} || true
+    - /usr/sbin/postmulti -i - -p stop
+    - sleep 5  # wait for daemons to terminate
+  artifacts:
+    - ".coverage*"
+
+
+
+coverage:
+  state: coverage
+  script:
+  - python3 -m coverage combine
+  - python3 -m coverage report --rcfile=setup.cfg
+
 
 
 build_wheel:

multischleuder.yml

@@ -0,0 +1,58 @@
+---
+
+api:
+  url: "https://localhost:4443"
+  token: 24125f2fe0ebc2fd853cf2e02f7599b3fa7f71a4c8e1519b
+  #cafile: /etc/schleuder/schleuder-certificate.pem
+  cafile: ca.pem
+
+lists:
+
+  - target: test@schleuder.example.org
+    unmanaged:
+      - admin@example.org
+    banned:
+      - banned@example.org
+    sources:
+      - test-basel@schleuder.example.org
+      - test-bern@schleuder.example.org
+      - test-zurich@schleuder.example.org
+    from: test-owner@schleuder.example.org
+
+smtp:
+  hostname: localhost
+  port: 8025
+
+conflict:
+  interval: 604800  # 1 week
+  statefile: /var/lib/multischleuder/conflict.json
+  template: |
+    Hi {subscriber},
+    
+    While compiling the subscriber list of {schleuder}, your
+    address {subscriber} was subscribed on multiple sub-lists with
+    different PGP keys.  There may be something fishy or malicious going on,
+    or this may simply have been a mistake by you or a list admin.
+    
+    You have only been subscribed to {schleuder} using the key you
+    have been subscribed with for the *longest* time:
+      
+    {chosen}
+      
+    Please review the following keys and talk to the admins of the
+    corresponding sub-lists to resolve this issue:
+    
+    Fingerprint                               Sub-List
+    -----------                               --------
+    {affected}
+  
+    For your convenience, this message has been encrypted with *all* of the
+    above keys.  If you have any questions, or do not understand this
+    message, please refer to your local Schleuder admin, or reply to this
+    message.
+  
+    Note that this automated message is unsigned, since MultiSchleuder does
+    not have access to Schleuder private keys.
+  
+    Regards
+    MultiSchleuder {schleuder}

multischleuder/__main__.py

@@ -0,0 +1,5 @@
+
+from multischleuder.main import main
+
+
+main()

multischleuder/api.py

@@ -50,7 +50,10 @@ class SchleuderApi:
         # Perform the actual request
         req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
         resp = urllib.request.urlopen(req, context=context)
-        return json.loads(resp.read().decode())
+        respdata: bytes = resp.read().decode()
+        if len(respdata) > 0:
+            return json.loads(respdata)
+        return None
 
     def dry_run(self):
         self._dry_run = True

multischleuder/conflict.py

@@ -163,7 +163,11 @@ class KeyConflictResolution:
         now = int(datetime.utcnow().timestamp())
         with open(self._state_file, 'a+') as f:
             f.seek(0)
+            try:
                 state: Dict[str, int] = json.load(f)
+            except:
+                # TODO: This could lead to a situation where multischleuder becomes a spammer
+                state = {}
             # Remove all state entries older than conflict_interval
             state = {k: v for k, v in state.items() if now-v < self._interval}
             # Remove all messages not already sent recently

multischleuder/main.py

@@ -2,6 +2,8 @@
 from typing import Any, Dict, List
 
 import argparse
+import logging
+import sys
 
 import yaml
 
@@ -60,12 +62,9 @@ def main():
     ap.add_argument('--version', action='version', version=__version__)
     ns = ap.parse_args(sys.argv[1:])
     if ns.verbose:
-        logger = logging.getLogger().setLevel('DEBUG')
+        logger = logging.getLogger()
+        logger.setLevel('DEBUG')
         logger.debug('Verbose logging enabled')
     lists = parse_config(ns)
     for lst in lists:
         lst.process(ns.dry_run)
-
-
-if __name__ == '__main__':
-    main()

multischleuder/test/test_api.py

@@ -69,8 +69,8 @@ _SUBSCRIBER_RESPONSE = '''
   {
     "id": 23,
     "list_id": 42,
-    "email": "foo@example.org",
+    "email": "andy.example@example.org",
-    "fingerprint": "2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6",
+    "fingerprint": "ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9",
     "admin": false,
     "delivery_enabled": true,
     "created_at": "2022-04-15T01:11:12.123Z",
@@ -84,7 +84,7 @@ _SUBSCRIBER_RESPONSE_NOKEY = '''
   {
     "id": 24,
     "list_id": 42,
-    "email": "foo@example.org",
+    "email": "andy.example@example.org",
     "fingerprint": "",
     "admin": false,
     "delivery_enabled": true,
@@ -96,15 +96,15 @@ _SUBSCRIBER_RESPONSE_NOKEY = '''
 
 _KEY_RESPONSE = '''
 {
-  "fingerprint": "2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6",
+  "fingerprint": "ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9",
-  "email": "foo@example.org",
+  "email": "andy.example@example.org",
   "expiry": null,
-  "generated_at": "2022-04-14T23:19:24.000Z",
+  "generated_at": "2022-04-16T23:19:24.000Z",
-  "primary_uid": "Multischleuder Test Key (TEST - DO NOT USE) <foo@example.org>",
+  "primary_uid": "Mutlischleuder Test User <andy.example@example.org>",
-  "oneline": "0x2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6 foo@example.org 2022-04-14",
+  "oneline": "0xADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 andy.example@example.org 2022-04-16",
   "trust_issues": null,
-  "description": "pub  ed25519 2022-04-14 [SC]\\n  2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6\\nuid  Multischleuder Test Key (TEST - DO NOT USE) <foo@example.org>\\nsub  cv25519 2022-04-14 [E]\\n",
+  "description": "pub    256?/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 2022-04-16\\nuid\\t\\tMutlischleuder Test User <andy.example@example.org>\\nsub    256?/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 2022-04-16\\nsub    256?/C0E8ED7A32F53626F2FCDC65F5035A1D90E35CAE 2022-04-16\\n",
-  "ascii": "pub  ed25519 2022-04-14 [SC]\\n  2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6\\nuid  Multischleuder Test Key (TEST - DO NOT USE) <foo@example.org>\\nsub  cv25519 2022-04-14 [E]\\n-----BEGIN PGP PUBLIC KEY BLOCK-----\\n\\nmDMEYlirsBYJKwYBBAHaRw8BAQdAGAHsSb3b3x+V6d7XouOXJryqW4mcjn1nDT2z\\nFgf5lEy0PU11bHRpc2NobGV1ZGVyIFRlc3QgS2V5IChURVNUIC0gRE8gTk9UIFVT\\nRSkgPGZvb0BleGFtcGxlLm9yZz6IkAQTFggAOBYhBC+7wN+X/b8eS3BO7eOe9PrE\\nIL62BQJiWKuwAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEOOe9PrEIL62\\nUZUA/RrRiqhz+eY5PBXWvNzN2FjL0aTsrbPsjQ3fzQ0lThQkAQD+tVjLfx495OXn\\n/Y2NwnZaKtM80FPBsy2C0u0rEd6uALg4BGJYq7ASCisGAQQBl1UBBQEBB0A5xq5f\\nUb1i2Ayvbt+ZFgxx+OL3KT12AkSkLcaAeRjKcQMBCAeIeAQYFggAIBYhBC+7wN+X\\n/b8eS3BO7eOe9PrEIL62BQJiWKuwAhsMAAoJEOOe9PrEIL624lEA/iyn0KNUx8AK\\nrSMLp7JawmsT+uD2pcw1uH3qZPHMja3gAP91/1vKQ8X5tEYzlE9OvVqtf9ESQBLj\\nzxMaMEdub5qiBQ==\\n=RCkT\\n-----END PGP PUBLIC KEY BLOCK-----\\n"
+  "ascii": "pub    256?/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 2022-04-16\\nuid\\t\\tMutlischleuder Test User <andy.example@example.org>\\nsub    256?/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 2022-04-16\\nsub    256?/C0E8ED7A32F53626F2FCDC65F5035A1D90E35CAE 2022-04-16\\n\\n\\n-----BEGIN PGP PUBLIC KEY BLOCK-----\\n\\nmDMEYlsHSBYJKwYBBAHaRw8BAQdAhGNoFKTXFsAOR8xiC7WWDB4gv+TZq5tmPG7X\\n8C3h4my0SU11dGxpc2NobGV1ZGVyIFRlc3QgVXNlciAoVEVTVCBLRVkgRE8gTk9U\\nIFVTRSkgPGFuZHkuZXhhbXBsZUBleGFtcGxlLm9yZz6IkAQTFggAOBYhBK25vGef\\n9TzI72b6w5NI/at6dmP5BQJiWwdIAhsjBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA\\nAAoJEJNI/at6dmP54NoBAMGktGRD7fgmruTviHhERbhUX9OmPGUuH1tsUFVAsePk\\nAP0Xt8Uq876t87FIMMil7zuo7Oc/lYqS+JONd0NEOIzUD7hXBGJbB0gSCSsGAQQB\\n2kcPAQIDBHhrny0kv/i58MlgJmR0g3dyadbPGt66Yht0dY6Azkz8eAbMuPG+Gqhu\\n/txLXnzPI1Gb99i934CCFUPgsvMorEIDAQgHiHgEGBYIACAWIQStubxnn/U8yO9m\\n+sOTSP2renZj+QUCYlsHSAIbDAAKCRCTSP2renZj+R9nAQDOcZRSgl9l7Z1inKjO\\nEwaQmYg/O9xked0C5mJwlV2mdgD9Gvamm5n6djU2D91X8Wbp49upWe1rAv2EgeAQ\\na5AcmwE=\\n=RIBQ\\n-----END PGP PUBLIC KEY BLOCK-----\\n\\n"
 }
 '''  # noqa E501
 
@@ -151,16 +151,16 @@ class TestSchleuderApi(unittest.TestCase):
         # Test request data
         self.assertEqual('https://localhost:4443/subscriptions.json?list_id=42',
                          mock.call_args_list[0][0][0].get_full_url())
-        self.assertEqual('https://localhost:4443/keys/2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6.json?list_id=42',
+        self.assertEqual('https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9.json?list_id=42',
                          mock.call_args_list[1][0][0].get_full_url())
         self.assertEqual(1, len(subs))
         self.assertEqual(23, subs[0].id)
-        self.assertEqual('foo@example.org', subs[0].email)
+        self.assertEqual('andy.example@example.org', subs[0].email)
         self.assertEqual(42, subs[0].schleuder)
         self.assertEqual(datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
                          subs[0].created_at)
-        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', subs[0].key.fingerprint)
+        self.assertEqual('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', subs[0].key.fingerprint)
-        self.assertEqual('foo@example.org', subs[0].key.email)
+        self.assertEqual('andy.example@example.org', subs[0].key.email)
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', subs[0].key.blob)
         self.assertEqual(42, subs[0].key.schleuder)
 
@@ -173,7 +173,7 @@ class TestSchleuderApi(unittest.TestCase):
                          mock.call_args_list[0][0][0].get_full_url())
         self.assertEqual(1, len(subs))
         self.assertEqual(24, subs[0].id)
-        self.assertEqual('foo@example.org', subs[0].email)
+        self.assertEqual('andy.example@example.org', subs[0].email)
         self.assertEqual(42, subs[0].schleuder)
         self.assertEqual(datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
                          subs[0].created_at)
@@ -182,10 +182,10 @@ class TestSchleuderApi(unittest.TestCase):
     @patch('urllib.request.urlopen')
     def test_get_subscriber(self, mock):
         api = self._mock_api(mock)
-        sub = api.get_subscriber('foo@example.org', SchleuderList(42, '', ''))
+        sub = api.get_subscriber('andy.example@example.org', SchleuderList(42, '', ''))
         self.assertEqual(23, sub.id)
-        self.assertEqual('foo@example.org', sub.email)
+        self.assertEqual('andy.example@example.org', sub.email)
-        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', sub.key.fingerprint)
+        self.assertEqual('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', sub.key.fingerprint)
         self.assertEqual(42, sub.key.schleuder)
         self.assertEqual(42, sub.schleuder)
         with self.assertRaises(KeyError):
@@ -194,17 +194,17 @@ class TestSchleuderApi(unittest.TestCase):
     @patch('urllib.request.urlopen')
     def test_get_subscriber_nokey(self, mock):
         api = self._mock_api(mock, nokey=True)
-        sub = api.get_subscriber('foo@example.org', SchleuderList(42, '', ''))
+        sub = api.get_subscriber('andy.example@example.org', SchleuderList(42, '', ''))
         self.assertEqual(24, sub.id)
-        self.assertEqual('foo@example.org', sub.email)
+        self.assertEqual('andy.example@example.org', sub.email)
         self.assertIsNone(sub.key)
 
     @patch('urllib.request.urlopen')
     def test_subscribe(self, mock):
         api = self._mock_api(mock)
         now = datetime.utcnow()
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
-        sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', key, 42, now)
         api.subscribe(sub, SchleuderList(42, '', ''))
         self.assertEqual('https://localhost:4443/subscriptions.json?list_id=42',
                          mock.call_args_list[-1][0][0].get_full_url())
@@ -215,7 +215,7 @@ class TestSchleuderApi(unittest.TestCase):
     def test_subscribe_nokey(self, mock):
         api = self._mock_api(mock)
         now = datetime.utcnow()
-        sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', None, 42, now)
         with self.assertRaises(ValueError):
             api.subscribe(sub, SchleuderList(42, '', ''))
 
@@ -223,8 +223,8 @@ class TestSchleuderApi(unittest.TestCase):
     def test_unsubscribe(self, mock):
         api = self._mock_api(mock)
         now = datetime.utcnow()
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
-        sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', key, 42, now)
         api.unsubscribe(sub, SchleuderList(42, '', ''))
         self.assertEqual('https://localhost:4443/subscriptions/23.json',
                          mock.call_args_list[-1][0][0].get_full_url())
@@ -235,8 +235,8 @@ class TestSchleuderApi(unittest.TestCase):
     def test_update_fingerprint(self, mock):
         api = self._mock_api(mock)
         now = datetime.utcnow()
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
-        sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', key, 42, now)
         api.update_fingerprint(sub, SchleuderList(42, '', ''))
         self.assertEqual('https://localhost:4443/subscriptions/23.json',
                          mock.call_args_list[-1][0][0].get_full_url())
@@ -247,25 +247,25 @@ class TestSchleuderApi(unittest.TestCase):
     def test_update_fingerprint_nokey(self, mock):
         api = self._mock_api(mock)
         now = datetime.utcnow()
-        sub = SchleuderSubscriber(23, 'foo@example.org', None, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', None, 42, now)
         with self.assertRaises(ValueError):
             api.update_fingerprint(sub, SchleuderList(42, '', ''))
 
     @patch('urllib.request.urlopen')
     def test_get_key(self, mock):
         api = self._mock_api(mock)
-        key = api.get_key('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', SchleuderList(42, '', ''))
+        key = api.get_key('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', SchleuderList(42, '', ''))
-        self.assertEqual('https://localhost:4443/keys/2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6.json?list_id=42',
+        self.assertEqual('https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9.json?list_id=42',
                          mock.call_args_list[0][0][0].get_full_url())
-        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', key.fingerprint)
+        self.assertEqual('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', key.fingerprint)
-        self.assertEqual('foo@example.org', key.email)
+        self.assertEqual('andy.example@example.org', key.email)
         self.assertEqual(42, key.schleuder)
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', key.blob)
 
     @patch('urllib.request.urlopen')
     def test_post_key(self, mock):
         api = self._mock_api(mock)
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
         api.post_key(key, SchleuderList(42, '', ''))
         self.assertEqual('https://localhost:4443/keys.json?list_id=42',
                          mock.call_args_list[0][0][0].get_full_url())
@@ -275,9 +275,9 @@ class TestSchleuderApi(unittest.TestCase):
     @patch('urllib.request.urlopen')
     def test_delete_key(self, mock):
         api = self._mock_api(mock)
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
         api.delete_key(key, SchleuderList(42, '', ''))
-        self.assertEqual('https://localhost:4443/keys/2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6.json?list_id=42',
+        self.assertEqual('https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9.json?list_id=42',
                          mock.call_args_list[0][0][0].get_full_url())
         self.assertEqual('DELETE', mock.call_args_list[0][0][0].method)
         # todo assert request payload
@@ -287,8 +287,8 @@ class TestSchleuderApi(unittest.TestCase):
         api = self._mock_api(mock)
         api.dry_run()
         now = datetime.utcnow()
-        key = SchleuderKey('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', 'foo@example.org', 'verylongpgpkeyblock', 42)
+        key = SchleuderKey('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', 'andy.example@example.org', 'verylongpgpkeyblock', 42)
-        sub = SchleuderSubscriber(23, 'foo@example.org', key, 42, now)
+        sub = SchleuderSubscriber(23, 'andy.example@example.org', key, 42, now)
         sch = SchleuderList(42, '', '')
         # create, update, delete should be no-ops; 5 requests for retrieving the keys & subscriptions in unsub & update
         api.subscribe(sub, sch)
@@ -300,5 +300,5 @@ class TestSchleuderApi(unittest.TestCase):
         # only reads should execute
         api.get_lists()
         api.get_subscribers(sch)
-        api.get_key('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', sch)
+        api.get_key('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', sch)
         self.assertLess(2, len(mock.call_args_list))

multischleuder/test/test_types.py

@@ -3,6 +3,7 @@ import datetime
 import json
 import unittest
 
+import pgpy  # type: ignore
 from dateutil.tz import tzutc
 
 from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
@@ -19,29 +20,33 @@ class TestSchleuderTypes(unittest.TestCase):
 
     def test_parse_key(self):
         k = SchleuderKey.from_api(42, **json.loads(_KEY_RESPONSE))
-        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', k.fingerprint)
+        self.assertEqual('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', k.fingerprint)
-        self.assertEqual('foo@example.org', k.email)
+        self.assertEqual('andy.example@example.org', k.email)
         self.assertEqual(42, k.schleuder)
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', k.blob)
-        self.assertIn('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6 (foo@example.org)', str(k))
+        self.assertIn('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 (andy.example@example.org)', str(k))
+        # Make sure the key can be used by PGPy
+        key, _ = pgpy.PGPKey.from_blob(k.blob)
+        msg = pgpy.PGPMessage.new('Hello World')
+        key.encrypt(msg)
 
     def test_parse_subscriber(self):
         k = SchleuderKey.from_api(42, **json.loads(_KEY_RESPONSE))
         s = SchleuderSubscriber.from_api(k, **json.loads(_SUBSCRIBER_RESPONSE)[0])
         self.assertEqual(23, s.id)
-        self.assertEqual('foo@example.org', str(s))
+        self.assertEqual('andy.example@example.org', str(s))
-        self.assertEqual('foo@example.org', s.email)
+        self.assertEqual('andy.example@example.org', s.email)
         self.assertEqual(42, s.schleuder)
         self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
                          s.created_at)
-        self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', s.key.fingerprint)
+        self.assertEqual('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9', s.key.fingerprint)
-        self.assertIn('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6 (foo@example.org)', str(k))
+        self.assertIn('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663F9 (andy.example@example.org)', str(k))
 
     def test_parse_subscriber_nokey(self):
         s = SchleuderSubscriber.from_api(None, **json.loads(_SUBSCRIBER_RESPONSE)[0])
         self.assertEqual(23, s.id)
-        self.assertEqual('foo@example.org', str(s))
+        self.assertEqual('andy.example@example.org', str(s))
-        self.assertEqual('foo@example.org', s.email)
+        self.assertEqual('andy.example@example.org', s.email)
         self.assertEqual(42, s.schleuder)
         self.assertEqual(datetime.datetime(2022, 4, 15, 1, 11, 12, 123000, tzinfo=tzutc()),
                          s.created_at)

multischleuder/types.py

@@ -1,5 +1,5 @@
 
-from typing import Optional
+from typing import List, Optional
 
 from dataclasses import dataclass, field, Field
 from datetime import datetime
@@ -55,7 +55,16 @@ class SchleuderKey:
                  email: str,
                  ascii: str,
                  *args, **kwargs) -> 'SchleuderKey':
-        return SchleuderKey(fingerprint, email, ascii, schleuder)
+        lines: List[str] = []
+        state = 0
+        for line in ascii.splitlines():
+            if '-----BEGIN PGP ' in line:
+                state = 1
+            if state == 1:
+                lines.append(line)
+            if '-----END PGP ' in line:
+                state = 1
+        return SchleuderKey(fingerprint, email, '\n'.join(lines), schleuder)
 
     def __str__(self) -> str:
         return f'{self.fingerprint} ({self.email})'

package/debian/multischleuder/etc/multischleuder/multischleuder.yml

@@ -0,0 +1,58 @@
+---
+
+api:
+  url: "https://localhost:4443"
+  token: 2d039a8cfe414e55d1ec9ce9d4d787afc27050a6f630a024ae6c7dc5ab6941e5
+  cafile: /etc/multischleuder/schleuder-ca.pem
+
+lists:
+
+  - target: global@schleuder.example.org
+    unmanaged:
+      - admin@example.org
+    banned:
+      - banned@example.org
+    sources:
+      - east@schleuder.example.org
+      - west@schleuder.example.org
+      - north@schleuder.example.org
+      - south@schleuder.example.org
+    from: global-owner@schleuder.example.org
+
+smtp:
+  hostname: localhost
+  port: 8025
+
+conflict:
+  interval: 604800  # 1 week
+  statefile: /var/lib/multischleuder/conflict.json
+  template: |
+    Hi {subscriber},
+    
+    While compiling the subscriber list of {schleuder}, your
+    address {subscriber} was subscribed on multiple sub-lists with
+    different PGP keys.  There may be something fishy or malicious going on,
+    or this may simply have been a mistake by you or a list admin.
+    
+    You have only been subscribed to {schleuder} using the key you
+    have been subscribed with for the *longest* time:
+      
+    {chosen}
+      
+    Please review the following keys and talk to the admins of the
+    corresponding sub-lists to resolve this issue:
+    
+    Fingerprint                               Sub-List
+    -----------                               --------
+    {affected}
+  
+    For your convenience, this message has been encrypted with *all* of the
+    above keys.  If you have any questions, or do not understand this
+    message, please refer to your local Schleuder admin, or reply to this
+    message.
+  
+    Note that this automated message is unsigned, since MultiSchleuder does
+    not have access to Schleuder private keys.
+  
+    Regards
+    MultiSchleuder {schleuder}

test/multischleuder.yml

@@ -0,0 +1,70 @@
+---
+
+api:
+  url: "https://localhost:4443"
+  token: "00000000000000000000000000000000"
+  cafile: /etc/schleuder/schleuder-certificate.pem
+
+lists:
+
+  - target: test-global@schleuder.example.org
+    unmanaged:
+      - admin@example.org
+      - admin2@example.org
+    banned:
+      - aspammer@example.org
+    sources:
+      - test-north@schleuder.example.org
+      - test-east@schleuder.example.org
+      - test-south@schleuder.example.org
+      - test-west@schleuder.example.org
+    from: test-global-owner@schleuder.example.org
+
+  - target: test2-global@schleuder.example.org
+    unmanaged:
+      - admin@example.org
+    banned:
+      - aspammer@example.org
+      - anotherspammer@example.org
+    sources:
+      - test-north@schleuder.example.org
+      - test-east@schleuder.example.org
+    from: test2-global-owner@schleuder.example.org
+
+smtp:
+  hostname: localhost
+  port: 25
+
+conflict:
+  interval: 3600  # 1 hour - you don't want this in production
+  statefile: conflict.json
+  template: |
+    Hi {subscriber},
+    
+    While compiling the subscriber list of {schleuder}, your
+    address {subscriber} was subscribed on multiple sub-lists with
+    different PGP keys.  There may be something fishy or malicious going on,
+    or this may simply have been a mistake by you or a list admin.
+    
+    You have only been subscribed to {schleuder} using the key you
+    have been subscribed with for the *longest* time:
+      
+    {chosen}
+      
+    Please review the following keys and talk to the admins of the
+    corresponding sub-lists to resolve this issue:
+    
+    Fingerprint                               Sub-List
+    -----------                               --------
+    {affected}
+  
+    For your convenience, this message has been encrypted with *all* of the
+    above keys.  If you have any questions, or do not understand this
+    message, please refer to your local Schleuder admin, or reply to this
+    message.
+  
+    Note that this automated message is unsigned, since MultiSchleuder does
+    not have access to Schleuder private keys.
+  
+    Regards
+    MultiSchleuder {schleuder}

test/prepare-schleuder.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+function gen_key {
+    echo "gen_key $@"
+    PUID="${1}"
+    shift 1
+    cat >/tmp/keygen <<EOF
+%no-protection
+%no-ask-passphrase
+%transient-key
+Key-Type: EDDSA
+Key-Curve: ed25519
+Subkey-Type: ECDH
+Subkey-Curve: ed25519
+Expire-Date: 0
+Name-Real: Mutlischleuder Test User
+Name-Comment: TEST KEY DO NOT USE
+Name-Email: ${PUID}
+EOF
+    gpg --batch --full-gen-key /tmp/keygen
+    for uid in $@; do
+	gpg --batch --quick-add-uid "${PUID}" "Mutlischleuder Test User (TEST KEY DO NOT USE) <${uid}>"
+    done
+    gpg --export --armor "${PUID}" > "/tmp/${PUID}.asc"
+    for uid in $@; do
+	gpg --export --armor "${uid}" > "/tmp/${uid}.asc"
+    done
+}
+
+function subscribe {
+    schleuder-cli subscriptions new "${1}" "${2}" "/tmp/${2}.asc"
+}
+
+gen_key admin@example.org
+gen_key admin2@example.org
+gen_key ada.lovelace@example.org
+gen_key alex.example@example.org
+gen_key aspammer@example.org
+gen_key anna.example@example.org
+mv /tmp/anna.example@example.org.asc /tmp/anna.example@example.org.old.asc
+
+gen_key anotherspammer@example.org
+gen_key andy.example@example.org
+mv /tmp/andy.example@example.org.asc /tmp/andy.example@example.org.1.asc
+gen_key aaron.example@example.org aaron.example@example.net
+gen_key amy.example@example.org
+
+install -m 0700 -d /tmp/gpg
+export GNUPGHOME=/tmp/gpg
+gen_key anna.example@example.org
+gen_key andy.example@example.org
+unset GNUPGHOME
+
+schleuder-cli lists new test@schleuder.example.org admin@example.org /tmp/admin@example.org.asc
+schleuder-cli lists new test-global@schleuder.example.org admin@example.org /tmp/admin@example.org.asc
+schleuder-cli lists new test-north@schleuder.example.org admin@example.org /tmp/admin@example.org.asc
+schleuder-cli lists new test-east@schleuder.example.org admin@example.org /tmp/admin@example.org.asc
+schleuder-cli lists new test-south@schleuder.example.org admin@example.org /tmp/admin@example.org.asc
+schleuder-cli lists new test-west@schleuder.example.org admin2@example.org /tmp/admin2@example.org.asc
+schleuder-cli lists new test2-global@schleuder.example.org admin2@example.org /tmp/admin2@example.org.asc
+
+subscribe test-global@schleuder.example.org ada.lovelace@example.org      # should be unsubscribed 
+subscribe test-global@schleuder.example.org aaron.example@example.org     # should remain as-is
+subscribe test-global@schleuder.example.org aaron.example@example.net     # should be unsubscribed, but key should remain
+subscribe test-global@schleuder.example.org alex.example@example.org      # should remain as-is
+schleuder-cli subscriptions new test-global@schleuder.example.org anna.example@example.org /tmp/anna.example@example.org.old.asc
+                                                                          # key should be updated
+subscribe test-global@schleuder.example.org aspammer@example.org          # should be unsubscribed
+
+subscribe test-north@schleuder.example.org alex.example@example.org       # should remain as-is
+subscribe test-north@schleuder.example.org aspammer@example.org           # should be ignored
+schleuder-cli subscriptions new test-north@schleuder.example.org arno.example@example.org
+                                                                          # should not be subscribed - no key
+
+subscribe test-east@schleuder.example.org anna.example@example.org        # key should be updated
+subscribe test-east@schleuder.example.org anotherspammer@example.org      # should not be subscribed
+subscribe test-east@schleuder.example.org aaron.example@example.org       # should remain as-is
+
+subscribe test-south@schleuder.example.org andy.example@example.org       # should be subscribed despite key conflict
+subscribe test-south@schleuder.example.org amy.example@example.org        # should be subscribed - conflict but same key
+
+sleep 5  # to get different subscription dates
+
+schleuder-cli subscriptions new test-west@schleuder.example.org andy.example@example.org /tmp/andy.example@example.org.1.asc
+                                                                          # should not be subscribed
+subscribe test-west@schleuder.example.org amy.example@example.org         # should be subscribed - conflict but same key

test/report.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo Expected:
+echo
+echo aaron.example@example.org
+echo admin@example.org
+echo alex.example@example.org
+echo amy.example@example.org
+echo andy.example@example.org
+echo anna.example@example.org
+echo anotherspammer@example.org
+echo -- ---
+echo Actual:
+echo
+schleuder-cli subscriptions list test-global@schleuder.example.org
+
+schleuder-cli keys list test-global@schleuder.example.org
+
+cat /var/spool/mail/root