feat: migrate from woodpecker to forgejo actions

Update CI pipeline to test against Schleuder 4

See merge request s3lph/multischleuder!4

.forgejo/workflows/package.yml

@@ -0,0 +1,38 @@
+---
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+
+  build_wheel:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Build Python wheel
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          python3 setup.py egg_info bdist_wheel
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3
+        with:
+          username: ${{ secrets.API_USERNAME }}
+          password: ${{ secrets.API_PASSWORD }}
+
+  build_debian:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5
+        with:
+          python_module: multischleuder
+          package_name: multischleuder
+          package_root: package/debian/multischleuder
+          package_output_path: package/debian
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2
+        with:
+          username: ${{ secrets.API_USERNAME }}
+          password: ${{ secrets.API_PASSWORD }}
+          deb: "package/debian/*.deb"

.forgejo/workflows/test.yml

@@ -0,0 +1,77 @@
+---
+
+on: push
+
+jobs:
+
+  test:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Unit Tests
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
+          python3 -m coverage combine
+          python3 -m coverage report --rcfile=setup.cfg
+
+  codestyle:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Code Style
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          pycodestyle multischleuder
+
+  mypy:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Static Type Checks
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          pip3 install --break-system-packages types-PyYAML types-python-dateutil
+          mypy multischleuder
+
+  schleuder:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Integration Test against schleuder
+        run: |
+          echo "postfix postfix/mailname string example.org" | debconf-set-selections
+          echo "postfix postfix/main_mailer_type string 'Local only'" | debconf-set-selections
+          apt update; apt install --yes python3-pip schleuder schleuder-cli postfix patch          
+          pip3 install --break-system-packages -e .[test]
+          /usr/lib/postfix/configure-instance.sh -
+          echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
+          /usr/sbin/postmulti -i - -p start
+          schleuder-cli lists list || true
+          export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
+          echo "  - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
+          cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
+          host: localhost
+          port: 4443
+          tls_fingerprint: ${CERT_FPR}
+          api_key: '00000000000000000000000000000000'
+          EOF
+          /usr/bin/schleuder-api-daemon &
+          sleep 5  # wait for daemons to start
+          export API_DAEMON_PID=$!
+          test/prepare-schleuder.sh
+          pip3 install --break-system-packages -e .
+          python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
+          python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
+          # Run a second time - should be idempotent and not trigger any new mails
+          python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
+          python3 -m coverage combine
+          python3 -m coverage report --rcfile=setup.cfg
+          sleep 5  # wait for mail delivery
+          test/report.py
+          kill -9 ${API_DAEMON_PID} || true
+          /usr/sbin/postmulti -i - -p stop
+          sleep 5  # wait for daemons to terminate

.gitlab-ci.yml

@@ -1,142 +0,0 @@
----
-image: python:3.9-bullseye
-
-stages:
-- test
-- coverage
-- build
-- deploy
-
-
-
-before_script:
-- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff
-- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
-
-
-
-test:
-  stage: test
-  script:
-  - pip3 install -e .
-  - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
-  artifacts:
-    paths:
-    - ".coverage*"
-
-codestyle:
-  stage: test
-  script:
-  - pip3 install -e .
-  - pycodestyle multischleuder
-
-mypy:
-  stage: test
-  script:
-  - pip3 install -e .
-  - mypy --install-types --non-interactive multischleuder
-  - mypy multischleuder
-
-schleuder:
-  stage: test
-  script:
-    - debconf-set-selections <<<"postfix postfix/mailname string example.org"
-    - debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'"
-    - apt update; apt install --yes schleuder schleuder-cli postfix
-    - /usr/lib/postfix/configure-instance.sh -
-    - echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
-    - /usr/sbin/postmulti -i - -p start
-    - schleuder-cli lists list || true
-    - export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
-    - echo "  - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
-    - |
-      cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
-      host: localhost
-      port: 4443
-      tls_fingerprint: ${CERT_FPR}
-      api_key: '00000000000000000000000000000000'
-      EOF
-    - /usr/bin/schleuder-api-daemon &
-    - sleep 5  # wait for daemons to start
-    - export API_DAEMON_PID=$!
-    - test/prepare-schleuder.sh
-    - pip3 install -e .
-    - python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
-    - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
-    # Run a second time - should be idempotent and not trigger any new mails
-    - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
-    - sleep 5  # wait for mail delivery
-    - test/report.py
-    - kill -9 ${API_DAEMON_PID} || true
-    - /usr/sbin/postmulti -i - -p stop
-    - sleep 5  # wait for daemons to terminate
-  artifacts:
-    paths:
-    - ".coverage*"
-
-
-
-coverage:
-  stage: coverage
-  script:
-  - python3 -m coverage combine
-  - python3 -m coverage report --rcfile=setup.cfg
-
-
-
-build_wheel:
-  stage: build
-  script:
-  - python3 setup.py egg_info bdist_wheel
-  - cd dist
-  - sha256sum *.whl > SHA256SUMS
-  artifacts:
-    paths:
-    - "dist/*.whl"
-    - dist/SHA256SUMS
-  only:
-  - tags
-
-build_debian:
-  stage: build
-  script:
-  - apt update && apt install --yes lintian rsync sudo
-  - echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-  - |
-    for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
-      echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-      cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/  */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-      echo "\n -- ${PACKAGE_AUTHOR}  $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-    done
-  - gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-  - python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
-  - cd package/debian
-  - sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
-  - mkdir -p multischleuder/usr/lib/python3/dist-packages/
-  - rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
-  - rm -rf multischleuder/usr/lib/python3.9/site-packages
-  - find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
-  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
-  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
-  - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
-  - find multischleuder -type f -exec chmod 0644 {} \;
-  - find multischleuder -type d -exec chmod 755 {} \;
-  - chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
-  - dpkg-deb --build multischleuder
-  - mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
-  - sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
-  - sha256sum *.deb > SHA256SUMS
-  artifacts:
-    paths:
-    - "package/debian/*.deb"
-    - package/debian/SHA256SUMS
-  only:
-  - tags
-
-
-release:
-  stage: deploy
-  script:
-  - python3 package/release.py
-  only:
-  - tags

CHANGELOG.md

@@ -1,5 +1,72 @@
 # MultiSchleuder Changelog
 
+<!-- BEGIN RELEASE v0.1.9 -->
+## Version 0.1.9
+
+Maintenance Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.9 -->
+- Migrate from Woodpecker to Forgejo Actions
+<!-- END CHANGES 0.1.9 -->
+
+<!-- END RELEASE v0.1.9 -->
+
+<!-- BEGIN RELEASE v0.1.8 -->
+## Version 0.1.8
+
+Maintenance Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.8 -->
+- Migrate from Gitlab-CI to Woodpecker
+<!-- END CHANGES 0.1.8 -->
+
+<!-- END RELEASE v0.1.8 -->
+
+<!-- BEGIN RELEASE v0.1.7 -->
+## Version 0.1.7
+
+Bugfix Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.7 -->
+- Remove and re-import keys whose expiry date has been changed
+- Don't report keys as changed if they appear to differ, but are treated as identical by GnuPG.
+<!-- END CHANGES 0.1.7 -->
+
+<!-- END RELEASE v0.1.7 -->
+
+<!-- BEGIN RELEASE v0.1.6 -->
+## Version 0.1.6
+
+Bugfix Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.6 -->
+- Better error handling for wrongfully configured keys
+<!-- END CHANGES 0.1.6 -->
+
+<!-- END RELEASE v0.1.6 -->
+
+<!-- BEGIN RELEASE v0.1.5 -->
+## Version 0.1.5
+
+Reporting changes
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.5 -->
+- Admin reports show the source sublist for each new subscriber
+- Add static code analysis CI jobs
+<!-- END CHANGES 0.1.5 -->
+
+<!-- END RELEASE v0.1.5 -->
+
 <!-- BEGIN RELEASE v0.1.4 -->
 ## Version 0.1.4
 

README.md

@@ -1,10 +1,5 @@
 # MultiSchleuder
 
-[![pipeline status](https://gitlab.com/s3lph/multischleuder/badges/main/pipeline.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
-[![coverage report](https://gitlab.com/s3lph/multischleuder/badges/main/coverage.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
-[![latest release](https://gitlab.com/s3lph/multischleuder/-/badges/release.svg)](https://gitlab.com/s3lph/multischleuder/-/releases)
-[![license](https://img.shields.io/badge/License-MIT-yellow.svg)](https://gitlab.com/s3lph/multischleuder/-/blob/main/LICENSE)
-
 Automatically and periodically merge subscribers and keys of multiple [Schleuder][schleuder] lists into one.
 
 ## Dependencies
@@ -15,7 +10,7 @@ Automatically and periodically merge subscribers and keys of multiple [Schleuder
 
 ## Installation
 
-You can find Debian packages and Python wheels over at [Releases][releases].
+You can find Debian packages and Python wheels over at [Packages][packages].
 
 ## Configuration
 
@@ -178,11 +173,11 @@ MultiSchleuder resolves conflicts in a simple, but primitive manner:
 1. Then it checks whether a subscriber has more than one key.  If so, the key used by the oldest subscription wins.
 
 This is by no means a perfect solution.
-It does however yield consisitent results.
+It does however yield consistent results.
 In both cases, if configured to do so, MultiSchleuder will send a notification message to all subscribers involved in a conflict, encrypting it with all keys involved in the conflict.
 If one or more keys are - for whatever reason - unusable, the message will not be encrypted.
-This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibilty would be to not notify a subscriber when something potentially malicious is going on.
+This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibility would be to not notify a subscriber when something potentially malicious is going on.
 
 
 [schleuder]: https://schleuder.org/
-[releases]: https://gitlab.com/s3lph/multischleuder/-/releases
+[packages]: https://git.kabelsalat.ch/s3lph/multischleuder/packages

multischleuder/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = '0.1.4'
+__version__ = '0.1.9'

multischleuder/api.py

@@ -3,8 +3,10 @@ from typing import List, Optional
 
 import base64
 import json
+import logging
 import os
 import ssl
+import urllib.error
 import urllib.request
 
 from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
@@ -49,7 +51,7 @@ class SchleuderApi:
             context = None
         # Perform the actual request
         req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
-        resp = urllib.request.urlopen(req, context=context)
+        resp = urllib.request.urlopen(req, context=context)  # nosec B310 baseurl is trusted
         respdata: str = resp.read().decode()
         if len(respdata) > 0:
             return json.loads(respdata)
@@ -129,8 +131,12 @@ class SchleuderApi:
 
     # Key Management
 
-    def get_key(self, fpr: str, schleuder: SchleuderList) -> SchleuderKey:
+    def get_key(self, fpr: str, schleuder: SchleuderList) -> Optional[SchleuderKey]:
-        key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
+        try:
+            key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
+        except urllib.error.HTTPError as e:
+            logging.exception(e)
+            return None
         return SchleuderKey.from_api(schleuder.id, **key)
 
     def post_key(self, key: SchleuderKey, schleuder: SchleuderList):

multischleuder/conflict.py

@@ -31,8 +31,7 @@ class KeyConflictResolution:
                 target: str,
                 mail_from: str,
                 subscriptions: List[SchleuderSubscriber],
-                sources: List[SchleuderList]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
+                sourcemap: Dict[int, str]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
-        sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
         conflicts: List[Optional[Message]] = []
 
         # First check for keys that are being used by more than one subscriber
@@ -174,7 +173,7 @@ class KeyConflictResolution:
         # Sort so the hash stays the same if the set of subscriptions is the same.
         # There is no guarantee that the subs are in any specific order.
         subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
-        h = hashlib.new('sha1')
+        h = hashlib.new('sha1')  # nosec B324
         # Include the chosen email an source sub-list
         h.update(struct.pack('!sd',
                              chosen.email.encode(),
@@ -191,7 +190,7 @@ class KeyConflictResolution:
         # Sort so the hash stays the same if the set of subscriptions is the same.
         # There is no guarantee that the subs are in any specific order.
         subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
-        h = hashlib.new('sha1')
+        h = hashlib.new('sha1')  # nosec B324
         assert chosen.key is not None  # Make mypy happy; it can't know that chosen.key can't be None
         # Include the chosen email an source sub-list
         h.update(struct.pack('!ssd',

multischleuder/processor.py

@@ -32,6 +32,7 @@ class MultiList:
     def process(self, dry_run: bool = False):
         logging.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
         target_list, sources = self._lists_by_name()
+        sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
         target_admins = self._api.get_list_admins(target_list)
         # Get current subs, except for unmanaged adresses
         current_subs: Set[SchleuderSubscriber] = set()
@@ -53,7 +54,7 @@ class MultiList:
                     continue
                 all_subs.append(s)
         # ... which is taken care of by the key conflict resolution routine
-        resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sources)
+        resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sourcemap)
         self._reporter.add_messages(conflicts)
         intended_subs: Set[SchleuderSubscriber] = set(resolved)
         intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None}
@@ -62,8 +63,14 @@ class MultiList:
         to_unsubscribe = current_subs.difference(intended_subs)
         to_remove = current_keys.difference(intended_keys)
         to_add = intended_keys.difference(current_keys)
+        # Already present keys that are being updated have to be removed and re-imported for convergence
+        to_pre_remove = {k for k in to_add if k.fingerprint in {o.fingerprint for o in to_remove}}
+        to_remove = {k for k in to_remove if k.fingerprint not in {o.fingerprint for o in to_pre_remove}}
         to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
         # Perform the actual list modifications in an order which avoids race conditions
+        for key in to_pre_remove:
+            self._api.delete_key(key, target_list)
+            logging.info(f'Pre-removed key:   {key}')
         for key in to_add:
             self._api.post_key(key, target_list)
             logging.info(f'Added key:         {key}')
@@ -80,6 +87,26 @@ class MultiList:
             self._api.delete_key(key, target_list)
             logging.info(f'Removed key:       {key}')
 
+        # Workaround for quirky gpg behaviour where some key signatures are exported from a sublist, but dropped on
+        # import into the target list, leading to a situation where the same key is imported over and over again.
+        new_subs = set()
+        # Get the new list of subscribers
+        for s in self._api.get_subscribers(target_list):
+            if s.email in self._unmanaged or s.email == self._target:
+                continue
+            if s.key is None or s.key.fingerprint == target_list.fingerprint:
+                continue
+            new_subs.add(s)
+        # Compare the key blobs to the ones present before this run
+        old_keys = {s.key.blob for s in current_subs if s.key is not None}
+        unchanged_subs = {s for s in new_subs if s.key is not None and s.key.blob in old_keys}
+        unchanged_fprs = {s.key.fingerprint for s in unchanged_subs if s.key is not None}
+        # Remove the unchanged keys from the changesets so that they are not included in the admin report
+        to_subscribe = {s for s in to_subscribe if s not in unchanged_subs}
+        to_update = {s for s in to_update if s not in unchanged_subs}
+        # need to compare by fpr because == includes the (potentially different) blob
+        to_add = {k for k in to_add if k.fingerprint not in unchanged_fprs}
+
         if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
             logging.info(f'No changes for {self._target}')
         else:
@@ -88,7 +115,7 @@ class MultiList:
                     report = AdminReport(self._target, admin.email, self._mail_from,
                                          admin.key.blob if admin.key is not None else None,
                                          to_subscribe, to_unsubscribe, to_update, to_add, to_remove,
-                                         conflicts)
+                                         conflicts, sourcemap)
                     self._reporter.add_message(report)
                 except BaseException:
                     logging.exception(f'Encryption to {admin.email} failed, not sending report')

multischleuder/reporting.py

@@ -208,7 +208,8 @@ class AdminReport(Message):
                  updated: Set[SchleuderSubscriber],
                  added: Set[SchleuderKey],
                  removed: Set[SchleuderKey],
-                 conflicts: List[Optional[Message]]):
+                 conflicts: List[Optional[Message]],
+                 sourcemap: Dict[int, str]):
         if len(subscribed) == 0 and len(unsubscribed) == 0 and len(removed) == 0 \
            and len(added) == 0 and len(updated) == 0 and len(conflicts) == 0:
             raise ValueError('No changes, not creating admin report')
@@ -233,15 +234,14 @@ class AdminReport(Message):
 >>> Subscribed:
 '''
             for s in subscribed:
-                fpr = 'no key' if s.key is None else s.key.fingerprint
+                sschleuder: str = sourcemap.get(s.schleuder, 'unknown')
-                content += f'{s.email} ({fpr})\n'
+                content += f'{s.email} ({sschleuder})\n'
         if len(unsubscribed) > 0:
             content += '''
 >>> Unsubscribed:
 '''
             for s in unsubscribed:
-                fpr = 'no key' if s.key is None else s.key.fingerprint
+                content += f'{s.email}\n'
-                content += f'{s.email} ({fpr})\n'
         if len(updated) > 0:
             content += '''
 >>> Subscriber keys changed:

multischleuder/test/test_api.py

@@ -1,5 +1,6 @@
 
 import unittest
+import urllib.error
 from datetime import datetime
 from unittest.mock import patch, MagicMock
 
@@ -111,7 +112,7 @@ _KEY_RESPONSE = '''
 
 class TestSchleuderApi(unittest.TestCase):
 
-    def _mock_api(self, mock, nokey=False):
+    def _mock_api(self, mock, nokey=False, error=None):
         m = MagicMock()
         m.getcode.return_value = 200
 
@@ -133,8 +134,14 @@ class TestSchleuderApi(unittest.TestCase):
         m.read = read
         m.__enter__.return_value = m
         mock.return_value = m
-        return SchleuderApi('https://localhost:4443',
+        api = SchleuderApi('https://localhost:4443',
-                            '86cf2676d065dc902548e563ab22b57868ed2eb6')
+                           '86cf2676d065dc902548e563ab22b57868ed2eb6')
+        if error is not None:
+            def __request(*args, **kwargs):
+                raise error
+            api._SchleuderApi__request = __request
+
+        return api
 
     @patch('urllib.request.urlopen')
     def test_get_lists(self, mock):
@@ -282,6 +289,13 @@ class TestSchleuderApi(unittest.TestCase):
         self.assertEqual(42, key.schleuder)
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', key.blob)
 
+    @patch('urllib.request.urlopen')
+    def test_get_key_404(self, mock):
+        url = 'https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA.json?list_id=42'
+        api = self._mock_api(mock, error=urllib.error.HTTPError(url=url, code=404, msg='Not Found', hdrs={}, fp=None))
+        key = api.get_key('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA', SchleuderList(42, '', ''))
+        self.assertIsNone(key)
+
     @patch('urllib.request.urlopen')
     def test_post_key(self, mock):
         api = self._mock_api(mock)

multischleuder/test/test_conflict.py

@@ -123,7 +123,7 @@ class TestKeyConflictResolution(unittest.TestCase):
         contents.seek(io.SEEK_END)  # Opened with 'a+'
         with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
             mock_statefile().__enter__.return_value = contents
-            resolved, messages = kcr.resolve('', '', [], [])
+            resolved, messages = kcr.resolve('', '', [], {})
         self.assertEqual(0, len(resolved))
         self.assertEqual(0, len(messages))
 
@@ -133,7 +133,7 @@ class TestKeyConflictResolution(unittest.TestCase):
         date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
         sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
         kcr = KeyConflictResolution(3600, '/tmp/state.json', _KEY_TEMPLATE, _USER_TEMPLATE)
-        resolved, messages = kcr.resolve('', '', [sub1], [sch1])
+        resolved, messages = kcr.resolve('', '', [sub1], {42: sch1})
         self.assertEqual(1, len(resolved))
         self.assertEqual(sub1, resolved[0])
         self.assertEqual(0, len(messages))
@@ -159,7 +159,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
@@ -187,7 +187,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint)
@@ -225,7 +225,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
@@ -246,7 +246,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1],
-                sources=[sch1])
+                sourcemap={42: sch1})
         self.assertEqual(0, len(resolved))
         self.assertEqual(0, len(messages))
 
@@ -271,7 +271,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('bar@example.org', resolved[0].email)
@@ -324,7 +324,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2, sub3],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(2, len(resolved))
         foo, bar = resolved
@@ -376,7 +376,7 @@ class TestKeyConflictResolution(unittest.TestCase):
             target='test@schleuder.example.org',
             mail_from='test-owner@schleuder.example.org',
             subscriptions=[sub1, sub2],
-            sources=[sch1, sch2])
+            sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(msgs))
 
     def test_send_messages_brokenstate(self):
@@ -399,7 +399,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(msgs))
 
     def test_send_messages_emptystate(self):
@@ -421,7 +421,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(msgs))
 
         now = datetime.utcnow().timestamp()
@@ -452,7 +452,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(msgs))
 
         now = datetime.utcnow().timestamp()
@@ -483,7 +483,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(messages))
         msg = messages[0]
 
@@ -515,7 +515,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(messages))
 
         now = datetime.utcnow().timestamp()
@@ -549,7 +549,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(messages))
 
         now = datetime.utcnow().timestamp()
@@ -582,7 +582,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2, sub3],
-                sources=[sch1, sch2, sch3])
+                sourcemap={42: sch1, 23: sch2, 7: sch1})
         self.assertEqual(1, len(messages))
         msg = messages[0].mime
         pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))

multischleuder/test/test_reporting.py

@@ -46,7 +46,8 @@ def one_of_each_kind():
         updated={},
         added={},
         removed={},
-        conflicts=[])
+        conflicts=[],
+        sourcemap={1: 'test@example.org'})
     msg3 = UserConflictMessage(
         schleuder='test@example.org',
         subscriber='bar@example.org',
@@ -124,4 +125,5 @@ class TestReporting(unittest.TestCase):
                 updated={},
                 added={},
                 removed={},
-                conflicts=[])
+                conflicts=[],
+                sourcemap={1: 'test@example.org'})

multischleuder/test/test_smtp.py

@@ -188,7 +188,8 @@ class TestSmtpClient(unittest.TestCase):
             updated={},
             added={},
             removed={},
-            conflicts=[])
+            conflicts=[],
+            sourcemap={1: 'foo@example.org'})
         client.send_messages([msg1, msg2])
         ctrl.stop()
         self.assertTrue(ctrl.handler.connected)

package/debian/multischleuder/DEBIAN/control

@@ -1,6 +1,6 @@
 Package: multischleuder
-Version: __MULTISCHLEUDER_VERSION__
+Version: __VERSION__
-Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
+Maintainer: s3lph <s3lph@kabelsalat.ch>
 Section: web
 Priority: optional
 Architecture: all

package/release.py

@@ -1,183 +0,0 @@
-
-from typing import Any, Dict, List, Optional, Tuple
-
-import os
-import sys
-import json
-import urllib.request
-import http.client
-from urllib.error import HTTPError
-
-
-def parse_changelog(tag: str) -> Optional[str]:
-    release_changelog: str = ''
-    with open('CHANGELOG.md', 'r') as f:
-        in_target: bool = False
-        done: bool = False
-        for line in f.readlines():
-            if in_target:
-                if f'<!-- END RELEASE {tag} -->' in line:
-                    done = True
-                    break
-                release_changelog += line
-            elif f'<!-- BEGIN RELEASE {tag} -->' in line:
-                in_target = True
-                continue
-    if not done:
-        return None
-    return release_changelog
-
-
-def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
-    url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
-    headers: Dict[str, str] = {
-        'Private-Token': api_token,
-        'User-Agent': 'curl/7.70.0'
-    }
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    resp_data: bytes = resp.read()
-    joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
-
-    jobidmap: Dict[str, str] = {}
-    for job in joblist:
-        name: str = job['name']
-        job_id: str = job['id']
-        jobidmap[name] = job_id
-    return jobidmap
-
-
-def fetch_single_shafile(url: str, api_token: str) -> str:
-    headers: Dict[str, str] = {
-        'User-Agent': 'curl/7.70.0',
-        'Private-Token': api_token
-    }
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    resp_data: bytes = resp.readline()
-    shafile: str = resp_data.decode()
-    filename: str = shafile.strip().split(' ')[-1].strip()
-    return filename
-
-
-def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
-    mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
-    wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
-                         '/artifacts/dist/SHA256SUMS'
-    wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
-    wheel_url: str = f'{mybase}/dist/{wheel_filename}'
-    return wheel_url, wheel_sha_url
-
-
-def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
-    mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
-    debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
-                          '/artifacts/package/debian/SHA256SUMS'
-    debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
-    debian_url: str = f'{mybase}/package/debian/{debian_filename}'
-    return debian_url, debian_sha_url
-
-
-def main():
-    api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
-    release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
-    project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
-    project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
-    pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
-    if api_token is None:
-        print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
-        sys.exit(1)
-    if release_tag is None:
-        print('CI_COMMIT_TAG is not set.', file=sys.stderr)
-        sys.exit(1)
-    if project_name is None:
-        print('CI_PROJECT_PATH is not set.', file=sys.stderr)
-        sys.exit(1)
-    if project_id is None:
-        print('CI_PROJECT_ID is not set.', file=sys.stderr)
-        sys.exit(1)
-    if pipeline_id is None:
-        print('CI_PIPELINE_ID is not set.', file=sys.stderr)
-        sys.exit(1)
-
-    changelog: Optional[str] = parse_changelog(release_tag)
-    if changelog is None:
-        print('Changelog could not be parsed.', file=sys.stderr)
-        sys.exit(1)
-
-    job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
-
-    base_url: str = f'https://gitlab.com/{project_name}/-'
-
-    wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
-    debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
-
-    augmented_changelog = f'''{changelog.strip()}
-
-### Download
-
-- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
-- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
-    # Docker currently not working
-    # - Docker image: registry.gitlab.com/{project_name}:{release_tag}
-
-    post_body: str = json.dumps({
-        'tag_name': release_tag,
-        'description': augmented_changelog,
-        'assets': {
-            'links': [
-                {
-                    'name': 'Python Wheel',
-                    'url': wheel_url,
-                    'link_type': 'package'
-                },
-                {
-                    'name': 'Debian Package',
-                    'url': debian_url,
-                    'link_type': 'package'
-                }
-            ]
-        }
-    })
-
-    gitlab_release_api_url: str = \
-        f'https://gitlab.com/api/v4/projects/{project_id}/releases'
-    headers: Dict[str, str] = {
-        'Private-Token': api_token,
-        'Content-Type': 'application/json; charset=utf-8',
-        'User-Agent': 'curl/7.70.0'
-    }
-
-    request = urllib.request.Request(
-        gitlab_release_api_url,
-        post_body.encode('utf-8'),
-        headers=headers,
-        method='POST'
-    )
-    try:
-        response: http.client.HTTPResponse = urllib.request.urlopen(request)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    response_bytes: bytes = response.read()
-    response_str: str = response_bytes.decode()
-    response_data: Dict[str, Any] = json.loads(response_str)
-
-    if response_data['tag_name'] != release_tag:
-        print('Something went wrong...', file=sys.stderr)
-        print(response_str, file=sys.stderr)
-        sys.exit(1)
-
-    print(response_data['description'])
-
-
-if __name__ == '__main__':
-    main()

setup.py

@@ -7,17 +7,27 @@ setup(
     name='multischleuder',
     version=__version__,
     author='s3lph',
-    author_email='1375407-s3lph@users.noreply.gitlab.com',
+    author_email='s3lph@kabelsalat.ch',
     description='Merge subscribers and keys of multiple Schleuder lists into one',
     license='MIT',
     keywords='schleuder,pgp',
-    url='https://gitlab.com/s3lph/multischleuder',
+    url='https://git.kabelsalat.ch/s3lph/multischleuder',
     packages=find_packages(exclude=['*.test']),
     install_requires=[
         'python-dateutil',
         'PyYAML',
         'PGPy',
     ],
+    extras_require={
+        'test': [
+            'aiosmtpd',
+            'coverage',
+            'pycodestyle',
+            'mypy',
+            'deepdiff',
+            'twine'
+        ]
+    },
     entry_points={
         'console_scripts': [
             'multischleuder = multischleuder.main:main'