feat: migrate from woodpecker to forgejo actions

Update CI pipeline to test against Schleuder 4

See merge request s3lph/multischleuder!4

.forgejo/workflows/package.yml

@@ -0,0 +1,38 @@
+---
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+
+  build_wheel:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Build Python wheel
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          python3 setup.py egg_info bdist_wheel
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3
+        with:
+          username: ${{ secrets.API_USERNAME }}
+          password: ${{ secrets.API_PASSWORD }}
+
+  build_debian:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5
+        with:
+          python_module: multischleuder
+          package_name: multischleuder
+          package_root: package/debian/multischleuder
+          package_output_path: package/debian
+      - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2
+        with:
+          username: ${{ secrets.API_USERNAME }}
+          password: ${{ secrets.API_PASSWORD }}
+          deb: "package/debian/*.deb"

.forgejo/workflows/test.yml

@@ -0,0 +1,77 @@
+---
+
+on: push
+
+jobs:
+
+  test:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Unit Tests
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
+          python3 -m coverage combine
+          python3 -m coverage report --rcfile=setup.cfg
+
+  codestyle:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Code Style
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          pycodestyle multischleuder
+
+  mypy:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Static Type Checks
+        run: |
+          apt update; apt install -y python3-pip
+          pip3 install --break-system-packages -e .[test]
+          pip3 install --break-system-packages types-PyYAML types-python-dateutil
+          mypy multischleuder
+
+  schleuder:
+    runs-on: docker
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Integration Test against schleuder
+        run: |
+          echo "postfix postfix/mailname string example.org" | debconf-set-selections
+          echo "postfix postfix/main_mailer_type string 'Local only'" | debconf-set-selections
+          apt update; apt install --yes python3-pip schleuder schleuder-cli postfix patch          
+          pip3 install --break-system-packages -e .[test]
+          /usr/lib/postfix/configure-instance.sh -
+          echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
+          /usr/sbin/postmulti -i - -p start
+          schleuder-cli lists list || true
+          export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
+          echo "  - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
+          cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
+          host: localhost
+          port: 4443
+          tls_fingerprint: ${CERT_FPR}
+          api_key: '00000000000000000000000000000000'
+          EOF
+          /usr/bin/schleuder-api-daemon &
+          sleep 5  # wait for daemons to start
+          export API_DAEMON_PID=$!
+          test/prepare-schleuder.sh
+          pip3 install --break-system-packages -e .
+          python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
+          python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
+          # Run a second time - should be idempotent and not trigger any new mails
+          python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
+          python3 -m coverage combine
+          python3 -m coverage report --rcfile=setup.cfg
+          sleep 5  # wait for mail delivery
+          test/report.py
+          kill -9 ${API_DAEMON_PID} || true
+          /usr/sbin/postmulti -i - -p stop
+          sleep 5  # wait for daemons to terminate

.gitlab-ci.yml

@@ -1,142 +0,0 @@
----
-image: python:3.9-bullseye
-
-stages:
-- test
-- coverage
-- build
-- deploy
-
-
-
-before_script:
-- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff
-- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
-
-
-
-test:
-  stage: test
-  script:
-  - pip3 install -e .
-  - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
-  artifacts:
-    paths:
-    - ".coverage*"
-
-codestyle:
-  stage: test
-  script:
-  - pip3 install -e .
-  - pycodestyle multischleuder
-
-mypy:
-  stage: test
-  script:
-  - pip3 install -e .
-  - mypy --install-types --non-interactive multischleuder
-  - mypy multischleuder
-
-schleuder:
-  stage: test
-  script:
-    - debconf-set-selections <<<"postfix postfix/mailname string example.org"
-    - debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'"
-    - apt update; apt install --yes schleuder schleuder-cli postfix
-    - /usr/lib/postfix/configure-instance.sh -
-    - echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
-    - /usr/sbin/postmulti -i - -p start
-    - schleuder-cli lists list || true
-    - export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
-    - echo "  - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
-    - |
-      cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
-      host: localhost
-      port: 4443
-      tls_fingerprint: ${CERT_FPR}
-      api_key: '00000000000000000000000000000000'
-      EOF
-    - /usr/bin/schleuder-api-daemon &
-    - sleep 5  # wait for daemons to start
-    - export API_DAEMON_PID=$!
-    - test/prepare-schleuder.sh
-    - pip3 install -e .
-    - python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
-    - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
-    # Run a second time - should be idempotent and not trigger any new mails
-    - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
-    - sleep 5  # wait for mail delivery
-    - test/report.py
-    - kill -9 ${API_DAEMON_PID} || true
-    - /usr/sbin/postmulti -i - -p stop
-    - sleep 5  # wait for daemons to terminate
-  artifacts:
-    paths:
-    - ".coverage*"
-
-
-
-coverage:
-  stage: coverage
-  script:
-  - python3 -m coverage combine
-  - python3 -m coverage report --rcfile=setup.cfg
-
-
-
-build_wheel:
-  stage: build
-  script:
-  - python3 setup.py egg_info bdist_wheel
-  - cd dist
-  - sha256sum *.whl > SHA256SUMS
-  artifacts:
-    paths:
-    - "dist/*.whl"
-    - dist/SHA256SUMS
-  only:
-  - tags
-
-build_debian:
-  stage: build
-  script:
-  - apt update && apt install --yes lintian rsync sudo
-  - echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-  - |
-    for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
-      echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-      cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/  */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-      echo "\n -- ${PACKAGE_AUTHOR}  $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-    done
-  - gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
-  - python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
-  - cd package/debian
-  - sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
-  - mkdir -p multischleuder/usr/lib/python3/dist-packages/
-  - rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
-  - rm -rf multischleuder/usr/lib/python3.9/site-packages
-  - find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
-  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
-  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
-  - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
-  - find multischleuder -type f -exec chmod 0644 {} \;
-  - find multischleuder -type d -exec chmod 755 {} \;
-  - chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
-  - dpkg-deb --build multischleuder
-  - mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
-  - sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
-  - sha256sum *.deb > SHA256SUMS
-  artifacts:
-    paths:
-    - "package/debian/*.deb"
-    - package/debian/SHA256SUMS
-  only:
-  - tags
-
-
-release:
-  stage: deploy
-  script:
-  - python3 package/release.py
-  only:
-  - tags

CHANGELOG.md

@@ -1,5 +1,101 @@
 # MultiSchleuder Changelog
 
+<!-- BEGIN RELEASE v0.1.9 -->
+## Version 0.1.9
+
+Maintenance Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.9 -->
+- Migrate from Woodpecker to Forgejo Actions
+<!-- END CHANGES 0.1.9 -->
+
+<!-- END RELEASE v0.1.9 -->
+
+<!-- BEGIN RELEASE v0.1.8 -->
+## Version 0.1.8
+
+Maintenance Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.8 -->
+- Migrate from Gitlab-CI to Woodpecker
+<!-- END CHANGES 0.1.8 -->
+
+<!-- END RELEASE v0.1.8 -->
+
+<!-- BEGIN RELEASE v0.1.7 -->
+## Version 0.1.7
+
+Bugfix Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.7 -->
+- Remove and re-import keys whose expiry date has been changed
+- Don't report keys as changed if they appear to differ, but are treated as identical by GnuPG.
+<!-- END CHANGES 0.1.7 -->
+
+<!-- END RELEASE v0.1.7 -->
+
+<!-- BEGIN RELEASE v0.1.6 -->
+## Version 0.1.6
+
+Bugfix Release
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.6 -->
+- Better error handling for wrongfully configured keys
+<!-- END CHANGES 0.1.6 -->
+
+<!-- END RELEASE v0.1.6 -->
+
+<!-- BEGIN RELEASE v0.1.5 -->
+## Version 0.1.5
+
+Reporting changes
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.5 -->
+- Admin reports show the source sublist for each new subscriber
+- Add static code analysis CI jobs
+<!-- END CHANGES 0.1.5 -->
+
+<!-- END RELEASE v0.1.5 -->
+
+<!-- BEGIN RELEASE v0.1.4 -->
+## Version 0.1.4
+
+Better error handling
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.4 -->
+- Processing failure in one list won't affect other lists
+- Admin reports are never sent unencrypted
+<!-- END CHANGES 0.1.4 -->
+
+<!-- END RELEASE v0.1.4 -->
+
+<!-- BEGIN RELEASE v0.1.3 -->
+## Version 0.1.3
+
+Bugfixes and small improvements in reporting
+
+### Changes
+
+<!-- BEGIN CHANGES 0.1.3 -->
+- RFC 3156 compliance: Don't base64-encode encrypted reports
+- Include conflicts in admin reports
+- Fix: user conflict message emails were sent multiple times to chosen email
+<!-- END CHANGES 0.1.3 -->
+
+<!-- END RELEASE v0.1.3 -->
+
 <!-- BEGIN RELEASE v0.1.2 -->
 ## Version 0.1.2
 

README.md

@@ -1,10 +1,5 @@
 # MultiSchleuder
 
-[![pipeline status](https://gitlab.com/s3lph/multischleuder/badges/main/pipeline.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
-[![coverage report](https://gitlab.com/s3lph/multischleuder/badges/main/coverage.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
-[![latest release](https://gitlab.com/s3lph/multischleuder/-/badges/release.svg)](https://gitlab.com/s3lph/multischleuder/-/releases)
-[![license](https://img.shields.io/badge/License-MIT-yellow.svg)](https://gitlab.com/s3lph/multischleuder/-/blob/main/LICENSE)
-
 Automatically and periodically merge subscribers and keys of multiple [Schleuder][schleuder] lists into one.
 
 ## Dependencies
@@ -15,7 +10,7 @@ Automatically and periodically merge subscribers and keys of multiple [Schleuder
 
 ## Installation
 
-You can find Debian packages and Python wheels over at [Releases][releases].
+You can find Debian packages and Python wheels over at [Packages][packages].
 
 ## Configuration
 
@@ -178,11 +173,11 @@ MultiSchleuder resolves conflicts in a simple, but primitive manner:
 1. Then it checks whether a subscriber has more than one key.  If so, the key used by the oldest subscription wins.
 
 This is by no means a perfect solution.
-It does however yield consisitent results.
+It does however yield consistent results.
 In both cases, if configured to do so, MultiSchleuder will send a notification message to all subscribers involved in a conflict, encrypting it with all keys involved in the conflict.
 If one or more keys are - for whatever reason - unusable, the message will not be encrypted.
-This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibilty would be to not notify a subscriber when something potentially malicious is going on.
+This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibility would be to not notify a subscriber when something potentially malicious is going on.
 
 
 [schleuder]: https://schleuder.org/
-[releases]: https://gitlab.com/s3lph/multischleuder/-/releases
+[packages]: https://git.kabelsalat.ch/s3lph/multischleuder/packages

multischleuder/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = '0.1.2'
+__version__ = '0.1.9'

multischleuder/api.py

@@ -3,8 +3,10 @@ from typing import List, Optional
 
 import base64
 import json
+import logging
 import os
 import ssl
+import urllib.error
 import urllib.request
 
 from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
@@ -49,7 +51,7 @@ class SchleuderApi:
             context = None
         # Perform the actual request
         req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
-        resp = urllib.request.urlopen(req, context=context)
+        resp = urllib.request.urlopen(req, context=context)  # nosec B310 baseurl is trusted
         respdata: str = resp.read().decode()
         if len(respdata) > 0:
             return json.loads(respdata)
@@ -129,8 +131,12 @@ class SchleuderApi:
 
     # Key Management
 
-    def get_key(self, fpr: str, schleuder: SchleuderList) -> SchleuderKey:
-        key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
+    def get_key(self, fpr: str, schleuder: SchleuderList) -> Optional[SchleuderKey]:
+        try:
+            key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
+        except urllib.error.HTTPError as e:
+            logging.exception(e)
+            return None
         return SchleuderKey.from_api(schleuder.id, **key)
 
     def post_key(self, key: SchleuderKey, schleuder: SchleuderList):

multischleuder/conflict.py

@@ -1,11 +1,6 @@
 
 from typing import Dict, List, Optional, Tuple
 
-import email.mime.base
-import email.mime.application
-import email.mime.multipart
-import email.mime.text
-import email.utils
 import hashlib
 import json
 import logging
@@ -36,8 +31,7 @@ class KeyConflictResolution:
                 target: str,
                 mail_from: str,
                 subscriptions: List[SchleuderSubscriber],
-                sources: List[SchleuderList]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
-        sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
+                sourcemap: Dict[int, str]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
         conflicts: List[Optional[Message]] = []
 
         # First check for keys that are being used by more than one subscriber
@@ -179,7 +173,7 @@ class KeyConflictResolution:
         # Sort so the hash stays the same if the set of subscriptions is the same.
         # There is no guarantee that the subs are in any specific order.
         subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
-        h = hashlib.new('sha1')
+        h = hashlib.new('sha1')  # nosec B324
         # Include the chosen email an source sub-list
         h.update(struct.pack('!sd',
                              chosen.email.encode(),
@@ -196,7 +190,7 @@ class KeyConflictResolution:
         # Sort so the hash stays the same if the set of subscriptions is the same.
         # There is no guarantee that the subs are in any specific order.
         subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
-        h = hashlib.new('sha1')
+        h = hashlib.new('sha1')  # nosec B324
         assert chosen.key is not None  # Make mypy happy; it can't know that chosen.key can't be None
         # Include the chosen email an source sub-list
         h.update(struct.pack('!ssd',

multischleuder/main.py

@@ -74,5 +74,8 @@ def main():
     logging.debug('Verbose logging enabled')
     lists, smtp = parse_config(ns)
     for lst in lists:
-        lst.process(ns.dry_run)
+        try:
+            lst.process(ns.dry_run)
+        except BaseException:
+            logging.exception(f'An error occurred while processing {lst._target}')
     smtp.send_messages(Reporter.get_messages())

multischleuder/processor.py

@@ -32,6 +32,7 @@ class MultiList:
     def process(self, dry_run: bool = False):
         logging.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
         target_list, sources = self._lists_by_name()
+        sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
         target_admins = self._api.get_list_admins(target_list)
         # Get current subs, except for unmanaged adresses
         current_subs: Set[SchleuderSubscriber] = set()
@@ -53,7 +54,7 @@ class MultiList:
                     continue
                 all_subs.append(s)
         # ... which is taken care of by the key conflict resolution routine
-        resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sources)
+        resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sourcemap)
         self._reporter.add_messages(conflicts)
         intended_subs: Set[SchleuderSubscriber] = set(resolved)
         intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None}
@@ -62,8 +63,14 @@ class MultiList:
         to_unsubscribe = current_subs.difference(intended_subs)
         to_remove = current_keys.difference(intended_keys)
         to_add = intended_keys.difference(current_keys)
+        # Already present keys that are being updated have to be removed and re-imported for convergence
+        to_pre_remove = {k for k in to_add if k.fingerprint in {o.fingerprint for o in to_remove}}
+        to_remove = {k for k in to_remove if k.fingerprint not in {o.fingerprint for o in to_pre_remove}}
         to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
         # Perform the actual list modifications in an order which avoids race conditions
+        for key in to_pre_remove:
+            self._api.delete_key(key, target_list)
+            logging.info(f'Pre-removed key:   {key}')
         for key in to_add:
             self._api.post_key(key, target_list)
             logging.info(f'Added key:         {key}')
@@ -80,14 +87,38 @@ class MultiList:
             self._api.delete_key(key, target_list)
             logging.info(f'Removed key:       {key}')
 
+        # Workaround for quirky gpg behaviour where some key signatures are exported from a sublist, but dropped on
+        # import into the target list, leading to a situation where the same key is imported over and over again.
+        new_subs = set()
+        # Get the new list of subscribers
+        for s in self._api.get_subscribers(target_list):
+            if s.email in self._unmanaged or s.email == self._target:
+                continue
+            if s.key is None or s.key.fingerprint == target_list.fingerprint:
+                continue
+            new_subs.add(s)
+        # Compare the key blobs to the ones present before this run
+        old_keys = {s.key.blob for s in current_subs if s.key is not None}
+        unchanged_subs = {s for s in new_subs if s.key is not None and s.key.blob in old_keys}
+        unchanged_fprs = {s.key.fingerprint for s in unchanged_subs if s.key is not None}
+        # Remove the unchanged keys from the changesets so that they are not included in the admin report
+        to_subscribe = {s for s in to_subscribe if s not in unchanged_subs}
+        to_update = {s for s in to_update if s not in unchanged_subs}
+        # need to compare by fpr because == includes the (potentially different) blob
+        to_add = {k for k in to_add if k.fingerprint not in unchanged_fprs}
+
         if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
             logging.info(f'No changes for {self._target}')
         else:
             for admin in target_admins:
-                report = AdminReport(self._target, admin.email, self._mail_from,
-                                     admin.key.blob if admin.key is not None else None,
-                                     to_subscribe, to_unsubscribe, to_update, to_add, to_remove)
-                self._reporter.add_message(report)
+                try:
+                    report = AdminReport(self._target, admin.email, self._mail_from,
+                                         admin.key.blob if admin.key is not None else None,
+                                         to_subscribe, to_unsubscribe, to_update, to_add, to_remove,
+                                         conflicts, sourcemap)
+                    self._reporter.add_message(report)
+                except BaseException:
+                    logging.exception(f'Encryption to {admin.email} failed, not sending report')
         logging.info(f'Finished processing: {self._target}')
 
     def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]:

multischleuder/reporting.py

@@ -2,6 +2,7 @@
 from typing import Dict, List, Optional, Set
 
 import abc
+import email.encoders
 import email.mime.base
 import email.mime.application
 import email.mime.multipart
@@ -26,23 +27,29 @@ class Message(abc.ABC):
                  mail_from: str,
                  mail_to: str,
                  content: str,
-                 encrypt_to: List[str]):
+                 encrypt_to: List[str],
+                 encrypt_may_fail: bool = False):
         self._schleuder: str = schleuder
         self._from: str = mail_from
         self._to: str = mail_to
         self._keys: List[str] = encrypt_to
-        self._mime: email.mime.base.MIMEBase = self._make_mime(content)
+        self._mime: email.mime.base.MIMEBase = self._make_mime(content, encrypt_may_fail)
 
     @property
     def mime(self) -> email.mime.base.MIMEBase:
         return self._mime
 
-    def _make_mime(self, content: str) -> email.mime.base.MIMEBase:
+    def _make_mime(self, content: str, encrypt_may_fail: bool) -> email.mime.base.MIMEBase:
         # Encrypt to all keys, if possible.  Fall back to unencrypted otherwise
         try:
             self._mime = self._encrypt_message(content)
-        except Exception:
-            self._mime = email.mime.text.MIMEText(content, _subtype='plain', _charset='utf-8')
+        except Exception as e:
+            if encrypt_may_fail:
+                logging.exception('Encryption failed; falling back to unencrypted message')
+                self._mime = email.mime.text.MIMEText(content, _subtype='plain', _charset='utf-8')
+            else:
+                logging.exception('Encryption failed; Not sending this message')
+                raise e
         # Set all the email headers
         self._mime['From'] = self._from
         self._mime['Reply-To'] = self._from
@@ -69,15 +76,21 @@ class Message(abc.ABC):
             del sessionkey
         # Build the MIME message
         # First the small "version" part ...
-        mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
+        mp1 = email.mime.application.MIMEApplication('Version: 1',
+                                                     _subtype='pgp-encrypted',
+                                                     _encoder=email.encoders.encode_noop)
         mp1['Content-Description'] = 'PGP/MIME version identification'
         mp1['Content-Disposition'] = 'attachment'
         # ... then the actual encrypted payload ...
-        mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
+        mp2 = email.mime.application.MIMEApplication(str(pgp),
+                                                     _subtype='octet-stream',
+                                                     _encoder=email.encoders.encode_noop,
+                                                     name='encrypted.asc')
         mp2['Content-Description'] = 'OpenPGP encrypted message'
         mp2['Content-Disposition'] = 'inline; filename="message.asc"'
         # ... and finally the root multipart container
-        mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
+        mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted',
+                                                 protocol='application/pgp-encrypted')
         mp0.attach(mp1)
         mp0.attach(mp2)
         return mp0
@@ -110,16 +123,28 @@ class KeyConflictMessage(Message):
         self._chosen: SchleuderSubscriber = chosen
         self._affected: List[SchleuderSubscriber] = affected
         self._template: str = template
+        self._sourcemap: Dict[int, str] = sourcemap
         super().__init__(
             schleuder=schleuder,
             mail_from=mail_from,
             mail_to=chosen.email,
             content=content,
-            encrypt_to=[s.key.blob for s in affected if s.key is not None]
+            encrypt_to=[s.key.blob for s in affected if s.key is not None],
+            encrypt_may_fail=True  # Permit unencrypted fallback so the user gets notified of the conflict anyway
         )
         self.mime['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
         self.mime['X-MultiSchleuder-Digest'] = digest
 
+    @property
+    def report_str(self) -> str:
+        fpr = 'no key' if self._chosen.key is None else self._chosen.key.fingerprint
+        s = f'{self._chosen.email} -> {fpr}\n'
+        for a in self._affected:
+            fpr = 'no key' if a.key is None else a.key.fingerprint
+            aschleuder: str = self._sourcemap.get(a.schleuder, 'unknown')
+            s += f'- {fpr}  ({aschleuder})\n'
+        return s
+
 
 class UserConflictMessage(Message):
 
@@ -149,16 +174,27 @@ class UserConflictMessage(Message):
         self._chosen: SchleuderSubscriber = chosen
         self._affected: List[SchleuderSubscriber] = affected
         self._template: str = template
+        self._sourcemap: Dict[int, str] = sourcemap
         super().__init__(
             schleuder=schleuder,
             mail_from=mail_from,
-            mail_to=chosen.email,
+            mail_to=subscriber,
             content=content,
-            encrypt_to=[chosen.key.blob]
+            encrypt_to=[chosen.key.blob],
+            encrypt_may_fail=True  # Permit unencrypted fallback so the user gets notified of the conflict anyway
         )
         self.mime['Subject'] = f'MultiSchleuder {self._schleuder} - Subscriber Conflict'
         self.mime['X-MultiSchleuder-Digest'] = digest
 
+    @property
+    def report_str(self) -> str:
+        assert self._chosen.key is not None
+        s = f'{self._chosen.key.fingerprint} -> {self._chosen.email}\n'
+        for a in self._affected:
+            aschleuder: str = self._sourcemap.get(a.schleuder, 'unknown')
+            s += f'- {a.email}  ({aschleuder})\n'
+        return s
+
 
 class AdminReport(Message):
 
@@ -171,10 +207,25 @@ class AdminReport(Message):
                  unsubscribed: Set[SchleuderSubscriber],
                  updated: Set[SchleuderSubscriber],
                  added: Set[SchleuderKey],
-                 removed: Set[SchleuderKey]):
-        if len(subscribed) == 0 and len(unsubscribed) == 0 and \
-           len(removed) == 0 and len(added) == 0 and len(updated) == 0:
+                 removed: Set[SchleuderKey],
+                 conflicts: List[Optional[Message]],
+                 sourcemap: Dict[int, str]):
+        if len(subscribed) == 0 and len(unsubscribed) == 0 and len(removed) == 0 \
+           and len(added) == 0 and len(updated) == 0 and len(conflicts) == 0:
             raise ValueError('No changes, not creating admin report')
+        key_conflicts: List[KeyConflictMessage] =\
+            [m for m in conflicts if m is not None and isinstance(m, KeyConflictMessage)]
+        _user_conflicts: Dict[str, UserConflictMessage] = {}
+        # Make sure the user conflicts are unique
+        for m in conflicts:
+            if m is None or not isinstance(m, UserConflictMessage):
+                continue
+            assert m._chosen.key is not None
+            fpr = m._chosen.key.fingerprint
+            if fpr not in _user_conflicts:
+                _user_conflicts[fpr] = m
+        user_conflicts: List[UserConflictMessage] = list(_user_conflicts.values())
+        # Assemble the content
         content = f'''
 == Admin Report for MultiSchleuder {schleuder} ==
 '''
@@ -183,15 +234,14 @@ class AdminReport(Message):
 >>> Subscribed:
 '''
             for s in subscribed:
-                fpr = 'no key' if s.key is None else s.key.fingerprint
-                content += f'{s.email} ({fpr})\n'
+                sschleuder: str = sourcemap.get(s.schleuder, 'unknown')
+                content += f'{s.email} ({sschleuder})\n'
         if len(unsubscribed) > 0:
             content += '''
 >>> Unsubscribed:
 '''
             for s in unsubscribed:
-                fpr = 'no key' if s.key is None else s.key.fingerprint
-                content += f'{s.email} ({fpr})\n'
+                content += f'{s.email}\n'
         if len(updated) > 0:
             content += '''
 >>> Subscriber keys changed:
@@ -211,6 +261,19 @@ class AdminReport(Message):
 '''
             for k in removed:
                 content += f'{k.fingerprint} ({k.email})\n'
+        if len(key_conflicts) > 0:
+            content += '''
+>>> Keys conflicts:
+'''
+            for c in key_conflicts:
+                content += f'{c.report_str}\n'
+        if len(user_conflicts) > 0:
+            content += '''
+>>> User conflicts:
+'''
+            for u in user_conflicts:
+                content += f'{u.report_str}\n'
+
         super().__init__(
             schleuder=schleuder,
             mail_from=mail_from,

multischleuder/test/test_api.py

@@ -1,5 +1,6 @@
 
 import unittest
+import urllib.error
 from datetime import datetime
 from unittest.mock import patch, MagicMock
 
@@ -111,7 +112,7 @@ _KEY_RESPONSE = '''
 
 class TestSchleuderApi(unittest.TestCase):
 
-    def _mock_api(self, mock, nokey=False):
+    def _mock_api(self, mock, nokey=False, error=None):
         m = MagicMock()
         m.getcode.return_value = 200
 
@@ -133,8 +134,14 @@ class TestSchleuderApi(unittest.TestCase):
         m.read = read
         m.__enter__.return_value = m
         mock.return_value = m
-        return SchleuderApi('https://localhost:4443',
-                            '86cf2676d065dc902548e563ab22b57868ed2eb6')
+        api = SchleuderApi('https://localhost:4443',
+                           '86cf2676d065dc902548e563ab22b57868ed2eb6')
+        if error is not None:
+            def __request(*args, **kwargs):
+                raise error
+            api._SchleuderApi__request = __request
+
+        return api
 
     @patch('urllib.request.urlopen')
     def test_get_lists(self, mock):
@@ -282,6 +289,13 @@ class TestSchleuderApi(unittest.TestCase):
         self.assertEqual(42, key.schleuder)
         self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', key.blob)
 
+    @patch('urllib.request.urlopen')
+    def test_get_key_404(self, mock):
+        url = 'https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA.json?list_id=42'
+        api = self._mock_api(mock, error=urllib.error.HTTPError(url=url, code=404, msg='Not Found', hdrs={}, fp=None))
+        key = api.get_key('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA', SchleuderList(42, '', ''))
+        self.assertIsNone(key)
+
     @patch('urllib.request.urlopen')
     def test_post_key(self, mock):
         api = self._mock_api(mock)

multischleuder/test/test_conflict.py

@@ -123,7 +123,7 @@ class TestKeyConflictResolution(unittest.TestCase):
         contents.seek(io.SEEK_END)  # Opened with 'a+'
         with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
             mock_statefile().__enter__.return_value = contents
-            resolved, messages = kcr.resolve('', '', [], [])
+            resolved, messages = kcr.resolve('', '', [], {})
         self.assertEqual(0, len(resolved))
         self.assertEqual(0, len(messages))
 
@@ -133,7 +133,7 @@ class TestKeyConflictResolution(unittest.TestCase):
         date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
         sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
         kcr = KeyConflictResolution(3600, '/tmp/state.json', _KEY_TEMPLATE, _USER_TEMPLATE)
-        resolved, messages = kcr.resolve('', '', [sub1], [sch1])
+        resolved, messages = kcr.resolve('', '', [sub1], {42: sch1})
         self.assertEqual(1, len(resolved))
         self.assertEqual(sub1, resolved[0])
         self.assertEqual(0, len(messages))
@@ -159,7 +159,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
@@ -187,7 +187,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint)
@@ -225,7 +225,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
@@ -246,7 +246,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1],
-                sources=[sch1])
+                sourcemap={42: sch1})
         self.assertEqual(0, len(resolved))
         self.assertEqual(0, len(messages))
 
@@ -271,7 +271,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(1, len(resolved))
         self.assertEqual('bar@example.org', resolved[0].email)
@@ -324,7 +324,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2, sub3],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
 
         self.assertEqual(2, len(resolved))
         foo, bar = resolved
@@ -376,7 +376,7 @@ class TestKeyConflictResolution(unittest.TestCase):
             target='test@schleuder.example.org',
             mail_from='test-owner@schleuder.example.org',
             subscriptions=[sub1, sub2],
-            sources=[sch1, sch2])
+            sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(msgs))
 
     def test_send_messages_brokenstate(self):
@@ -399,7 +399,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(msgs))
 
     def test_send_messages_emptystate(self):
@@ -421,7 +421,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(msgs))
 
         now = datetime.utcnow().timestamp()
@@ -452,7 +452,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(msgs))
 
         now = datetime.utcnow().timestamp()
@@ -483,7 +483,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(messages))
         msg = messages[0]
 
@@ -515,7 +515,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(0, len(messages))
 
         now = datetime.utcnow().timestamp()
@@ -549,7 +549,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2],
-                sources=[sch1, sch2])
+                sourcemap={42: sch1, 23: sch2})
         self.assertEqual(1, len(messages))
 
         now = datetime.utcnow().timestamp()
@@ -582,7 +582,7 @@ class TestKeyConflictResolution(unittest.TestCase):
                 target='test@schleuder.example.org',
                 mail_from='test-owner@schleuder.example.org',
                 subscriptions=[sub1, sub2, sub3],
-                sources=[sch1, sch2, sch3])
+                sourcemap={42: sch1, 23: sch2, 7: sch1})
         self.assertEqual(1, len(messages))
         msg = messages[0].mime
         pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))

multischleuder/test/test_multilist.py

@@ -9,6 +9,7 @@ from dateutil.tz import tzutc
 
 from multischleuder.processor import MultiList
 from multischleuder.reporting import Message
+from multischleuder.test.test_conflict import _PRIVKEY_1
 from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
 
 
@@ -38,7 +39,7 @@ def _list_lists():
 
 def _get_key(fpr: str, schleuder: SchleuderList):
     key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
-                        'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
+                        'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
     key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
                         'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
     key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',
@@ -72,7 +73,7 @@ def _get_admins(schleuder: SchleuderList):
     if schleuder.id != 2:
         return []
     key = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
-                       'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
+                       'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
     date = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
     admin = SchleuderSubscriber(0, 'admin@example.org', key, schleuder.id, date)
     return [admin]
@@ -80,7 +81,7 @@ def _get_admins(schleuder: SchleuderList):
 
 def _get_subs(schleuder: SchleuderList):
     key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
-                        'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
+                        'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
     key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
                         'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
     key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',

multischleuder/test/test_reporting.py

@@ -3,11 +3,27 @@ import unittest
 
 from datetime import datetime
 
+import pgpy.errors  # type: ignore
+
 from multischleuder.reporting import KeyConflictMessage, AdminReport, Reporter, UserConflictMessage
 from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
 from multischleuder.test.test_conflict import _PRIVKEY_1
 
 
+BROKENKEY = '''
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEYmcMbxYJKwYBBAHaRw8BAQdAKUohRdnuTSldKwawfLdwwUvOJjz/pHx3fXS2
+v2dUQx+0SU11bHRpc2NobGV1ZGVyIEJyb2tlbiBBZG1pbiBLZXkgKFRFU1QgS0VZ
+IERPIE5PVCBVU0UpIDxhZG1pbkBleGFtcGxlLm9yZz6IkAQTFggAOBYhBGtuFOnz
+PJOCOdfv6OuAwhfh1Uj8BQJiZwxvAhsBBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA
+AAoJEOuAwhfh1Uj8PnkBAM6PfYUZbvvYEkSdwzmZXDwhPRsSA0bhjL5aVwIeCCdp
+AQDeImNI6czSLVAuwObKv8FnpmbFi3HxTNzakp44DoD8Aw==
+=JtdI
+-----END PGP PUBLIC KEY BLOCK-----
+'''
+
+
 def one_of_each_kind():
     sub = SchleuderSubscriber(1, 'foo@example.org', None, 1, datetime.utcnow())
     key = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), 1)
@@ -24,12 +40,14 @@ def one_of_each_kind():
         schleuder='test@example.org',
         mail_to='admin@example.org',
         mail_from='test-owner@example.org',
-        encrypt_to=None,
+        encrypt_to=str(_PRIVKEY_1.pubkey),
         subscribed={},
         unsubscribed={sub},
         updated={},
         added={},
-        removed={})
+        removed={},
+        conflicts=[],
+        sourcemap={1: 'test@example.org'})
     msg3 = UserConflictMessage(
         schleuder='test@example.org',
         subscriber='bar@example.org',
@@ -93,3 +111,19 @@ class TestReporting(unittest.TestCase):
         r.add_messages([None])
         self.assertEqual(0, len(Reporter.get_messages()))
         Reporter.clear_messages()
+
+    def test_admin_report_nokey(self):
+        sub = SchleuderSubscriber(1, 'foo@example.org', None, 1, datetime.utcnow())
+        with self.assertRaises(pgpy.errors.PGPError):
+            AdminReport(
+                schleuder='test@example.org',
+                mail_to='admin@example.org',
+                mail_from='test-owner@example.org',
+                encrypt_to=BROKENKEY,
+                subscribed={sub},
+                unsubscribed={},
+                updated={},
+                added={},
+                removed={},
+                conflicts=[],
+                sourcemap={1: 'test@example.org'})

multischleuder/test/test_smtp.py

@@ -187,7 +187,9 @@ class TestSmtpClient(unittest.TestCase):
             unsubscribed={sub},
             updated={},
             added={},
-            removed={})
+            removed={},
+            conflicts=[],
+            sourcemap={1: 'foo@example.org'})
         client.send_messages([msg1, msg2])
         ctrl.stop()
         self.assertTrue(ctrl.handler.connected)

package/debian/multischleuder/DEBIAN/control

@@ -1,6 +1,6 @@
 Package: multischleuder
-Version: __MULTISCHLEUDER_VERSION__
-Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
+Version: __VERSION__
+Maintainer: s3lph <s3lph@kabelsalat.ch>
 Section: web
 Priority: optional
 Architecture: all

package/release.py

@@ -1,183 +0,0 @@
-
-from typing import Any, Dict, List, Optional, Tuple
-
-import os
-import sys
-import json
-import urllib.request
-import http.client
-from urllib.error import HTTPError
-
-
-def parse_changelog(tag: str) -> Optional[str]:
-    release_changelog: str = ''
-    with open('CHANGELOG.md', 'r') as f:
-        in_target: bool = False
-        done: bool = False
-        for line in f.readlines():
-            if in_target:
-                if f'<!-- END RELEASE {tag} -->' in line:
-                    done = True
-                    break
-                release_changelog += line
-            elif f'<!-- BEGIN RELEASE {tag} -->' in line:
-                in_target = True
-                continue
-    if not done:
-        return None
-    return release_changelog
-
-
-def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
-    url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
-    headers: Dict[str, str] = {
-        'Private-Token': api_token,
-        'User-Agent': 'curl/7.70.0'
-    }
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    resp_data: bytes = resp.read()
-    joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
-
-    jobidmap: Dict[str, str] = {}
-    for job in joblist:
-        name: str = job['name']
-        job_id: str = job['id']
-        jobidmap[name] = job_id
-    return jobidmap
-
-
-def fetch_single_shafile(url: str, api_token: str) -> str:
-    headers: Dict[str, str] = {
-        'User-Agent': 'curl/7.70.0',
-        'Private-Token': api_token
-    }
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    resp_data: bytes = resp.readline()
-    shafile: str = resp_data.decode()
-    filename: str = shafile.strip().split(' ')[-1].strip()
-    return filename
-
-
-def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
-    mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
-    wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
-                         '/artifacts/dist/SHA256SUMS'
-    wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
-    wheel_url: str = f'{mybase}/dist/{wheel_filename}'
-    return wheel_url, wheel_sha_url
-
-
-def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
-    mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
-    debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
-                          '/artifacts/package/debian/SHA256SUMS'
-    debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
-    debian_url: str = f'{mybase}/package/debian/{debian_filename}'
-    return debian_url, debian_sha_url
-
-
-def main():
-    api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
-    release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
-    project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
-    project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
-    pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
-    if api_token is None:
-        print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
-        sys.exit(1)
-    if release_tag is None:
-        print('CI_COMMIT_TAG is not set.', file=sys.stderr)
-        sys.exit(1)
-    if project_name is None:
-        print('CI_PROJECT_PATH is not set.', file=sys.stderr)
-        sys.exit(1)
-    if project_id is None:
-        print('CI_PROJECT_ID is not set.', file=sys.stderr)
-        sys.exit(1)
-    if pipeline_id is None:
-        print('CI_PIPELINE_ID is not set.', file=sys.stderr)
-        sys.exit(1)
-
-    changelog: Optional[str] = parse_changelog(release_tag)
-    if changelog is None:
-        print('Changelog could not be parsed.', file=sys.stderr)
-        sys.exit(1)
-
-    job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
-
-    base_url: str = f'https://gitlab.com/{project_name}/-'
-
-    wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
-    debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
-
-    augmented_changelog = f'''{changelog.strip()}
-
-### Download
-
-- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
-- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
-    # Docker currently not working
-    # - Docker image: registry.gitlab.com/{project_name}:{release_tag}
-
-    post_body: str = json.dumps({
-        'tag_name': release_tag,
-        'description': augmented_changelog,
-        'assets': {
-            'links': [
-                {
-                    'name': 'Python Wheel',
-                    'url': wheel_url,
-                    'link_type': 'package'
-                },
-                {
-                    'name': 'Debian Package',
-                    'url': debian_url,
-                    'link_type': 'package'
-                }
-            ]
-        }
-    })
-
-    gitlab_release_api_url: str = \
-        f'https://gitlab.com/api/v4/projects/{project_id}/releases'
-    headers: Dict[str, str] = {
-        'Private-Token': api_token,
-        'Content-Type': 'application/json; charset=utf-8',
-        'User-Agent': 'curl/7.70.0'
-    }
-
-    request = urllib.request.Request(
-        gitlab_release_api_url,
-        post_body.encode('utf-8'),
-        headers=headers,
-        method='POST'
-    )
-    try:
-        response: http.client.HTTPResponse = urllib.request.urlopen(request)
-    except HTTPError as e:
-        print(e.read().decode())
-        sys.exit(1)
-    response_bytes: bytes = response.read()
-    response_str: str = response_bytes.decode()
-    response_data: Dict[str, Any] = json.loads(response_str)
-
-    if response_data['tag_name'] != release_tag:
-        print('Something went wrong...', file=sys.stderr)
-        print(response_str, file=sys.stderr)
-        sys.exit(1)
-
-    print(response_data['description'])
-
-
-if __name__ == '__main__':
-    main()

setup.py

@@ -7,17 +7,27 @@ setup(
     name='multischleuder',
     version=__version__,
     author='s3lph',
-    author_email='1375407-s3lph@users.noreply.gitlab.com',
+    author_email='s3lph@kabelsalat.ch',
     description='Merge subscribers and keys of multiple Schleuder lists into one',
     license='MIT',
     keywords='schleuder,pgp',
-    url='https://gitlab.com/s3lph/multischleuder',
+    url='https://git.kabelsalat.ch/s3lph/multischleuder',
     packages=find_packages(exclude=['*.test']),
     install_requires=[
         'python-dateutil',
         'PyYAML',
         'PGPy',
     ],
+    extras_require={
+        'test': [
+            'aiosmtpd',
+            'coverage',
+            'pycodestyle',
+            'mypy',
+            'deepdiff',
+            'twine'
+        ]
+    },
     entry_points={
         'console_scripts': [
             'multischleuder = multischleuder.main:main'

test/prepare-schleuder.sh

@@ -44,6 +44,7 @@ gen_key andy.example@example.org
 mv /tmp/andy.example@example.org.asc /tmp/andy.example@example.org.1.asc
 gen_key aaron.example@example.org aaron.example@example.net
 gen_key amy.example@example.org
+gen_key alice.example@example.org alice.example@example.net
 
 install -m 0700 -d /tmp/gpg
 export GNUPGHOME=/tmp/gpg
@@ -75,6 +76,7 @@ schleuder-cli subscriptions new test-north@schleuder.example.org arno.example@ex
 subscribe test-east@schleuder.example.org anna.example@example.org        # key should be updated
 subscribe test-east@schleuder.example.org anotherspammer@example.org      # should not be subscribed
 subscribe test-east@schleuder.example.org aaron.example@example.org       # should remain as-is
+subscribe test-east@schleuder.example.org alice.example@example.net       # should be subscriped despite conflict
 
 subscribe test-south@schleuder.example.org andy.example@example.org       # should be subscribed despite key conflict
 subscribe test-south@schleuder.example.org amy.example@example.org        # should be subscribed - conflict but same key
@@ -84,6 +86,7 @@ sleep 5  # to get different subscription dates
 schleuder-cli subscriptions new test-west@schleuder.example.org andy.example@example.org /tmp/andy.example@example.org.1.asc
                                                                           # should not be subscribed
 subscribe test-west@schleuder.example.org amy.example@example.org         # should be subscribed - conflict but same key
+subscribe test-west@schleuder.example.org alice.example@example.org       # should not be subscriped - key used by other UID
 
 schleuder-cli subscriptions new test2-global@schleuder.example.org arno.example@example.org
                                                                           # should be unsubscribed - no key

test/report.py

@@ -38,6 +38,7 @@ expected_subscribers = subscribermap([
     'aaron.example@example.org',
     'admin@example.org',
     'alex.example@example.org',
+    'alice.example@example.net',
     'amy.example@example.org',
     'andy.example@example.org',
     'anna.example@example.org',
@@ -62,6 +63,7 @@ expected_keys = keymap([
     'aaron.example@example.org',
     'admin@example.org',
     'alex.example@example.org',
+    'alice.example@example.org',  # schleuder returns the primary UID
     'amy.example@example.org',
     'andy.example@example.org',
     'anna.example@example.org',
@@ -84,20 +86,23 @@ if len(keysdiff) > 0:
 # Test mbox
 
 mbox = mailbox.mbox('/var/spool/mail/root')
-if len(mbox) != 2:
-    print(f'Expected 2 messages in mbox, got {len(mbox)}')
+if len(mbox) != 4:
+    print(f'Expected 4 messages in mbox, got {len(mbox)}')
     exit(1)
-_, msg1 = mbox.popitem()
-_, msg2 = mbox.popitem()
+messages = []
+for i in range(4):
+    messages.append(mbox.popitem()[1])
 mbox.close()
-if 'X-MultiSchleuder-Digest' not in msg1:
-    msg1, msg2 = msg2, msg1
 
+msg1 = [m for m in messages if m['To'] == 'andy.example@example.org'][0]
+msg2 = [m for m in messages if m['To'] == 'admin@example.org'][0]
+msg3 = [m for m in messages if m['To'] == 'alice.example@example.org'][0]
+msg4 = [m for m in messages if m['To'] == 'alice.example@example.net'][0]
 
 if 'X-MultiSchleuder-Digest' not in msg1:
     print(f'Key conflict message should have a X-MultiSchleuder-Digest header, missing')
     exit(1)
-digest = msg1['X-MultiSchleuder-Digest'].strip()
+digest1 = msg1['X-MultiSchleuder-Digest'].strip()
 if msg1['From'] != 'test-global-owner@schleuder.example.org':
     print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg1["From"]}')
     exit(1)
@@ -127,27 +132,74 @@ if msg2['Precedence'] != 'list':
     print(f'Expected "Precedence: list", got {msg2["Precedence"]}')
     exit(1)
 
+if 'X-MultiSchleuder-Digest' not in msg3:
+    print(f'User conflict message should have a X-MultiSchleuder-Digest header, missing')
+    exit(1)
+digest3 = msg3['X-MultiSchleuder-Digest'].strip()
+if msg3['From'] != 'test-global-owner@schleuder.example.org':
+    print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg3["From"]}')
+    exit(1)
+if msg3['To'] != 'alice.example@example.org':
+    print(f'Expected "To: alice.example@example.org", got {msg3["To"]}')
+    exit(1)
+if msg3['Auto-Submitted'] != 'auto-generated':
+    print(f'Expected "Auto-Submitted: auto-generated", got {msg3["Auto-Submitted"]}')
+    exit(1)
+if msg3['Precedence'] != 'list':
+    print(f'Expected "Precedence: list", got {msg3["Precedence"]}')
+    exit(1)
+
+if 'X-MultiSchleuder-Digest' not in msg4:
+    print(f'User conflict message should have a X-MultiSchleuder-Digest header, missing')
+    exit(1)
+digest4 = msg4['X-MultiSchleuder-Digest'].strip()
+if msg4['From'] != 'test-global-owner@schleuder.example.org':
+    print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg4["From"]}')
+    exit(1)
+if msg4['To'] != 'alice.example@example.net':
+    print(f'Expected "To: alice.example@example.net", got {msg4["To"]}')
+    exit(1)
+if msg4['Auto-Submitted'] != 'auto-generated':
+    print(f'Expected "Auto-Submitted: auto-generated", got {msg4["Auto-Submitted"]}')
+    exit(1)
+if msg4['Precedence'] != 'list':
+    print(f'Expected "Precedence: list", got {msg4["Precedence"]}')
+    exit(1)
+    
+if digest3 != digest4:
+    print(f'User conflict messages should have the same digest, got: "{digest3}" vs "{digest4}"')
+    exit(1)
+
 gpg1 = subprocess.Popen(['/usr/bin/gpg', '-d'],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 gpg1o, _ = gpg1.communicate(msg1.get_payload()[1].get_payload(decode=True))
-print(f'Key conflict message (decrypted):\n{gpg1o.decode()}')
+print(f'\nKey conflict message (decrypted):\n{gpg1o.decode()}')
 gpg2 = subprocess.Popen(['/usr/bin/gpg', '-d'],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 gpg2o, _ = gpg2.communicate(msg2.get_payload()[1].get_payload(decode=True))
-print(f'Admin report message (decrypted):\n{gpg2o.decode()}')
+print(f'\nAdmin report message (decrypted):\n{gpg2o.decode()}')
+gpg3 = subprocess.Popen(['/usr/bin/gpg', '-d'],
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+gpg3o, _ = gpg3.communicate(msg3.get_payload()[1].get_payload(decode=True))
+print(f'\nUser conflict message (decrypted):\n{gpg3o.decode()}')
 
 # Test conflict statefile
 
 with open('/tmp/conflict.json', 'r') as f:
     conflict = json.load(f)
-if len(conflict) != 1:
+if len(conflict) != 2:
     print('Expected 1 entry in conflict statefile, got:')
     print(json.dumps(conflict))
     exit(1)
-if digest not in conflict:
-    print(f'Expected key "{digest}" in conflict statefile, got:')
+if digest1 not in conflict:
+    print(f'Expected key "{digest1}" in conflict statefile, got:')
+    print(json.dumps(conflict))
+    exit(1)
+if digest3 not in conflict:
+    print(f'Expected key "{digest3}" in conflict statefile, got:')
     print(json.dumps(conflict))
     exit(1)