Cleanup, proper project setup, codestyle & typechecking

.gitignore

@@ -1,2 +1,11 @@
+**/.idea/
+*.iml
+
+**/__pycache__/
+*.pyc
+**/*.egg-info/
+*.coverage
+**/.mypy_cache/
+
 ca.pem
 multischleuder.yml

.gitlab-ci.yml

@@ -0,0 +1,95 @@
+---
+image: python:3.9-bullseye
+
+stages:
+- test
+- build
+- deploy
+
+
+
+before_script:
+- pip3 install coverage pycodestyle mypy
+- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
+
+
+
+test:
+  stage: test
+  script:
+  - pip3 install -e .
+  - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
+  - python3 -m coverage combine
+  - python3 -m coverage report --rcfile=setup.cfg
+
+codestyle:
+  stage: test
+  script:
+  - pip3 install -e .
+  - pycodestyle multischleuder
+
+mypy:
+  stage: test
+  script:
+  - pip3 install -e .
+  - mypy --install-types
+  - mypy multischleuder
+
+
+
+build_wheel:
+  stage: build
+  script:
+  - python3 setup.py egg_info bdist_wheel
+  - cd dist
+  - sha256sum *.whl > SHA256SUMS
+  artifacts:
+    paths:
+    - "dist/*.whl"
+    - dist/SHA256SUMS
+  only:
+  - tags
+
+build_debian:
+  stage: build
+  script:
+  - apt update && apt install --yes lintian rsync sudo
+  - echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
+  - |
+    for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
+      echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
+      cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/  */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
+      echo "\n -- ${PACKAGE_AUTHOR}  $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
+    done
+  - gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
+  - python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
+  - cd package/debian
+  - sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
+  - mkdir -p multischleuder/usr/lib/python3/dist-packages/
+  - rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
+  - rm -rf multischleuder/usr/lib/python3.9/site-packages
+  - find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
+  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
+  - find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
+  - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
+  - find multischleuder -type f -exec chmod 0644 {} \;
+  - find multischleuder -type d -exec chmod 755 {} \;
+  - chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
+  - dpkg-deb --build multischleuder
+  - mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
+  - sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
+  - sha256sum *.deb > SHA256SUMS
+  artifacts:
+    paths:
+    - "package/debian/*.deb"
+    - package/debian/SHA256SUMS
+  only:
+  - tags
+
+
+release:
+  stage: deploy
+  script:
+  - python3 package/release.py
+  only:
+  - tags

multischleuder.py

@@ -1,404 +0,0 @@
-#!/usr/bin/env python3
-
-
-from typing import Any, Dict, List, Optional
-
-import argparse
-import base64
-import email
-import json
-import logging
-import os
-import re
-import urllib.request
-import smtplib
-import ssl
-import sys
-import hashlib
-
-import yaml
-import pgpy
-
-from dataclasses import dataclass, field, Field
-from datetime import datetime
-from dateutil.parser import isoparse
-
-
-LOG: logging.Logger = logging.getLogger('multischleuder')
-
-
-class SmtpServerConfig:
-    
-    def __init__(self, hostname: str, port: int, tls: bool, username: str, password: str):
-        self.hostname = hostname
-        self.port = port
-        self.tls = tls
-        self.username = username
-        self.password = password
-        self._smtp = None
-
-    @staticmethod
-    def parse(config: Dict[str, Any]) -> 'SmtpServerConfig':
-        tls = config.get('tls', 'PLAIN')
-        port = {
-            'SSL': 465,
-            'STARTTLS': 587
-        }.get(tls, 25)
-        return SmtpServerConfig(
-            hostname=config['hostname'],
-            port=config.get('port', port),
-            tls=tls,
-            username=config.get('username'),
-            password=config.get('password')
-        )
-
-    def __enter__(self):
-        cls = smtplib.SMTP if self.tls != 'SSL' else smtplib.SMTP_SSL
-        self._smtp = cls(self.hostname, self.port)
-        smtp = self._smtp.__enter__()
-        if self.tls == 'STARTTLS':
-            smtp.starttls()
-        if self.username is not None and self.password is not None:
-            smtp.login(self.username, self.password)
-        return smtp
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        ret = self._smtp.__exit__(exc_type, exc_val, exc_tb)
-        self._smtp = None
-        return ret
-
-    def send_message(self, msg):
-        if self._smtp is None:
-            raise RuntimeError('SMTP connection is not established')
-        self._smtp.send_message(msg)
-
-
-
-@dataclass
-class SchleuderSubscriber:
-    email: str
-    key: 'SchleuderKey'
-    sub_id: int
-    schleuder: 'SchleuderList'
-    created_at: datetime
-
-    def __str__(self) -> str:
-        return f'{self.email}'
-
-    def __hash__(self) -> int:
-        return hash(self.email)
-
-    def __eq__(self, o) -> bool:
-        if not isinstance(o, SchleuderSubscriber):
-            return False
-        return self.email == o.email
-
-
-@dataclass
-class SchleuderKey:
-    fingerprint: str
-    email: str
-    blob: Optional[str] = field(repr=False, default=None)
-    source_list: int
-
-    def __hash__(self) -> int:
-        return hash((self.fingerprint, self.email))
-
-    def __str__(self) -> str:
-        return f'{self.fingerprint} ({self.email})'
-
-
-@dataclass
-class SchleuderList:
-    id: int
-    name: str
-    fingerprint: str
-
-    def __hash__(self) -> int:
-        return hash(self.id)
-
-
-class ConflictMessage:
-
-    def __init__(self,
-                 multilist: 'MultiList',
-                 chosen: 'SchleuderSubscriber',
-                 affected: List['SchleuderSubscriber']):
-        self.multlist = multilist
-        self.chosen = chosen
-        self.affected = affected
-        # Generate a SHA1 digest that only changes when the subscription list changes
-        self.affected.sort(key=lambda x: x.created_at)
-        digest = hashlib.new('sha1')
-        digest.update(self.chosen.email)
-        digest.update(self.chosen.scheluder.name)
-        for affected in self.affected:
-            digest.update(affected.scheluder.name)
-            digest.update(affected.key.blob)
-        self.now = datetime.utcnow()
-        self.digest = digest.hexdigest()
-
-    def to_mime(self):
-        _chosen = f'{self.chosen.key.fingerprint}  {self.chosen.email}'
-        for affected in self.affected:
-            _affected += f'{affected.key.fingerprint}  {affected.schleuder.name}\n'
-        msg = self.multilist.conflict_message.format(
-            subscriber=self.chosen.email,
-            schleuder=self.multilist.list,
-            chosen=_chosen,
-            affected=_affected
-        )
-        pgp = pgpy.PGPMessage.new(msg)
-        # Encrypt the message to all keys
-        cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
-        sessionkey = cipher.gen_key()
-        try:
-            for affected in self.affected:
-                key, _ = pgpy.PGPKey.from_blob(self.affected.key.blob)
-                key._require_usage_flags = False
-                pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
-        finally:
-            del sessionkey
-        # Build the MIME message
-        mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
-        mp1['Content-Description'] = 'PGP/MIME version identification'
-        mp1['Content-Disposition'] = 'attachment'
-        mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
-        mp2['Content-Description'] = 'OpenPGP encrypted message'
-        mp2['Content-Disposition'] = 'inline; filename="message.asc"'
-        mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
-        mp0.attach(mp1)
-        mp0.attach(mp2)
-        mp0['Subject'] = f'MultiSchleuder {self.multilist.name} - Key Conflict'
-        mp0['From'] = self.multilist.conflict_from
-        mp0['Reply-To'] = self.multilist.conflict_from
-        mp0['To'] = self.chosen.address
-        mp0['Date'] = email.util.formatdate(self.no)
-        mp0['Auto-Submitted'] = 'auto-generated'
-        mp0['Precedence'] = 'list'
-        mp0['List-Id'] = f'<{self.multilist.list.replace("@", ".")}>'
-        mp0['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
-        mp0['X-MultiSchleuder-Digest'] = self.digest
-        return mp0
-        
-        
-class MultiList:
-
-    def __init__(self,
-                 ns: argparse.Namespace,
-                 list: str,
-                 source: List[str],
-                 unmanaged: List[str],
-                 banned: List[str],
-                 from_address: str,
-                 api_url: str,
-                 api_token: str,
-                 api_cafile: str,
-                 conflict_state_file: str,
-                 conflict_message: str,
-                 conflict_interval: int,
-                 conflict_smtp: SmtpServerConfig):
-        self.ns: argparse.Namespace = ns
-        self.list: str = list
-        self.source: List[str] = source
-        self.unmanaged: List[str] = unmanaged
-        self.banned: List[str] = banned
-        self.api_url: str = api_url
-        self.basic_auth = {'Authorization': b'Basic ' + base64.b64encode(f'schleuder:{api_token}'.encode())}
-        self.api_cafile: str = api_cafile
-        self.conflict_messages = []
-        self.conflict_from = from_address
-        self.conflict_state_file: str = conflict_state_file
-        self.conflict_message: str = conflict_message
-        self.conflict_interval: int = conflict_interval
-        self.conflict_smtp: SmtpServerConfig = conflict_smtp
-
-    def request(self, path: str, list_id: Optional[int] = None, data: Optional[str] = None, method: str = 'GET', fmt=[]):
-        if fmt:
-            path = path.format(*fmt)
-        url = os.path.join(self.api_url, path)
-        if list_id is not None:
-            url += f'?list_id={list_id}'
-        payload: Optional[bytes] = data.encode() if data is not None else None
-        context = ssl.create_default_context(cafile=self.api_cafile)
-        context.check_hostname = False
-        req = urllib.request.Request(url, data=payload, method=method, headers=self.basic_auth)
-        resp = urllib.request.urlopen(req, context=context)
-        return json.loads(resp.read().decode())
-
-    def resolve_lists(self) -> Dict[str, 'SchleuderList']:
-        response = self.request('lists.json')
-        lists = {}
-        for r in response:
-            l = SchleuderList(r['id'], r['email'], r['fingerprint'])
-            if l.name == self.list or l.name in self.source:
-                lists[l.name] = l
-        return lists
-
-    def get_list_subscribers(self, list: 'SchleuderList') -> Dict[str, 'SchleuderSubscriber']:
-        response = self.request('subscriptions.json', list.id)
-        subs = {}
-        for r in response:
-            key = SchleuderKey(r['fingerprint'], r['email'], list.id)
-            sub = SchleuderSubscriber(r['email'], key, isoparse(r['created_at']), list, r['id'])
-            subs[sub.email] = sub
-        return subs
-
-    def get_key(self, key: 'SchleuderKey') -> 'SchleuderKey':
-        if key.blob is not None:
-            return key
-        r = self.request('keys/{}.json', list_id=key.source_list, fmt=[key.fingerprint])
-        key.blob = r['ascii']
-        return key
-
-    def post_key(self, key: 'SchleuderKey', list: 'SchleuderList'):
-        if self.ns.dry_run:
-            return
-        if not key.blob:
-            raise RuntimeError('Key blob needs to be retrieved first')
-        data = json.dumps({
-            'keymaterial': key.blob
-        })
-        self.request('keys.json', list_id=list.id, data=data, method='POST')
-
-    def delete_key(self, key: SchleuderKey, list: 'SchleuderList'):
-        if self.ns.dry_run:
-            return
-        self.request('keys/{}.json', list_id=list.id, fmt=[key.fingerprint], method='DELETE')
-
-    def subscribe(self, sub: 'SchleuderSubscriber', list: 'SchleuderList'):
-        if self.ns.dry_run:
-            return
-        data = json.dumps({
-            'email': sub.email,
-            'fingerprint': sub.key.fingerprint
-        })
-        self.request('subscriptions.json', list_id=list.id, data=data, method='POST')
-
-    def unsubscribe(self, sub: 'SchleuderSubscriber'):
-        if self.ns.dry_run:
-            return
-        self.request('subscriptions/{}.json', fmt=[sub.sub_id], method='DELETE')
-
-    def resolve_subscriber_conflicts(self, subscriptions: List['SchleuderSubscription']) -> 'SchleuderSubscription':
-        if len(subscriptions) == 1:
-            return subscriptions[0]
-        earliest = min(subscriptions, key=lambda x: x.created_at)
-        LOG.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder.name}:')
-        for s in subscriptions:
-            LOG.debug(f'  - {s.schleuder.name}: {s.key.fingerprint}')
-        for s in subscriptions:
-            self.get_key(s.key, s.list.id)
-        self.conflict_messages.append(ConflictMessage(earliest, subscriptions))
-        return earliest
-
-    def send_conflict_messages(self):
-        now = datetime.utcnow().timestamp()
-        with open(self.conflict_state_file, 'a+') as f:
-            f.seek(0)
-            state = json.load(f)
-            # Remove all state entries older than conflict_interval
-            state = {k: v for k, v in state.items() if now-v < self.conflict_interval}
-            # Remove all messages not already sent recently
-            msgs = [m for m in self.conflict_messages if m.digest not in state]
-            # Add all remaining messages to state dict
-            for m in msgs:
-                state[m.digest] = now
-            if not self.ns.dry_run:
-                f.seek(0)
-                f.truncate()
-                json.dump(state)
-        # Finally send the mails
-        for m in msgs:
-            LOG.info(f'Sending key conflict message to {m["To"]}')
-            LOG.debug(f'MIME Message:\n{str(m)}')
-            if not self.ns.dry_run:
-                with self.smtp_config as smtp:
-                    smtp.send_message(m)
-        # Clear conflict messages
-        self.conflict_messages = []
-
-    def process(self):
-        LOG.info(f'Processing: {self.list} {"DRY RUN" if self.ns.dry_run else ""}')
-        # todo: conflict handling - what if a user is subscribed through 2 lists (and possibly with different keys?)
-        lists: Dict[str, SchleuderList] = self.resolve_lists()
-        target_list = lists[self.list]
-        # Get current subs, except for unmanaged adresses
-        current_subs = {s
-                        for s in self.get_list_subscribers(target_list).values()
-                        if s.email not in self.unmanaged}
-        current_keys = {s.key for s in current_subs}
-        intended_subs = set()
-        intended_keys = set()
-        all_subs = dict()
-        # This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists
-        for source in self.source:
-            for sub in self.get_list_subscribers(lists[source]).values():
-                # Don't include banned or unmanaged adresses
-                if sub.email in self.banned or s.email in self.unmanaged:
-                    continue
-                all_subs.setdefault(sub.email, []).append(sub)
-        # Remove 
-        for subs in all_subs.values():
-            sub = self.resolve_subscriber_conflicts(subs)
-            intended_subs.add(sub)
-            intended_keys.add(sub.key)
-        # Determine the change set
-        to_subscribe = intended_subs.difference(current_subs)
-        to_unsubscribe = current_subs.difference(intended_subs)
-        to_remove = current_keys.difference(intended_keys)
-        to_add = intended_keys.difference(current_keys)
-        # Retrieve actual key blobs
-        for key in to_add:
-            self.get_key(key)
-        # Perform the actual list modifications in an order where nothing breaks
-        for key in to_add:
-            self.post_key(key, target_list_id)
-            LOG.info(f'Added key: {key}')
-        for email in to_subscribe:
-            self.subscribe(email, target_list_id)
-            LOG.info(f'Subscribed user: {email}')
-        for sub_id in to_unsubscribe:
-            self.unsubscribe(sub_id)
-            LOG.info(f'Unsubscribed user: {sub_id}')
-        for key in to_remove:
-            self.delete_key(key, target_list_id)
-            LOG.info(f'Removed key: {key}')
-        self.send_conflict_messages()
-        if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
-            LOG.info(f'No changes for {self.list}')
-        else:
-            LOG.info(f'Finished processing: {self.list}')
-
-
-def parse_config(ns: argparse.Namespace) -> List['MultiList']:
-    with open(ns.config, 'r') as f:
-        y = yaml.safe_load(f)
-    lists = []
-    for l in y['lists']:
-        lc = MultiList(ns,
-                       l['name'], l['source'], l.get('unmanaged', []), l.get('banned', []),
-                       l.get('from'),
-                       y['api']['url'], y['api']['token'], y['api']['cafile'],
-                       y['conflict']['state'], y['conflict']['message'], y['conflict']['interval'],
-                       SmtpServerConfig.parse(y['conflict']['smtp']))
-        lists.append(lc)
-    return lists
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument('--config', '-c', type=str, default='/etc/multischleuder/config.yml')
-    ap.add_argument('--dry-run', '-n', action='store_true', default=False)
-    ap.add_argument('--verbose', '-v', action='store_true', default=False)
-    ns = ap.parse_args(sys.argv[1:])
-    if ns.verbose:
-        LOG.setLevel('DEBUG')
-        LOG.debug('Enabled verbose logging')
-    lists = parse_config(ns)
-    for l in lists:
-        l.process()
-
-
-if __name__ == '__main__':
-    main()

multischleuder/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1'

multischleuder/api.py

@@ -0,0 +1,107 @@
+
+from typing import List, Optional
+
+import base64
+import json
+import os
+import ssl
+import urllib.request
+
+from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
+
+
+class SchleuderApi:
+
+    def __init__(self,
+                 url: str,
+                 token: str,
+                 cafile: str):
+        self._url = url
+        self._cafile = cafile
+        self._dry_run = False
+        # The API token is not used as a Bearer token, but as the
+        # password for the static "schleuder" user.
+        auth = base64.b64encode(f'schleuder:{token}'.encode()).decode()
+        self._headers = {
+            'Authorization': f'Basic {auth}'
+        }
+
+    def __request(self,
+                  path: str,
+                  list_id: Optional[int] = None,
+                  data: Optional[str] = None,
+                  method: str = 'GET',
+                  fmt=[]):
+        # Compose and render fully-qualified URL
+        if fmt:
+            path = path.format(*fmt)
+        url = os.path.join(self._url, path)
+        if list_id is not None:
+            url += f'?list_id={list_id}'
+
+        payload: Optional[bytes] = data.encode() if data is not None else None
+        # Create a custom SSL context that does not validate the hostname, but
+        # validates against the self-signed schleuder-api-daemon certificate
+        context = ssl.create_default_context(cafile=self._cafile)
+        context.check_hostname = False
+        # Perform the actual request
+        req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
+        resp = urllib.request.urlopen(req, context=context)
+        return json.loads(resp.read().decode())
+
+    def dry_run(self):
+        self._dry_run = True
+
+    # List Management
+
+    def get_lists(self) -> List['SchleuderList']:
+        lists = self.__request('lists.json')
+        return [SchleuderList(**s) for s in lists]
+
+    # Subscriber Management
+
+    def get_subscribers(self, schleuder: 'SchleuderList') -> List['SchleuderSubscriber']:
+        response = self.__request('subscriptions.json', schleuder.id)
+        subs: List[SchleuderSubscriber] = []
+        for r in response:
+            key = self.__request('keys/{}.json', schleuder.id, fmt=[r['fingerprint']])
+            sub = SchleuderSubscriber.from_api(key, **r)
+            subs.append(sub)
+        return subs
+
+    def subscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'):
+        if self._dry_run:
+            return
+        data = json.dumps({
+            'email': sub.email,
+            'fingerprint': sub.key.fingerprint
+        })
+        self.__request('subscriptions.json', list_id=schleuder.id, data=data, method='POST')
+
+    def unsubscribe(self, sub: 'SchleuderSubscriber'):
+        if self._dry_run:
+            return
+        self.__request('subscriptions/{}.json', fmt=[sub.id], method='DELETE')
+
+    def update_fingerprint(self, sub: 'SchleuderSubscriber'):
+        if self._dry_run:
+            return
+        data = json.dumps({'fingerprint': sub.key.fingerprint})
+        self.__request('subscriptions/{}.json', fmt=[sub.id], data=data, method='PATCH')
+
+    # Key Management
+
+    def post_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
+        if self._dry_run:
+            return
+        if not key.blob:
+            raise ValueError('Cannot upload a key stub')
+        data = json.dumps({
+            'keymaterial': key.blob
+        })
+        self.__request('keys.json', list_id=schleuder.id, data=data, method='POST')
+
+    def delete_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
+        if self._dry_run:
+            return
+        self.__request('keys/{}.json', list_id=schleuder.id, fmt=[key.fingerprint], method='DELETE')

multischleuder/conflict.py

@@ -0,0 +1,179 @@
+
+from typing import Dict, List
+
+import email.mime.application
+import email.mime.multipart
+import email.utils
+import hashlib
+import json
+import logging
+import struct
+from collections import OrderedDict
+from datetime import datetime
+
+import pgpy  # type: ignore
+
+from multischleuder.types import SchleuderKey, SchleuderSubscriber
+from multischleuder.smtp import SmtpClient
+
+
+class ConflictMessage:
+
+    def __init__(self,
+                 schleuder: str,
+                 chosen: SchleuderSubscriber,
+                 affected: List[SchleuderSubscriber],
+                 mail_from: str,
+                 template: str):
+        self._schleuder: str = schleuder
+        self._chosen: SchleuderSubscriber = chosen
+        self._affected: List[SchleuderSubscriber] = affected
+        self._template: str = template
+        self._from: str = mail_from
+        # Generate a SHA1 digest that only changes when the subscription list changes
+        self._digest = self._make_digest()
+
+    def _make_digest(self) -> str:
+        # Sort so the hash stays the same if the set of subscriptions is the same.
+        # There is no guarantee that the subs are in any specific order.
+        subs: List[SchleuderSubscriber] = list(self._affected)
+        subs.sort(key=lambda x: x.schleuder)
+        h = hashlib.new('sha1')
+        # Include the chosen email an source sub-list
+        h.update(struct.pack('!sd',
+                             self._chosen.email,
+                             self._chosen.schleuder))
+        # Include all subscriptions, including the FULL key
+        for s in subs:
+            h.update(struct.pack('!ds',
+                                 s.schleuder,
+                                 s.key.blob))
+        return h.hexdigest()
+
+    @property
+    def digest(self) -> str:
+        return self._digest
+
+    @property
+    def mime(self) -> email.mime.multipart.MIMEMultipart:
+        # Render the message body
+        _chosen = f'{self._chosen.key.fingerprint}  {self._chosen.email}'
+        _affected = ''
+        for affected in self._affected:
+            _affected += f'{affected.key.fingerprint}  {affected.schleuder}\n'
+        msg: str = self._template.format(
+            subscriber=self._chosen.email,
+            schleuder=self._schleuder,
+            chosen=_chosen,
+            affected=_affected
+        )
+        pgp = pgpy.PGPMessage.new(msg)
+        # Encrypt the message to all keys
+        cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
+        sessionkey = cipher.gen_key()
+        try:
+            for affected in self._affected:
+                key, _ = pgpy.PGPKey.from_blob(affected.key.blob)
+                key._require_usage_flags = False
+                pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
+        finally:
+            del sessionkey
+        # Build the MIME message
+        # First the small "version" part ...
+        mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
+        mp1['Content-Description'] = 'PGP/MIME version identification'
+        mp1['Content-Disposition'] = 'attachment'
+        # ... then the actual encrypted payload ...
+        mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
+        mp2['Content-Description'] = 'OpenPGP encrypted message'
+        mp2['Content-Disposition'] = 'inline; filename="message.asc"'
+        # ... and finally the root multipart container
+        mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
+        mp0.attach(mp1)
+        mp0.attach(mp2)
+        # Set all the email headers
+        mp0['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
+        mp0['From'] = self._from
+        mp0['Reply-To'] = self._from
+        mp0['To'] = self._chosen.email
+        mp0['Date'] = email.utils.formatdate()
+        mp0['Auto-Submitted'] = 'auto-generated'
+        mp0['Precedence'] = 'list'
+        mp0['List-Id'] = f'<{self._schleuder.replace("@", ".")}>'
+        mp0['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
+        mp0['X-MultiSchleuder-Digest'] = self._digest
+        return mp0
+
+
+class KeyConflictResolution:
+
+    def __init__(self, smtp: 'SmtpClient', interval: int, statefile: str, template: str):
+        self._smtp = smtp
+        self._interval: int
+        self._state_file: str = statefile
+        self._template: str = statefile
+        self._messages: List[ConflictMessage] = []
+        self._logger: logging.Logger = logging.getLogger()
+
+    def resolve(self,
+                target: str,
+                mail_from: str,
+                subscriptions: List['SchleuderSubscriber']) -> List['SchleuderSubscriber']:
+        subs: Dict[str, List[SchleuderSubscriber]] = OrderedDict()
+        for s in subscriptions:
+            subs.setdefault(s.email, []).append(s)
+        # Perform conflict resolution for each set of subscribers with the same email
+        return [self._resolve(target, mail_from, s) for s in subs.values()]
+
+    def _resolve(self,
+                 target: str,
+                 mail_from: str,
+                 subscriptions: List['SchleuderSubscriber']) -> 'SchleuderSubscriber':
+        if len({s.email for s in subscriptions}) != 1:
+            raise ValueError('Number of unique subscriptions must be 1')
+        if len(subscriptions) == 1:
+            return subscriptions[0]
+        # Conflict Resolution: Choose the OLDEST subscriptions, but notify using ALL keys
+        earliest: SchleuderSubscriber = min(subscriptions, key=lambda x: x.created_at)
+        self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder}:')
+        for s in subscriptions:
+            self._logger.debug(f'  - {s.schleuder}: {s.key.fingerprint}')
+        # At this point, the messages are only queued locally, they will be sent afterwards
+        msg = ConflictMessage(
+            target,
+            earliest,
+            subscriptions,
+            self._template,
+            mail_from
+        )
+        self._messages.append(msg)
+        # Return the result of conflict resolution
+        return earliest
+
+    def send_messages(self, dry_run: bool = False):
+        now = int(datetime.utcnow().timestamp())
+        with open(self._state_file, 'a+') as f:
+            f.seek(0)
+            state: Dict[str, int] = json.load(f)
+            # Remove all state entries older than conflict_interval
+            state = {k: v for k, v in state.items() if now-v < self._interval}
+            # Remove all messages not already sent recently
+            msgs = [m for m in self._messages if m.digest not in state]
+            # Add all remaining messages to state dict
+            for m in msgs:
+                state[m.digest] = now
+            # Write the new state to file
+            if not dry_run:
+                f.seek(0)
+                f.truncate()
+                json.dump(state, f)
+        # Finally send the mails
+        with self._smtp as smtp:
+            for m in msgs:
+                msg = m.mime
+                self._logger.debug(f'MIME Message:\n{str(m)}')
+                if not dry_run:
+                    self._logger.info(f'Sending key conflict message to {msg["To"]}')
+                    smtp.send_message(msg)
+        # Clear conflict messages
+        self._messages = []

multischleuder/main.py

@@ -0,0 +1,69 @@
+
+from typing import Any, Dict, List
+
+import argparse
+
+import yaml
+
+from multischleuder.api import SchleuderApi
+from multischleuder.conflict import KeyConflictResolution
+from multischleuder.processor import MultiList
+from multischleuder.smtp import SmtpClient
+
+
+def parse_list_config(api: SchleuderApi,
+                      kcr: KeyConflictResolution,
+                      config: Dict[str, Any]) -> 'MultiList':
+    target = config['target']
+    default_from = target.replace('@', '-owner@')
+    mail_from = config.get('from', default_from)
+    banned = config.get('banned', [])
+    unmanaged = config.get('unmanaged', [])
+    return MultiList(
+        sources=config['sources'],
+        target=target,
+        banned=banned,
+        unmanaged=unmanaged,
+        mail_from=mail_from,
+        api=api,
+        kcr=kcr
+    )
+
+
+def parse_config(ns: argparse.Namespace) -> List['MultiList']:
+    with open(ns.config, 'r') as f:
+        c = yaml.safe_load(f)
+
+    api = SchleuderApi(**c['api'])
+    if ns.dry_run:
+        api.dry_run()
+
+    smtp_config = c.get('smtp', {})
+    smtp = SmtpClient.parse(**smtp_config)
+
+    kcr_config = c.get('conflict', {})
+    kcr = KeyConflictResolution(smtp, **kcr_config)
+
+    lists = []
+    for clist in c['lists']:
+        ml = parse_list_config(api, kcr, clist)
+        lists.append(ml)
+    return lists
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--config', '-c', type=str, default='/etc/multischleuder/config.yml')
+    ap.add_argument('--dry-run', '-n', action='store_true', default=False)
+    ap.add_argument('--verbose', '-v', action='store_true', default=False)
+    ns = ap.parse_args(sys.argv[1:])
+    if ns.verbose:
+        logger = logging.getLogger().setLevel('DEBUG')
+        logger.debug('Verbose logging enabled')
+    lists = parse_config(ns)
+    for lst in lists:
+        lst.process(ns.dry_run)
+
+
+if __name__ == '__main__':
+    main()

multischleuder/processor.py

@@ -0,0 +1,85 @@
+
+from typing import Dict, List, Set, Tuple
+
+import logging
+
+from multischleuder.api import SchleuderApi
+from multischleuder.conflict import KeyConflictResolution
+from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
+
+
+class MultiList:
+
+    def __init__(self,
+                 sources: List[str],
+                 target: str,
+                 unmanaged: List[str],
+                 banned: List[str],
+                 mail_from: str,
+                 api: SchleuderApi,
+                 kcr: KeyConflictResolution):
+        self._sources: List[str] = sources
+        self._target: str = target
+        self._unmanaged: List[str] = unmanaged
+        self._banned: List[str] = banned
+        self._mail_from: str = mail_from
+        self._api: SchleuderApi = api
+        self._kcr: KeyConflictResolution = kcr
+        self._logger: logging.Logger = logging.getLogger()
+
+    def process(self, dry_run: bool = False):
+        self._logger.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
+        target_list, sources = self._lists_by_name()
+        # Get current subs, except for unmanaged adresses
+        current_subs: Set[SchleuderSubscriber] = {
+            s
+            for s in self._api.get_subscribers(target_list)
+            if s.email not in self._unmanaged
+        }
+        current_keys: Set[SchleuderKey] = {s.key for s in current_subs}
+        all_subs: List[SchleuderSubscriber] = []
+        # This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists ...
+        for source in sources:
+            subs: List[SchleuderSubscriber] = self._api.get_subscribers(source)
+            for s in subs:
+                if s.email in self._banned or s.email in self._unmanaged:
+                    continue
+                all_subs.append(s)
+        # ... which is taken care of by the key conflict resolution routine
+        self._kcr.resolve(self._target, self._mail_from, all_subs)
+        intended_subs: Set[SchleuderSubscriber] = set(all_subs)
+        intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs}
+        # Determine the change set
+        to_subscribe = intended_subs.difference(current_subs)
+        to_unsubscribe = current_subs.difference(intended_subs)
+        to_remove = current_keys.difference(intended_keys)
+        to_add = intended_keys.difference(current_keys)
+        to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
+        # Perform the actual list modifications in an order which avoids race conditions
+        for key in to_add:
+            self._api.post_key(key, target_list)
+            self._logger.info(f'Added key: {key}')
+        for sub in to_subscribe:
+            self._api.subscribe(sub, target_list)
+            self._logger.info(f'Subscribed user: {sub}')
+        for sub in to_update:
+            self._api.update_fingerprint(sub)
+            self._logger.info(f'Updated key:     {sub}')
+        for sub in to_unsubscribe:
+            self._api.unsubscribe(sub)
+            self._logger.info(f'Unsubscribed user: {sub}')
+        for key in to_remove:
+            self._api.delete_key(key, target_list)
+            self._logger.info(f'Removed key: {key}')
+        # Finally, send any queued key conflict messages.
+        self._kcr.send_messages()
+        if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
+            self._logger.info(f'No changes for {self._target}')
+        else:
+            self._logger.info(f'Finished processing: {self._target}')
+
+    def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]:
+        lists = self._api.get_lists()
+        target = [x for x in lists if x.name == self._target][0]
+        sources = [x for x in lists if x.name in self._sources]
+        return target, sources

multischleuder/smtp.py

@@ -0,0 +1,81 @@
+
+from typing import Any, Dict, Optional
+
+import email
+import enum
+import logging
+import smtplib
+
+
+class TlsMode(enum.Enum):
+    PLAIN = TlsMode('smtp', 25)
+    SMTPS = TlsMode('smtps', 465)
+    STARTTLS = TlsMode('smtp+starttls', 587)
+
+    def __init__(self, proto: str, port: int):
+        self._proto = proto
+        self._port = port
+
+    @property
+    def proto(self):
+        return self._proto
+
+    @property
+    def port(self):
+        return self._port
+
+
+class SmtpClient:
+
+    def __init__(self, hostname: str, port: int, tls: 'TlsMode',
+                 username: Optional[str], password: Optional[str]):
+        self._hostname: str = hostname
+        self._port: int = port
+        self._tls: TlsMode = tls
+        self._username: Optional[str] = username
+        self._password: Optional[str] = password
+        self._smtp: Optional[smtplib.SMTP] = None
+        self._logger = logging.getLogger()
+
+    @staticmethod
+    def parse(config: Dict[str, Any]) -> 'SmtpClient':
+        tls = TlsMode[config.get('tls', 'PLAIN')]
+        return SmtpClient(
+            hostname=config.get('hostname', 'localhost'),
+            port=config.get('port', tls.port),
+            tls=tls,
+            username=config.get('username'),
+            password=config.get('password')
+        )
+
+    def send_message(self, msg: email.message.Message):
+        if self._smtp is None:
+            raise RuntimeError('SMTP connection is not established')
+        self._smtp.send_message(msg)
+        self._logger.debug(f'Sent email message.')
+
+    def __enter__(self):
+        # TLS from the start requires a different class
+        cls = smtplib.SMTP if self._tls != TlsMode.SMTPS else smtplib.SMTP_SSL
+        self._smtp = cls(self._hostname, self._port)
+        # Establish the connection
+        smtp = self._smtp.__enter__()
+        if self._tls == TlsMode.STARTTLS:
+            smtp.starttls()
+        # Only sign in if both username and password are provided
+        if self._username is not None and self._password is not None:
+            smtp.login(self._username, self._password)
+        self._logger.debug(f'SMTP connection to {str(self)} established')
+        return smtp
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._smtp is None:
+            raise RuntimeError('SMTP connection is not established')
+        self._smtp.quit()
+        ret = self._smtp.__exit__(exc_type, exc_val, exc_tb)
+        self._logger.debug(f'SMTP connection to {str(self)} closed')
+        self._smtp = None
+        return ret
+
+    def __str__(self) -> str:
+        return f'{self._tls.proto}://{self._username}@{self._hostname}:{self._port}'

multischleuder/types.py

@@ -0,0 +1,68 @@
+
+from dataclasses import dataclass, field, Field
+from datetime import datetime
+from dateutil.parser import isoparse
+
+
+@dataclass
+class SchleuderList:
+    id: int
+    name: str
+    fingerprint: str
+
+    @staticmethod
+    def from_api(id: int,
+                 email: str,
+                 fingerprint: str,
+                 *args, **kwargs) -> 'SchleuderList':
+        return SchleuderList(id, email, fingerprint)
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+
+@dataclass
+class SchleuderSubscriber:
+    id: int
+    email: str
+    key: 'SchleuderKey'
+    schleuder: int
+    created_at: datetime
+
+    @staticmethod
+    def from_api(key: 'SchleuderKey',
+                 id: int,
+                 list_id: int,
+                 email: str,
+                 fingerprint: str,
+                 admin: bool,
+                 delivery_enabled: bool,
+                 created_at: str,
+                 updated_at: str) -> 'SchleuderSubscriber':
+        created = isoparse(created_at)
+        return SchleuderSubscriber(id, email, key, list_id, created)
+
+    def __str__(self) -> str:
+        return f'{self.email}'
+
+    def __hash__(self) -> int:
+        return hash(self.email)
+
+    def __eq__(self, o) -> bool:
+        if not isinstance(o, SchleuderSubscriber):
+            return False
+        return self.email == o.email
+
+
+@dataclass
+class SchleuderKey:
+    fingerprint: str
+    email: str
+    blob: str = field(repr=False)
+    schleuder: int
+
+    def __hash__(self) -> int:
+        return hash((self.fingerprint, self.email))
+
+    def __str__(self) -> str:
+        return f'{self.fingerprint} ({self.email})'

package/debian/multischleuder/DEBIAN/conffiles

@@ -0,0 +1 @@
+/etc/multischleuder/multischleuder.yml

package/debian/multischleuder/DEBIAN/control

@@ -0,0 +1,11 @@
+Package: multischleuder
+Version: __MULTISCHLEUDER_VERSION__
+Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
+Section: web
+Priority: optional
+Architecture: all
+Depends: python3 (>= 3.6), python3-dateutil, python3-pgpy, python3-yaml
+Recommends: schleuder (>= 3.6)
+Description: Schleuder subscriber merging tool
+ Automatically and periodically merge subscribers and keys of multiple
+ Schleuder lists into one.

package/debian/multischleuder/DEBIAN/postinst

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "configure" ]]; then
+
+    if ! getent group multischleuder >/dev/null; then
+        groupadd --system multischleuder
+    fi
+
+    if ! getent passwd multischleuder >/dev/null; then
+        useradd --system --create-home --gid multischleuder --home-dir /var/lib/multischleuder --shell /usr/sbin/nologin multischleuder
+    fi
+
+    chown multischleuder:multischleuder /var/lib/multischleuder
+    chmod 0750 /var/lib/multischleuder
+
+    chown root:multischleuder /etc/multischleuder
+    chmod 0750 /etc/multischleuder
+
+    deb-systemd-helper enable multischleuder.timer
+    deb-systemd-invoke restart multischleuder.timer
+
+fi

package/debian/multischleuder/DEBIAN/postrm

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "remove" ]]; then
+
+    systemctl daemon-reload || true
+
+fi

package/debian/multischleuder/DEBIAN/prerm

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "remove" ]]; then
+
+    deb-systemd-invoke stop multischleuder.timer
+
+fi

package/debian/multischleuder/lib/systemd/system/multischleuder.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Multischleuder Sync Job
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/multischleuder --config=/etc/multischleuder/config.yaml
+User=multischleuder
+Group=multischleuder
+WorkingDirectory=/var/lib/multischleuder

package/debian/multischleuder/lib/systemd/system/multischleuder.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Multischleuder Sync Job
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]e
+WantedBy=timers.target

package/debian/multischleuder/usr/share/doc/multischleuder/copyright

@@ -0,0 +1,16 @@
+Copyright 2022 s3lph
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/release.py

@@ -0,0 +1,183 @@
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import os
+import sys
+import json
+import urllib.request
+import http.client
+from urllib.error import HTTPError
+
+
+def parse_changelog(tag: str) -> Optional[str]:
+    release_changelog: str = ''
+    with open('CHANGELOG.md', 'r') as f:
+        in_target: bool = False
+        done: bool = False
+        for line in f.readlines():
+            if in_target:
+                if f'<!-- END RELEASE {tag} -->' in line:
+                    done = True
+                    break
+                release_changelog += line
+            elif f'<!-- BEGIN RELEASE {tag} -->' in line:
+                in_target = True
+                continue
+    if not done:
+        return None
+    return release_changelog
+
+
+def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
+    url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
+    headers: Dict[str, str] = {
+        'Private-Token': api_token,
+        'User-Agent': 'curl/7.70.0'
+    }
+    req = urllib.request.Request(url, headers=headers)
+    try:
+        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
+    except HTTPError as e:
+        print(e.read().decode())
+        sys.exit(1)
+    resp_data: bytes = resp.read()
+    joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
+
+    jobidmap: Dict[str, str] = {}
+    for job in joblist:
+        name: str = job['name']
+        job_id: str = job['id']
+        jobidmap[name] = job_id
+    return jobidmap
+
+
+def fetch_single_shafile(url: str, api_token: str) -> str:
+    headers: Dict[str, str] = {
+        'User-Agent': 'curl/7.70.0',
+        'Private-Token': api_token
+    }
+    req = urllib.request.Request(url, headers=headers)
+    try:
+        resp: http.client.HTTPResponse = urllib.request.urlopen(req)
+    except HTTPError as e:
+        print(e.read().decode())
+        sys.exit(1)
+    resp_data: bytes = resp.readline()
+    shafile: str = resp_data.decode()
+    filename: str = shafile.strip().split(' ')[-1].strip()
+    return filename
+
+
+def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
+    mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
+    wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
+                         '/artifacts/dist/SHA256SUMS'
+    wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
+    wheel_url: str = f'{mybase}/dist/{wheel_filename}'
+    return wheel_url, wheel_sha_url
+
+
+def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
+    mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
+    debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
+                          '/artifacts/package/debian/SHA256SUMS'
+    debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
+    debian_url: str = f'{mybase}/package/debian/{debian_filename}'
+    return debian_url, debian_sha_url
+
+
+def main():
+    api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
+    release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
+    project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
+    project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
+    pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
+    if api_token is None:
+        print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
+        sys.exit(1)
+    if release_tag is None:
+        print('CI_COMMIT_TAG is not set.', file=sys.stderr)
+        sys.exit(1)
+    if project_name is None:
+        print('CI_PROJECT_PATH is not set.', file=sys.stderr)
+        sys.exit(1)
+    if project_id is None:
+        print('CI_PROJECT_ID is not set.', file=sys.stderr)
+        sys.exit(1)
+    if pipeline_id is None:
+        print('CI_PIPELINE_ID is not set.', file=sys.stderr)
+        sys.exit(1)
+
+    changelog: Optional[str] = parse_changelog(release_tag)
+    if changelog is None:
+        print('Changelog could not be parsed.', file=sys.stderr)
+        sys.exit(1)
+
+    job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
+
+    base_url: str = f'https://gitlab.com/{project_name}/-'
+
+    wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
+    debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
+
+    augmented_changelog = f'''{changelog.strip()}
+
+### Download
+
+- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
+- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
+    # Docker currently not working
+    # - Docker image: registry.gitlab.com/{project_name}:{release_tag}
+
+    post_body: str = json.dumps({
+        'tag_name': release_tag,
+        'description': augmented_changelog,
+        'assets': {
+            'links': [
+                {
+                    'name': 'Python Wheel',
+                    'url': wheel_url,
+                    'link_type': 'package'
+                },
+                {
+                    'name': 'Debian Package',
+                    'url': debian_url,
+                    'link_type': 'package'
+                }
+            ]
+        }
+    })
+
+    gitlab_release_api_url: str = \
+        f'https://gitlab.com/api/v4/projects/{project_id}/releases'
+    headers: Dict[str, str] = {
+        'Private-Token': api_token,
+        'Content-Type': 'application/json; charset=utf-8',
+        'User-Agent': 'curl/7.70.0'
+    }
+
+    request = urllib.request.Request(
+        gitlab_release_api_url,
+        post_body.encode('utf-8'),
+        headers=headers,
+        method='POST'
+    )
+    try:
+        response: http.client.HTTPResponse = urllib.request.urlopen(request)
+    except HTTPError as e:
+        print(e.read().decode())
+        sys.exit(1)
+    response_bytes: bytes = response.read()
+    response_str: str = response_bytes.decode()
+    response_data: Dict[str, Any] = json.loads(response_str)
+
+    if response_data['tag_name'] != release_tag:
+        print('Something went wrong...', file=sys.stderr)
+        print(response_str, file=sys.stderr)
+        sys.exit(1)
+
+    print(response_data['description'])
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -1,3 +0,0 @@
-dateutil
-PyYAML
-PGPy

setup.cfg

@@ -0,0 +1,22 @@
+
+#
+# PyCodestyle
+#
+
+[pycodestyle]
+max-line-length = 120
+statistics = True
+
+#
+# Coverage
+#
+
+[run]
+branch = True
+parallel = True
+source = multischleuder/
+
+[report]
+show_missing = True
+include = multischleuder/*
+omit = */test/*.py

setup.py

@@ -0,0 +1,29 @@
+from setuptools import setup, find_packages
+
+from multischleuder import __version__
+
+
+setup(
+    name='multischleuder',
+    version=__version__,
+    author='s3lph',
+    author_email='1375407-s3lph@users.noreply.gitlab.com',
+    description='Merge subscribers and keys of multiple Schleuder lists into one',
+    license='MIT',
+    keywords='schleuder,pgp',
+    url='https://gitlab.com/s3lph/multischleuder',
+    packages=find_packages(exclude=['*.test']),
+    install_requires=[
+        'python-dateutil',
+        'PyYAML',
+        'PGPy',
+    ],
+    test_requirements=[
+        'mypy'
+    ],
+    entry_points={
+        'console_scripts': [
+            'multischleuder = multischleuder.main:main'
+        ]
+    },
+)