Cleanup, proper project setup, codestyle & typechecking
This commit is contained in:
s3lph
parent
8e2a22a6f1
commit
61fcf7a5be
22 changed files with 1017 additions and 407 deletions
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -1,2 +1,11 @@
|
|||
**/.idea/
|
||||
*.iml
|
||||
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
**/*.egg-info/
|
||||
*.coverage
|
||||
**/.mypy_cache/
|
||||
|
||||
ca.pem
|
||||
multischleuder.yml
|
||||
95
.gitlab-ci.yml
Normal file
95
.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
---
|
||||
image: python:3.9-bullseye
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- deploy
|
||||
|
||||
|
||||
|
||||
before_script:
|
||||
- pip3 install coverage pycodestyle mypy
|
||||
- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
|
||||
|
||||
|
||||
|
||||
test:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
|
||||
- python3 -m coverage combine
|
||||
- python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
codestyle:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- pycodestyle multischleuder
|
||||
|
||||
mypy:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- mypy --install-types
|
||||
- mypy multischleuder
|
||||
|
||||
|
||||
|
||||
build_wheel:
|
||||
stage: build
|
||||
script:
|
||||
- python3 setup.py egg_info bdist_wheel
|
||||
- cd dist
|
||||
- sha256sum *.whl > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "dist/*.whl"
|
||||
- dist/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
build_debian:
|
||||
stage: build
|
||||
script:
|
||||
- apt update && apt install --yes lintian rsync sudo
|
||||
- echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
- |
|
||||
for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
|
||||
echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
done
|
||||
- gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
- python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
|
||||
- cd package/debian
|
||||
- sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
|
||||
- mkdir -p multischleuder/usr/lib/python3/dist-packages/
|
||||
- rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
|
||||
- rm -rf multischleuder/usr/lib/python3.9/site-packages
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
|
||||
- sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
|
||||
- find multischleuder -type f -exec chmod 0644 {} \;
|
||||
- find multischleuder -type d -exec chmod 755 {} \;
|
||||
- chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
|
||||
- dpkg-deb --build multischleuder
|
||||
- mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||
- sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||
- sha256sum *.deb > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "package/debian/*.deb"
|
||||
- package/debian/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
|
||||
release:
|
||||
stage: deploy
|
||||
script:
|
||||
- python3 package/release.py
|
||||
only:
|
||||
- tags
|
||||
|
|
@ -1,404 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import email
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
import smtplib
|
||||
import ssl
|
||||
import sys
|
||||
import hashlib
|
||||
|
||||
import yaml
|
||||
import pgpy
|
||||
|
||||
from dataclasses import dataclass, field, Field
|
||||
from datetime import datetime
|
||||
from dateutil.parser import isoparse
|
||||
|
||||
|
||||
LOG: logging.Logger = logging.getLogger('multischleuder')
|
||||
|
||||
|
||||
class SmtpServerConfig:
|
||||
|
||||
def __init__(self, hostname: str, port: int, tls: bool, username: str, password: str):
|
||||
self.hostname = hostname
|
||||
self.port = port
|
||||
self.tls = tls
|
||||
self.username = username
|
||||
self.password = password
|
||||
self._smtp = None
|
||||
|
||||
@staticmethod
|
||||
def parse(config: Dict[str, Any]) -> 'SmtpServerConfig':
|
||||
tls = config.get('tls', 'PLAIN')
|
||||
port = {
|
||||
'SSL': 465,
|
||||
'STARTTLS': 587
|
||||
}.get(tls, 25)
|
||||
return SmtpServerConfig(
|
||||
hostname=config['hostname'],
|
||||
port=config.get('port', port),
|
||||
tls=tls,
|
||||
username=config.get('username'),
|
||||
password=config.get('password')
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
cls = smtplib.SMTP if self.tls != 'SSL' else smtplib.SMTP_SSL
|
||||
self._smtp = cls(self.hostname, self.port)
|
||||
smtp = self._smtp.__enter__()
|
||||
if self.tls == 'STARTTLS':
|
||||
smtp.starttls()
|
||||
if self.username is not None and self.password is not None:
|
||||
smtp.login(self.username, self.password)
|
||||
return smtp
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
ret = self._smtp.__exit__(exc_type, exc_val, exc_tb)
|
||||
self._smtp = None
|
||||
return ret
|
||||
|
||||
def send_message(self, msg):
|
||||
if self._smtp is None:
|
||||
raise RuntimeError('SMTP connection is not established')
|
||||
self._smtp.send_message(msg)
|
||||
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderSubscriber:
|
||||
email: str
|
||||
key: 'SchleuderKey'
|
||||
sub_id: int
|
||||
schleuder: 'SchleuderList'
|
||||
created_at: datetime
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self.email}'
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.email)
|
||||
|
||||
def __eq__(self, o) -> bool:
|
||||
if not isinstance(o, SchleuderSubscriber):
|
||||
return False
|
||||
return self.email == o.email
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderKey:
|
||||
fingerprint: str
|
||||
email: str
|
||||
blob: Optional[str] = field(repr=False, default=None)
|
||||
source_list: int
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.fingerprint, self.email))
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self.fingerprint} ({self.email})'
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderList:
|
||||
id: int
|
||||
name: str
|
||||
fingerprint: str
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.id)
|
||||
|
||||
|
||||
class ConflictMessage:
|
||||
|
||||
def __init__(self,
|
||||
multilist: 'MultiList',
|
||||
chosen: 'SchleuderSubscriber',
|
||||
affected: List['SchleuderSubscriber']):
|
||||
self.multlist = multilist
|
||||
self.chosen = chosen
|
||||
self.affected = affected
|
||||
# Generate a SHA1 digest that only changes when the subscription list changes
|
||||
self.affected.sort(key=lambda x: x.created_at)
|
||||
digest = hashlib.new('sha1')
|
||||
digest.update(self.chosen.email)
|
||||
digest.update(self.chosen.scheluder.name)
|
||||
for affected in self.affected:
|
||||
digest.update(affected.scheluder.name)
|
||||
digest.update(affected.key.blob)
|
||||
self.now = datetime.utcnow()
|
||||
self.digest = digest.hexdigest()
|
||||
|
||||
def to_mime(self):
|
||||
_chosen = f'{self.chosen.key.fingerprint} {self.chosen.email}'
|
||||
for affected in self.affected:
|
||||
_affected += f'{affected.key.fingerprint} {affected.schleuder.name}\n'
|
||||
msg = self.multilist.conflict_message.format(
|
||||
subscriber=self.chosen.email,
|
||||
schleuder=self.multilist.list,
|
||||
chosen=_chosen,
|
||||
affected=_affected
|
||||
)
|
||||
pgp = pgpy.PGPMessage.new(msg)
|
||||
# Encrypt the message to all keys
|
||||
cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
|
||||
sessionkey = cipher.gen_key()
|
||||
try:
|
||||
for affected in self.affected:
|
||||
key, _ = pgpy.PGPKey.from_blob(self.affected.key.blob)
|
||||
key._require_usage_flags = False
|
||||
pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
|
||||
finally:
|
||||
del sessionkey
|
||||
# Build the MIME message
|
||||
mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
|
||||
mp1['Content-Description'] = 'PGP/MIME version identification'
|
||||
mp1['Content-Disposition'] = 'attachment'
|
||||
mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
|
||||
mp2['Content-Description'] = 'OpenPGP encrypted message'
|
||||
mp2['Content-Disposition'] = 'inline; filename="message.asc"'
|
||||
mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
|
||||
mp0.attach(mp1)
|
||||
mp0.attach(mp2)
|
||||
mp0['Subject'] = f'MultiSchleuder {self.multilist.name} - Key Conflict'
|
||||
mp0['From'] = self.multilist.conflict_from
|
||||
mp0['Reply-To'] = self.multilist.conflict_from
|
||||
mp0['To'] = self.chosen.address
|
||||
mp0['Date'] = email.util.formatdate(self.no)
|
||||
mp0['Auto-Submitted'] = 'auto-generated'
|
||||
mp0['Precedence'] = 'list'
|
||||
mp0['List-Id'] = f'<{self.multilist.list.replace("@", ".")}>'
|
||||
mp0['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
|
||||
mp0['X-MultiSchleuder-Digest'] = self.digest
|
||||
return mp0
|
||||
|
||||
|
||||
class MultiList:
|
||||
|
||||
def __init__(self,
|
||||
ns: argparse.Namespace,
|
||||
list: str,
|
||||
source: List[str],
|
||||
unmanaged: List[str],
|
||||
banned: List[str],
|
||||
from_address: str,
|
||||
api_url: str,
|
||||
api_token: str,
|
||||
api_cafile: str,
|
||||
conflict_state_file: str,
|
||||
conflict_message: str,
|
||||
conflict_interval: int,
|
||||
conflict_smtp: SmtpServerConfig):
|
||||
self.ns: argparse.Namespace = ns
|
||||
self.list: str = list
|
||||
self.source: List[str] = source
|
||||
self.unmanaged: List[str] = unmanaged
|
||||
self.banned: List[str] = banned
|
||||
self.api_url: str = api_url
|
||||
self.basic_auth = {'Authorization': b'Basic ' + base64.b64encode(f'schleuder:{api_token}'.encode())}
|
||||
self.api_cafile: str = api_cafile
|
||||
self.conflict_messages = []
|
||||
self.conflict_from = from_address
|
||||
self.conflict_state_file: str = conflict_state_file
|
||||
self.conflict_message: str = conflict_message
|
||||
self.conflict_interval: int = conflict_interval
|
||||
self.conflict_smtp: SmtpServerConfig = conflict_smtp
|
||||
|
||||
def request(self, path: str, list_id: Optional[int] = None, data: Optional[str] = None, method: str = 'GET', fmt=[]):
|
||||
if fmt:
|
||||
path = path.format(*fmt)
|
||||
url = os.path.join(self.api_url, path)
|
||||
if list_id is not None:
|
||||
url += f'?list_id={list_id}'
|
||||
payload: Optional[bytes] = data.encode() if data is not None else None
|
||||
context = ssl.create_default_context(cafile=self.api_cafile)
|
||||
context.check_hostname = False
|
||||
req = urllib.request.Request(url, data=payload, method=method, headers=self.basic_auth)
|
||||
resp = urllib.request.urlopen(req, context=context)
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def resolve_lists(self) -> Dict[str, 'SchleuderList']:
|
||||
response = self.request('lists.json')
|
||||
lists = {}
|
||||
for r in response:
|
||||
l = SchleuderList(r['id'], r['email'], r['fingerprint'])
|
||||
if l.name == self.list or l.name in self.source:
|
||||
lists[l.name] = l
|
||||
return lists
|
||||
|
||||
def get_list_subscribers(self, list: 'SchleuderList') -> Dict[str, 'SchleuderSubscriber']:
|
||||
response = self.request('subscriptions.json', list.id)
|
||||
subs = {}
|
||||
for r in response:
|
||||
key = SchleuderKey(r['fingerprint'], r['email'], list.id)
|
||||
sub = SchleuderSubscriber(r['email'], key, isoparse(r['created_at']), list, r['id'])
|
||||
subs[sub.email] = sub
|
||||
return subs
|
||||
|
||||
def get_key(self, key: 'SchleuderKey') -> 'SchleuderKey':
|
||||
if key.blob is not None:
|
||||
return key
|
||||
r = self.request('keys/{}.json', list_id=key.source_list, fmt=[key.fingerprint])
|
||||
key.blob = r['ascii']
|
||||
return key
|
||||
|
||||
def post_key(self, key: 'SchleuderKey', list: 'SchleuderList'):
|
||||
if self.ns.dry_run:
|
||||
return
|
||||
if not key.blob:
|
||||
raise RuntimeError('Key blob needs to be retrieved first')
|
||||
data = json.dumps({
|
||||
'keymaterial': key.blob
|
||||
})
|
||||
self.request('keys.json', list_id=list.id, data=data, method='POST')
|
||||
|
||||
def delete_key(self, key: SchleuderKey, list: 'SchleuderList'):
|
||||
if self.ns.dry_run:
|
||||
return
|
||||
self.request('keys/{}.json', list_id=list.id, fmt=[key.fingerprint], method='DELETE')
|
||||
|
||||
def subscribe(self, sub: 'SchleuderSubscriber', list: 'SchleuderList'):
|
||||
if self.ns.dry_run:
|
||||
return
|
||||
data = json.dumps({
|
||||
'email': sub.email,
|
||||
'fingerprint': sub.key.fingerprint
|
||||
})
|
||||
self.request('subscriptions.json', list_id=list.id, data=data, method='POST')
|
||||
|
||||
def unsubscribe(self, sub: 'SchleuderSubscriber'):
|
||||
if self.ns.dry_run:
|
||||
return
|
||||
self.request('subscriptions/{}.json', fmt=[sub.sub_id], method='DELETE')
|
||||
|
||||
def resolve_subscriber_conflicts(self, subscriptions: List['SchleuderSubscription']) -> 'SchleuderSubscription':
|
||||
if len(subscriptions) == 1:
|
||||
return subscriptions[0]
|
||||
earliest = min(subscriptions, key=lambda x: x.created_at)
|
||||
LOG.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder.name}:')
|
||||
for s in subscriptions:
|
||||
LOG.debug(f' - {s.schleuder.name}: {s.key.fingerprint}')
|
||||
for s in subscriptions:
|
||||
self.get_key(s.key, s.list.id)
|
||||
self.conflict_messages.append(ConflictMessage(earliest, subscriptions))
|
||||
return earliest
|
||||
|
||||
def send_conflict_messages(self):
|
||||
now = datetime.utcnow().timestamp()
|
||||
with open(self.conflict_state_file, 'a+') as f:
|
||||
f.seek(0)
|
||||
state = json.load(f)
|
||||
# Remove all state entries older than conflict_interval
|
||||
state = {k: v for k, v in state.items() if now-v < self.conflict_interval}
|
||||
# Remove all messages not already sent recently
|
||||
msgs = [m for m in self.conflict_messages if m.digest not in state]
|
||||
# Add all remaining messages to state dict
|
||||
for m in msgs:
|
||||
state[m.digest] = now
|
||||
if not self.ns.dry_run:
|
||||
f.seek(0)
|
||||
f.truncate()
|
||||
json.dump(state)
|
||||
# Finally send the mails
|
||||
for m in msgs:
|
||||
LOG.info(f'Sending key conflict message to {m["To"]}')
|
||||
LOG.debug(f'MIME Message:\n{str(m)}')
|
||||
if not self.ns.dry_run:
|
||||
with self.smtp_config as smtp:
|
||||
smtp.send_message(m)
|
||||
# Clear conflict messages
|
||||
self.conflict_messages = []
|
||||
|
||||
def process(self):
|
||||
LOG.info(f'Processing: {self.list} {"DRY RUN" if self.ns.dry_run else ""}')
|
||||
# todo: conflict handling - what if a user is subscribed through 2 lists (and possibly with different keys?)
|
||||
lists: Dict[str, SchleuderList] = self.resolve_lists()
|
||||
target_list = lists[self.list]
|
||||
# Get current subs, except for unmanaged adresses
|
||||
current_subs = {s
|
||||
for s in self.get_list_subscribers(target_list).values()
|
||||
if s.email not in self.unmanaged}
|
||||
current_keys = {s.key for s in current_subs}
|
||||
intended_subs = set()
|
||||
intended_keys = set()
|
||||
all_subs = dict()
|
||||
# This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists
|
||||
for source in self.source:
|
||||
for sub in self.get_list_subscribers(lists[source]).values():
|
||||
# Don't include banned or unmanaged adresses
|
||||
if sub.email in self.banned or s.email in self.unmanaged:
|
||||
continue
|
||||
all_subs.setdefault(sub.email, []).append(sub)
|
||||
# Remove
|
||||
for subs in all_subs.values():
|
||||
sub = self.resolve_subscriber_conflicts(subs)
|
||||
intended_subs.add(sub)
|
||||
intended_keys.add(sub.key)
|
||||
# Determine the change set
|
||||
to_subscribe = intended_subs.difference(current_subs)
|
||||
to_unsubscribe = current_subs.difference(intended_subs)
|
||||
to_remove = current_keys.difference(intended_keys)
|
||||
to_add = intended_keys.difference(current_keys)
|
||||
# Retrieve actual key blobs
|
||||
for key in to_add:
|
||||
self.get_key(key)
|
||||
# Perform the actual list modifications in an order where nothing breaks
|
||||
for key in to_add:
|
||||
self.post_key(key, target_list_id)
|
||||
LOG.info(f'Added key: {key}')
|
||||
for email in to_subscribe:
|
||||
self.subscribe(email, target_list_id)
|
||||
LOG.info(f'Subscribed user: {email}')
|
||||
for sub_id in to_unsubscribe:
|
||||
self.unsubscribe(sub_id)
|
||||
LOG.info(f'Unsubscribed user: {sub_id}')
|
||||
for key in to_remove:
|
||||
self.delete_key(key, target_list_id)
|
||||
LOG.info(f'Removed key: {key}')
|
||||
self.send_conflict_messages()
|
||||
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
|
||||
LOG.info(f'No changes for {self.list}')
|
||||
else:
|
||||
LOG.info(f'Finished processing: {self.list}')
|
||||
|
||||
|
||||
def parse_config(ns: argparse.Namespace) -> List['MultiList']:
|
||||
with open(ns.config, 'r') as f:
|
||||
y = yaml.safe_load(f)
|
||||
lists = []
|
||||
for l in y['lists']:
|
||||
lc = MultiList(ns,
|
||||
l['name'], l['source'], l.get('unmanaged', []), l.get('banned', []),
|
||||
l.get('from'),
|
||||
y['api']['url'], y['api']['token'], y['api']['cafile'],
|
||||
y['conflict']['state'], y['conflict']['message'], y['conflict']['interval'],
|
||||
SmtpServerConfig.parse(y['conflict']['smtp']))
|
||||
lists.append(lc)
|
||||
return lists
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--config', '-c', type=str, default='/etc/multischleuder/config.yml')
|
||||
ap.add_argument('--dry-run', '-n', action='store_true', default=False)
|
||||
ap.add_argument('--verbose', '-v', action='store_true', default=False)
|
||||
ns = ap.parse_args(sys.argv[1:])
|
||||
if ns.verbose:
|
||||
LOG.setLevel('DEBUG')
|
||||
LOG.debug('Enabled verbose logging')
|
||||
lists = parse_config(ns)
|
||||
for l in lists:
|
||||
l.process()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
2
multischleuder/__init__.py
Normal file
2
multischleuder/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
__version__ = '0.1'
|
||||
107
multischleuder/api.py
Normal file
107
multischleuder/api.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
|
||||
from typing import List, Optional
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import urllib.request
|
||||
|
||||
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
|
||||
|
||||
|
||||
class SchleuderApi:
|
||||
|
||||
def __init__(self,
|
||||
url: str,
|
||||
token: str,
|
||||
cafile: str):
|
||||
self._url = url
|
||||
self._cafile = cafile
|
||||
self._dry_run = False
|
||||
# The API token is not used as a Bearer token, but as the
|
||||
# password for the static "schleuder" user.
|
||||
auth = base64.b64encode(f'schleuder:{token}'.encode()).decode()
|
||||
self._headers = {
|
||||
'Authorization': f'Basic {auth}'
|
||||
}
|
||||
|
||||
def __request(self,
|
||||
path: str,
|
||||
list_id: Optional[int] = None,
|
||||
data: Optional[str] = None,
|
||||
method: str = 'GET',
|
||||
fmt=[]):
|
||||
# Compose and render fully-qualified URL
|
||||
if fmt:
|
||||
path = path.format(*fmt)
|
||||
url = os.path.join(self._url, path)
|
||||
if list_id is not None:
|
||||
url += f'?list_id={list_id}'
|
||||
|
||||
payload: Optional[bytes] = data.encode() if data is not None else None
|
||||
# Create a custom SSL context that does not validate the hostname, but
|
||||
# validates against the self-signed schleuder-api-daemon certificate
|
||||
context = ssl.create_default_context(cafile=self._cafile)
|
||||
context.check_hostname = False
|
||||
# Perform the actual request
|
||||
req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
|
||||
resp = urllib.request.urlopen(req, context=context)
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def dry_run(self):
|
||||
self._dry_run = True
|
||||
|
||||
# List Management
|
||||
|
||||
def get_lists(self) -> List['SchleuderList']:
|
||||
lists = self.__request('lists.json')
|
||||
return [SchleuderList(**s) for s in lists]
|
||||
|
||||
# Subscriber Management
|
||||
|
||||
def get_subscribers(self, schleuder: 'SchleuderList') -> List['SchleuderSubscriber']:
|
||||
response = self.__request('subscriptions.json', schleuder.id)
|
||||
subs: List[SchleuderSubscriber] = []
|
||||
for r in response:
|
||||
key = self.__request('keys/{}.json', schleuder.id, fmt=[r['fingerprint']])
|
||||
sub = SchleuderSubscriber.from_api(key, **r)
|
||||
subs.append(sub)
|
||||
return subs
|
||||
|
||||
def subscribe(self, sub: 'SchleuderSubscriber', schleuder: 'SchleuderList'):
|
||||
if self._dry_run:
|
||||
return
|
||||
data = json.dumps({
|
||||
'email': sub.email,
|
||||
'fingerprint': sub.key.fingerprint
|
||||
})
|
||||
self.__request('subscriptions.json', list_id=schleuder.id, data=data, method='POST')
|
||||
|
||||
def unsubscribe(self, sub: 'SchleuderSubscriber'):
|
||||
if self._dry_run:
|
||||
return
|
||||
self.__request('subscriptions/{}.json', fmt=[sub.id], method='DELETE')
|
||||
|
||||
def update_fingerprint(self, sub: 'SchleuderSubscriber'):
|
||||
if self._dry_run:
|
||||
return
|
||||
data = json.dumps({'fingerprint': sub.key.fingerprint})
|
||||
self.__request('subscriptions/{}.json', fmt=[sub.id], data=data, method='PATCH')
|
||||
|
||||
# Key Management
|
||||
|
||||
def post_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
|
||||
if self._dry_run:
|
||||
return
|
||||
if not key.blob:
|
||||
raise ValueError('Cannot upload a key stub')
|
||||
data = json.dumps({
|
||||
'keymaterial': key.blob
|
||||
})
|
||||
self.__request('keys.json', list_id=schleuder.id, data=data, method='POST')
|
||||
|
||||
def delete_key(self, key: 'SchleuderKey', schleuder: 'SchleuderList'):
|
||||
if self._dry_run:
|
||||
return
|
||||
self.__request('keys/{}.json', list_id=schleuder.id, fmt=[key.fingerprint], method='DELETE')
|
||||
179
multischleuder/conflict.py
Normal file
179
multischleuder/conflict.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
|
||||
from typing import Dict, List
|
||||
|
||||
import email.mime.application
|
||||
import email.mime.multipart
|
||||
import email.utils
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import struct
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
import pgpy # type: ignore
|
||||
|
||||
from multischleuder.types import SchleuderKey, SchleuderSubscriber
|
||||
from multischleuder.smtp import SmtpClient
|
||||
|
||||
|
||||
class ConflictMessage:
|
||||
|
||||
def __init__(self,
|
||||
schleuder: str,
|
||||
chosen: SchleuderSubscriber,
|
||||
affected: List[SchleuderSubscriber],
|
||||
mail_from: str,
|
||||
template: str):
|
||||
self._schleuder: str = schleuder
|
||||
self._chosen: SchleuderSubscriber = chosen
|
||||
self._affected: List[SchleuderSubscriber] = affected
|
||||
self._template: str = template
|
||||
self._from: str = mail_from
|
||||
# Generate a SHA1 digest that only changes when the subscription list changes
|
||||
self._digest = self._make_digest()
|
||||
|
||||
def _make_digest(self) -> str:
|
||||
# Sort so the hash stays the same if the set of subscriptions is the same.
|
||||
# There is no guarantee that the subs are in any specific order.
|
||||
subs: List[SchleuderSubscriber] = list(self._affected)
|
||||
subs.sort(key=lambda x: x.schleuder)
|
||||
h = hashlib.new('sha1')
|
||||
# Include the chosen email an source sub-list
|
||||
h.update(struct.pack('!sd',
|
||||
self._chosen.email,
|
||||
self._chosen.schleuder))
|
||||
# Include all subscriptions, including the FULL key
|
||||
for s in subs:
|
||||
h.update(struct.pack('!ds',
|
||||
s.schleuder,
|
||||
s.key.blob))
|
||||
return h.hexdigest()
|
||||
|
||||
@property
|
||||
def digest(self) -> str:
|
||||
return self._digest
|
||||
|
||||
@property
|
||||
def mime(self) -> email.mime.multipart.MIMEMultipart:
|
||||
# Render the message body
|
||||
_chosen = f'{self._chosen.key.fingerprint} {self._chosen.email}'
|
||||
_affected = ''
|
||||
for affected in self._affected:
|
||||
_affected += f'{affected.key.fingerprint} {affected.schleuder}\n'
|
||||
msg: str = self._template.format(
|
||||
subscriber=self._chosen.email,
|
||||
schleuder=self._schleuder,
|
||||
chosen=_chosen,
|
||||
affected=_affected
|
||||
)
|
||||
pgp = pgpy.PGPMessage.new(msg)
|
||||
# Encrypt the message to all keys
|
||||
cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
|
||||
sessionkey = cipher.gen_key()
|
||||
try:
|
||||
for affected in self._affected:
|
||||
key, _ = pgpy.PGPKey.from_blob(affected.key.blob)
|
||||
key._require_usage_flags = False
|
||||
pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
|
||||
finally:
|
||||
del sessionkey
|
||||
# Build the MIME message
|
||||
# First the small "version" part ...
|
||||
mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
|
||||
mp1['Content-Description'] = 'PGP/MIME version identification'
|
||||
mp1['Content-Disposition'] = 'attachment'
|
||||
# ... then the actual encrypted payload ...
|
||||
mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
|
||||
mp2['Content-Description'] = 'OpenPGP encrypted message'
|
||||
mp2['Content-Disposition'] = 'inline; filename="message.asc"'
|
||||
# ... and finally the root multipart container
|
||||
mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
|
||||
mp0.attach(mp1)
|
||||
mp0.attach(mp2)
|
||||
# Set all the email headers
|
||||
mp0['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
|
||||
mp0['From'] = self._from
|
||||
mp0['Reply-To'] = self._from
|
||||
mp0['To'] = self._chosen.email
|
||||
mp0['Date'] = email.utils.formatdate()
|
||||
mp0['Auto-Submitted'] = 'auto-generated'
|
||||
mp0['Precedence'] = 'list'
|
||||
mp0['List-Id'] = f'<{self._schleuder.replace("@", ".")}>'
|
||||
mp0['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
|
||||
mp0['X-MultiSchleuder-Digest'] = self._digest
|
||||
return mp0
|
||||
|
||||
|
||||
class KeyConflictResolution:
|
||||
|
||||
def __init__(self, smtp: 'SmtpClient', interval: int, statefile: str, template: str):
|
||||
self._smtp = smtp
|
||||
self._interval: int
|
||||
self._state_file: str = statefile
|
||||
self._template: str = statefile
|
||||
self._messages: List[ConflictMessage] = []
|
||||
self._logger: logging.Logger = logging.getLogger()
|
||||
|
||||
def resolve(self,
|
||||
target: str,
|
||||
mail_from: str,
|
||||
subscriptions: List['SchleuderSubscriber']) -> List['SchleuderSubscriber']:
|
||||
subs: Dict[str, List[SchleuderSubscriber]] = OrderedDict()
|
||||
for s in subscriptions:
|
||||
subs.setdefault(s.email, []).append(s)
|
||||
# Perform conflict resolution for each set of subscribers with the same email
|
||||
return [self._resolve(target, mail_from, s) for s in subs.values()]
|
||||
|
||||
def _resolve(self,
|
||||
target: str,
|
||||
mail_from: str,
|
||||
subscriptions: List['SchleuderSubscriber']) -> 'SchleuderSubscriber':
|
||||
if len({s.email for s in subscriptions}) != 1:
|
||||
raise ValueError('Number of unique subscriptions must be 1')
|
||||
if len(subscriptions) == 1:
|
||||
return subscriptions[0]
|
||||
# Conflict Resolution: Choose the OLDEST subscriptions, but notify using ALL keys
|
||||
earliest: SchleuderSubscriber = min(subscriptions, key=lambda x: x.created_at)
|
||||
self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.schleuder}:')
|
||||
for s in subscriptions:
|
||||
self._logger.debug(f' - {s.schleuder}: {s.key.fingerprint}')
|
||||
# At this point, the messages are only queued locally, they will be sent afterwards
|
||||
msg = ConflictMessage(
|
||||
target,
|
||||
earliest,
|
||||
subscriptions,
|
||||
self._template,
|
||||
mail_from
|
||||
)
|
||||
self._messages.append(msg)
|
||||
# Return the result of conflict resolution
|
||||
return earliest
|
||||
|
||||
def send_messages(self, dry_run: bool = False):
|
||||
now = int(datetime.utcnow().timestamp())
|
||||
with open(self._state_file, 'a+') as f:
|
||||
f.seek(0)
|
||||
state: Dict[str, int] = json.load(f)
|
||||
# Remove all state entries older than conflict_interval
|
||||
state = {k: v for k, v in state.items() if now-v < self._interval}
|
||||
# Remove all messages not already sent recently
|
||||
msgs = [m for m in self._messages if m.digest not in state]
|
||||
# Add all remaining messages to state dict
|
||||
for m in msgs:
|
||||
state[m.digest] = now
|
||||
# Write the new state to file
|
||||
if not dry_run:
|
||||
f.seek(0)
|
||||
f.truncate()
|
||||
json.dump(state, f)
|
||||
# Finally send the mails
|
||||
with self._smtp as smtp:
|
||||
for m in msgs:
|
||||
msg = m.mime
|
||||
self._logger.debug(f'MIME Message:\n{str(m)}')
|
||||
if not dry_run:
|
||||
self._logger.info(f'Sending key conflict message to {msg["To"]}')
|
||||
smtp.send_message(msg)
|
||||
# Clear conflict messages
|
||||
self._messages = []
|
||||
69
multischleuder/main.py
Normal file
69
multischleuder/main.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import argparse
|
||||
|
||||
import yaml
|
||||
|
||||
from multischleuder.api import SchleuderApi
|
||||
from multischleuder.conflict import KeyConflictResolution
|
||||
from multischleuder.processor import MultiList
|
||||
from multischleuder.smtp import SmtpClient
|
||||
|
||||
|
||||
def parse_list_config(api: SchleuderApi,
|
||||
kcr: KeyConflictResolution,
|
||||
config: Dict[str, Any]) -> 'MultiList':
|
||||
target = config['target']
|
||||
default_from = target.replace('@', '-owner@')
|
||||
mail_from = config.get('from', default_from)
|
||||
banned = config.get('banned', [])
|
||||
unmanaged = config.get('unmanaged', [])
|
||||
return MultiList(
|
||||
sources=config['sources'],
|
||||
target=target,
|
||||
banned=banned,
|
||||
unmanaged=unmanaged,
|
||||
mail_from=mail_from,
|
||||
api=api,
|
||||
kcr=kcr
|
||||
)
|
||||
|
||||
|
||||
def parse_config(ns: argparse.Namespace) -> List['MultiList']:
|
||||
with open(ns.config, 'r') as f:
|
||||
c = yaml.safe_load(f)
|
||||
|
||||
api = SchleuderApi(**c['api'])
|
||||
if ns.dry_run:
|
||||
api.dry_run()
|
||||
|
||||
smtp_config = c.get('smtp', {})
|
||||
smtp = SmtpClient.parse(**smtp_config)
|
||||
|
||||
kcr_config = c.get('conflict', {})
|
||||
kcr = KeyConflictResolution(smtp, **kcr_config)
|
||||
|
||||
lists = []
|
||||
for clist in c['lists']:
|
||||
ml = parse_list_config(api, kcr, clist)
|
||||
lists.append(ml)
|
||||
return lists
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--config', '-c', type=str, default='/etc/multischleuder/config.yml')
|
||||
ap.add_argument('--dry-run', '-n', action='store_true', default=False)
|
||||
ap.add_argument('--verbose', '-v', action='store_true', default=False)
|
||||
ns = ap.parse_args(sys.argv[1:])
|
||||
if ns.verbose:
|
||||
logger = logging.getLogger().setLevel('DEBUG')
|
||||
logger.debug('Verbose logging enabled')
|
||||
lists = parse_config(ns)
|
||||
for lst in lists:
|
||||
lst.process(ns.dry_run)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
85
multischleuder/processor.py
Normal file
85
multischleuder/processor.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
import logging
|
||||
|
||||
from multischleuder.api import SchleuderApi
|
||||
from multischleuder.conflict import KeyConflictResolution
|
||||
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
|
||||
|
||||
|
||||
class MultiList:
|
||||
|
||||
def __init__(self,
|
||||
sources: List[str],
|
||||
target: str,
|
||||
unmanaged: List[str],
|
||||
banned: List[str],
|
||||
mail_from: str,
|
||||
api: SchleuderApi,
|
||||
kcr: KeyConflictResolution):
|
||||
self._sources: List[str] = sources
|
||||
self._target: str = target
|
||||
self._unmanaged: List[str] = unmanaged
|
||||
self._banned: List[str] = banned
|
||||
self._mail_from: str = mail_from
|
||||
self._api: SchleuderApi = api
|
||||
self._kcr: KeyConflictResolution = kcr
|
||||
self._logger: logging.Logger = logging.getLogger()
|
||||
|
||||
def process(self, dry_run: bool = False):
|
||||
self._logger.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
|
||||
target_list, sources = self._lists_by_name()
|
||||
# Get current subs, except for unmanaged adresses
|
||||
current_subs: Set[SchleuderSubscriber] = {
|
||||
s
|
||||
for s in self._api.get_subscribers(target_list)
|
||||
if s.email not in self._unmanaged
|
||||
}
|
||||
current_keys: Set[SchleuderKey] = {s.key for s in current_subs}
|
||||
all_subs: List[SchleuderSubscriber] = []
|
||||
# This loop may return multiple subscriptions for some users if they are subscribed on multiple sub-lists ...
|
||||
for source in sources:
|
||||
subs: List[SchleuderSubscriber] = self._api.get_subscribers(source)
|
||||
for s in subs:
|
||||
if s.email in self._banned or s.email in self._unmanaged:
|
||||
continue
|
||||
all_subs.append(s)
|
||||
# ... which is taken care of by the key conflict resolution routine
|
||||
self._kcr.resolve(self._target, self._mail_from, all_subs)
|
||||
intended_subs: Set[SchleuderSubscriber] = set(all_subs)
|
||||
intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs}
|
||||
# Determine the change set
|
||||
to_subscribe = intended_subs.difference(current_subs)
|
||||
to_unsubscribe = current_subs.difference(intended_subs)
|
||||
to_remove = current_keys.difference(intended_keys)
|
||||
to_add = intended_keys.difference(current_keys)
|
||||
to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
|
||||
# Perform the actual list modifications in an order which avoids race conditions
|
||||
for key in to_add:
|
||||
self._api.post_key(key, target_list)
|
||||
self._logger.info(f'Added key: {key}')
|
||||
for sub in to_subscribe:
|
||||
self._api.subscribe(sub, target_list)
|
||||
self._logger.info(f'Subscribed user: {sub}')
|
||||
for sub in to_update:
|
||||
self._api.update_fingerprint(sub)
|
||||
self._logger.info(f'Updated key: {sub}')
|
||||
for sub in to_unsubscribe:
|
||||
self._api.unsubscribe(sub)
|
||||
self._logger.info(f'Unsubscribed user: {sub}')
|
||||
for key in to_remove:
|
||||
self._api.delete_key(key, target_list)
|
||||
self._logger.info(f'Removed key: {key}')
|
||||
# Finally, send any queued key conflict messages.
|
||||
self._kcr.send_messages()
|
||||
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
|
||||
self._logger.info(f'No changes for {self._target}')
|
||||
else:
|
||||
self._logger.info(f'Finished processing: {self._target}')
|
||||
|
||||
def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]:
|
||||
lists = self._api.get_lists()
|
||||
target = [x for x in lists if x.name == self._target][0]
|
||||
sources = [x for x in lists if x.name in self._sources]
|
||||
return target, sources
|
||||
81
multischleuder/smtp.py
Normal file
81
multischleuder/smtp.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import email
|
||||
import enum
|
||||
import logging
|
||||
import smtplib
|
||||
|
||||
|
||||
class TlsMode(enum.Enum):
|
||||
PLAIN = TlsMode('smtp', 25)
|
||||
SMTPS = TlsMode('smtps', 465)
|
||||
STARTTLS = TlsMode('smtp+starttls', 587)
|
||||
|
||||
def __init__(self, proto: str, port: int):
|
||||
self._proto = proto
|
||||
self._port = port
|
||||
|
||||
@property
|
||||
def proto(self):
|
||||
return self._proto
|
||||
|
||||
@property
|
||||
def port(self):
|
||||
return self._port
|
||||
|
||||
|
||||
class SmtpClient:
|
||||
|
||||
def __init__(self, hostname: str, port: int, tls: 'TlsMode',
|
||||
username: Optional[str], password: Optional[str]):
|
||||
self._hostname: str = hostname
|
||||
self._port: int = port
|
||||
self._tls: TlsMode = tls
|
||||
self._username: Optional[str] = username
|
||||
self._password: Optional[str] = password
|
||||
self._smtp: Optional[smtplib.SMTP] = None
|
||||
self._logger = logging.getLogger()
|
||||
|
||||
@staticmethod
|
||||
def parse(config: Dict[str, Any]) -> 'SmtpClient':
|
||||
tls = TlsMode[config.get('tls', 'PLAIN')]
|
||||
return SmtpClient(
|
||||
hostname=config.get('hostname', 'localhost'),
|
||||
port=config.get('port', tls.port),
|
||||
tls=tls,
|
||||
username=config.get('username'),
|
||||
password=config.get('password')
|
||||
)
|
||||
|
||||
def send_message(self, msg: email.message.Message):
|
||||
if self._smtp is None:
|
||||
raise RuntimeError('SMTP connection is not established')
|
||||
self._smtp.send_message(msg)
|
||||
self._logger.debug(f'Sent email message.')
|
||||
|
||||
def __enter__(self):
|
||||
# TLS from the start requires a different class
|
||||
cls = smtplib.SMTP if self._tls != TlsMode.SMTPS else smtplib.SMTP_SSL
|
||||
self._smtp = cls(self._hostname, self._port)
|
||||
# Establish the connection
|
||||
smtp = self._smtp.__enter__()
|
||||
if self._tls == TlsMode.STARTTLS:
|
||||
smtp.starttls()
|
||||
# Only sign in if both username and password are provided
|
||||
if self._username is not None and self._password is not None:
|
||||
smtp.login(self._username, self._password)
|
||||
self._logger.debug(f'SMTP connection to {str(self)} established')
|
||||
return smtp
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self._smtp is None:
|
||||
raise RuntimeError('SMTP connection is not established')
|
||||
self._smtp.quit()
|
||||
ret = self._smtp.__exit__(exc_type, exc_val, exc_tb)
|
||||
self._logger.debug(f'SMTP connection to {str(self)} closed')
|
||||
self._smtp = None
|
||||
return ret
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self._tls.proto}://{self._username}@{self._hostname}:{self._port}'
|
||||
68
multischleuder/types.py
Normal file
68
multischleuder/types.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
|
||||
from dataclasses import dataclass, field, Field
|
||||
from datetime import datetime
|
||||
from dateutil.parser import isoparse
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderList:
|
||||
id: int
|
||||
name: str
|
||||
fingerprint: str
|
||||
|
||||
@staticmethod
|
||||
def from_api(id: int,
|
||||
email: str,
|
||||
fingerprint: str,
|
||||
*args, **kwargs) -> 'SchleuderList':
|
||||
return SchleuderList(id, email, fingerprint)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderSubscriber:
|
||||
id: int
|
||||
email: str
|
||||
key: 'SchleuderKey'
|
||||
schleuder: int
|
||||
created_at: datetime
|
||||
|
||||
@staticmethod
|
||||
def from_api(key: 'SchleuderKey',
|
||||
id: int,
|
||||
list_id: int,
|
||||
email: str,
|
||||
fingerprint: str,
|
||||
admin: bool,
|
||||
delivery_enabled: bool,
|
||||
created_at: str,
|
||||
updated_at: str) -> 'SchleuderSubscriber':
|
||||
created = isoparse(created_at)
|
||||
return SchleuderSubscriber(id, email, key, list_id, created)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self.email}'
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.email)
|
||||
|
||||
def __eq__(self, o) -> bool:
|
||||
if not isinstance(o, SchleuderSubscriber):
|
||||
return False
|
||||
return self.email == o.email
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchleuderKey:
|
||||
fingerprint: str
|
||||
email: str
|
||||
blob: str = field(repr=False)
|
||||
schleuder: int
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.fingerprint, self.email))
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'{self.fingerprint} ({self.email})'
|
||||
1
package/debian/multischleuder/DEBIAN/conffiles
Normal file
1
package/debian/multischleuder/DEBIAN/conffiles
Normal file
|
|
@ -0,0 +1 @@
|
|||
/etc/multischleuder/multischleuder.yml
|
||||
11
package/debian/multischleuder/DEBIAN/control
Normal file
11
package/debian/multischleuder/DEBIAN/control
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
Package: multischleuder
|
||||
Version: __MULTISCHLEUDER_VERSION__
|
||||
Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
|
||||
Section: web
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
Depends: python3 (>= 3.6), python3-dateutil, python3-pgpy, python3-yaml
|
||||
Recommends: schleuder (>= 3.6)
|
||||
Description: Schleuder subscriber merging tool
|
||||
Automatically and periodically merge subscribers and keys of multiple
|
||||
Schleuder lists into one.
|
||||
24
package/debian/multischleuder/DEBIAN/postinst
Executable file
24
package/debian/multischleuder/DEBIAN/postinst
Executable file
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "configure" ]]; then
|
||||
|
||||
if ! getent group multischleuder >/dev/null; then
|
||||
groupadd --system multischleuder
|
||||
fi
|
||||
|
||||
if ! getent passwd multischleuder >/dev/null; then
|
||||
useradd --system --create-home --gid multischleuder --home-dir /var/lib/multischleuder --shell /usr/sbin/nologin multischleuder
|
||||
fi
|
||||
|
||||
chown multischleuder:multischleuder /var/lib/multischleuder
|
||||
chmod 0750 /var/lib/multischleuder
|
||||
|
||||
chown root:multischleuder /etc/multischleuder
|
||||
chmod 0750 /etc/multischleuder
|
||||
|
||||
deb-systemd-helper enable multischleuder.timer
|
||||
deb-systemd-invoke restart multischleuder.timer
|
||||
|
||||
fi
|
||||
9
package/debian/multischleuder/DEBIAN/postrm
Executable file
9
package/debian/multischleuder/DEBIAN/postrm
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "remove" ]]; then
|
||||
|
||||
systemctl daemon-reload || true
|
||||
|
||||
fi
|
||||
9
package/debian/multischleuder/DEBIAN/prerm
Executable file
9
package/debian/multischleuder/DEBIAN/prerm
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "remove" ]]; then
|
||||
|
||||
deb-systemd-invoke stop multischleuder.timer
|
||||
|
||||
fi
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=Multischleuder Sync Job
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/multischleuder --config=/etc/multischleuder/config.yaml
|
||||
User=multischleuder
|
||||
Group=multischleuder
|
||||
WorkingDirectory=/var/lib/multischleuder
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[Unit]
|
||||
Description=Multischleuder Sync Job
|
||||
|
||||
[Timer]
|
||||
OnCalendar=hourly
|
||||
Persistent=true
|
||||
|
||||
[Install]e
|
||||
WantedBy=timers.target
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
Copyright 2022 s3lph
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
183
package/release.py
Executable file
183
package/release.py
Executable file
|
|
@ -0,0 +1,183 @@
|
|||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib.request
|
||||
import http.client
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def parse_changelog(tag: str) -> Optional[str]:
|
||||
release_changelog: str = ''
|
||||
with open('CHANGELOG.md', 'r') as f:
|
||||
in_target: bool = False
|
||||
done: bool = False
|
||||
for line in f.readlines():
|
||||
if in_target:
|
||||
if f'<!-- END RELEASE {tag} -->' in line:
|
||||
done = True
|
||||
break
|
||||
release_changelog += line
|
||||
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
|
||||
in_target = True
|
||||
continue
|
||||
if not done:
|
||||
return None
|
||||
return release_changelog
|
||||
|
||||
|
||||
def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
|
||||
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'User-Agent': 'curl/7.70.0'
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.read()
|
||||
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
|
||||
|
||||
jobidmap: Dict[str, str] = {}
|
||||
for job in joblist:
|
||||
name: str = job['name']
|
||||
job_id: str = job['id']
|
||||
jobidmap[name] = job_id
|
||||
return jobidmap
|
||||
|
||||
|
||||
def fetch_single_shafile(url: str, api_token: str) -> str:
|
||||
headers: Dict[str, str] = {
|
||||
'User-Agent': 'curl/7.70.0',
|
||||
'Private-Token': api_token
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.readline()
|
||||
shafile: str = resp_data.decode()
|
||||
filename: str = shafile.strip().split(' ')[-1].strip()
|
||||
return filename
|
||||
|
||||
|
||||
def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
|
||||
wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
|
||||
'/artifacts/dist/SHA256SUMS'
|
||||
wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
|
||||
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
|
||||
return wheel_url, wheel_sha_url
|
||||
|
||||
|
||||
def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
|
||||
debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
|
||||
'/artifacts/package/debian/SHA256SUMS'
|
||||
debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
|
||||
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
|
||||
return debian_url, debian_sha_url
|
||||
|
||||
|
||||
def main():
|
||||
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
|
||||
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
|
||||
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
|
||||
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
|
||||
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
|
||||
if api_token is None:
|
||||
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if release_tag is None:
|
||||
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_name is None:
|
||||
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_id is None:
|
||||
print('CI_PROJECT_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if pipeline_id is None:
|
||||
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
changelog: Optional[str] = parse_changelog(release_tag)
|
||||
if changelog is None:
|
||||
print('Changelog could not be parsed.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
|
||||
|
||||
base_url: str = f'https://gitlab.com/{project_name}/-'
|
||||
|
||||
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
|
||||
debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
|
||||
|
||||
augmented_changelog = f'''{changelog.strip()}
|
||||
|
||||
### Download
|
||||
|
||||
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
|
||||
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
|
||||
# Docker currently not working
|
||||
# - Docker image: registry.gitlab.com/{project_name}:{release_tag}
|
||||
|
||||
post_body: str = json.dumps({
|
||||
'tag_name': release_tag,
|
||||
'description': augmented_changelog,
|
||||
'assets': {
|
||||
'links': [
|
||||
{
|
||||
'name': 'Python Wheel',
|
||||
'url': wheel_url,
|
||||
'link_type': 'package'
|
||||
},
|
||||
{
|
||||
'name': 'Debian Package',
|
||||
'url': debian_url,
|
||||
'link_type': 'package'
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
gitlab_release_api_url: str = \
|
||||
f'https://gitlab.com/api/v4/projects/{project_id}/releases'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'User-Agent': 'curl/7.70.0'
|
||||
}
|
||||
|
||||
request = urllib.request.Request(
|
||||
gitlab_release_api_url,
|
||||
post_body.encode('utf-8'),
|
||||
headers=headers,
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
response: http.client.HTTPResponse = urllib.request.urlopen(request)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
response_bytes: bytes = response.read()
|
||||
response_str: str = response_bytes.decode()
|
||||
response_data: Dict[str, Any] = json.loads(response_str)
|
||||
|
||||
if response_data['tag_name'] != release_tag:
|
||||
print('Something went wrong...', file=sys.stderr)
|
||||
print(response_str, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(response_data['description'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
dateutil
|
||||
PyYAML
|
||||
PGPy
|
||||
22
setup.cfg
Normal file
22
setup.cfg
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
|
||||
#
|
||||
# PyCodestyle
|
||||
#
|
||||
|
||||
[pycodestyle]
|
||||
max-line-length = 120
|
||||
statistics = True
|
||||
|
||||
#
|
||||
# Coverage
|
||||
#
|
||||
|
||||
[run]
|
||||
branch = True
|
||||
parallel = True
|
||||
source = multischleuder/
|
||||
|
||||
[report]
|
||||
show_missing = True
|
||||
include = multischleuder/*
|
||||
omit = */test/*.py
|
||||
29
setup.py
Executable file
29
setup.py
Executable file
|
|
@ -0,0 +1,29 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
from multischleuder import __version__
|
||||
|
||||
|
||||
setup(
|
||||
name='multischleuder',
|
||||
version=__version__,
|
||||
author='s3lph',
|
||||
author_email='1375407-s3lph@users.noreply.gitlab.com',
|
||||
description='Merge subscribers and keys of multiple Schleuder lists into one',
|
||||
license='MIT',
|
||||
keywords='schleuder,pgp',
|
||||
url='https://gitlab.com/s3lph/multischleuder',
|
||||
packages=find_packages(exclude=['*.test']),
|
||||
install_requires=[
|
||||
'python-dateutil',
|
||||
'PyYAML',
|
||||
'PGPy',
|
||||
],
|
||||
test_requirements=[
|
||||
'mypy'
|
||||
],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'multischleuder = multischleuder.main:main'
|
||||
]
|
||||
},
|
||||
)
|
||||
Loading…
Reference in a new issue