Compare commits
38 commits
Author | SHA1 | Date | |
---|---|---|---|
a02f2d9e68 | |||
adaa10b3d3 | |||
30be0bb2dd | |||
a986ad236a | |||
05b9bf009f | |||
da0b9f7e28 | |||
b4a42a770d | |||
3605726162 | |||
|
16c66fc950 | ||
|
d7603f2c87 | ||
|
d1e9403913 | ||
|
caa2bd6a06 | ||
|
026e4987a6 | ||
|
91e81f6931 | ||
|
00d49c1451 | ||
|
fbe8300a6e | ||
|
a682cd9e7e | ||
|
eccd08a8bc | ||
|
95a2c481b7 | ||
|
edcd5bd152 | ||
|
ebb836f2dd | ||
|
6af68e4ce5 | ||
|
0be2f19c10 | ||
|
30472cd530 | ||
|
cdcc8fbf33 | ||
|
3d66918202 | ||
|
2954920c65 | ||
|
ddd71a28f0 | ||
|
8d4b84669f | ||
|
a160d22789 | ||
|
210bff48fb | ||
|
dcb03e8449 | ||
|
1060c8e8d0 | ||
|
72a8a67dbf | ||
|
259a6fe696 | ||
|
b317c2ee23 | ||
|
eb74f5e296 | ||
|
612334ae8d |
19 changed files with 361 additions and 391 deletions
38
.forgejo/workflows/package.yml
Normal file
38
.forgejo/workflows/package.yml
Normal file
|
@ -0,0 +1,38 @@
|
|||
---
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
jobs:
|
||||
|
||||
build_wheel:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Build Python wheel
|
||||
run: |
|
||||
apt update; apt install -y python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
python3 setup.py egg_info bdist_wheel
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3
|
||||
with:
|
||||
username: ${{ secrets.API_USERNAME }}
|
||||
password: ${{ secrets.API_PASSWORD }}
|
||||
|
||||
build_debian:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5
|
||||
with:
|
||||
python_module: multischleuder
|
||||
package_name: multischleuder
|
||||
package_root: package/debian/multischleuder
|
||||
package_output_path: package/debian
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2
|
||||
with:
|
||||
username: ${{ secrets.API_USERNAME }}
|
||||
password: ${{ secrets.API_PASSWORD }}
|
||||
deb: "package/debian/*.deb"
|
77
.forgejo/workflows/test.yml
Normal file
77
.forgejo/workflows/test.yml
Normal file
|
@ -0,0 +1,77 @@
|
|||
---
|
||||
|
||||
on: push
|
||||
|
||||
jobs:
|
||||
|
||||
test:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Unit Tests
|
||||
run: |
|
||||
apt update; apt install -y python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
|
||||
python3 -m coverage combine
|
||||
python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
codestyle:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Code Style
|
||||
run: |
|
||||
apt update; apt install -y python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
pycodestyle multischleuder
|
||||
|
||||
mypy:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Static Type Checks
|
||||
run: |
|
||||
apt update; apt install -y python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
pip3 install --break-system-packages types-PyYAML types-python-dateutil
|
||||
mypy multischleuder
|
||||
|
||||
schleuder:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Integration Test against schleuder
|
||||
run: |
|
||||
echo "postfix postfix/mailname string example.org" | debconf-set-selections
|
||||
echo "postfix postfix/main_mailer_type string 'Local only'" | debconf-set-selections
|
||||
apt update; apt install --yes python3-pip schleuder schleuder-cli postfix patch
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
/usr/lib/postfix/configure-instance.sh -
|
||||
echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
|
||||
/usr/sbin/postmulti -i - -p start
|
||||
schleuder-cli lists list || true
|
||||
export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
|
||||
echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
|
||||
cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
|
||||
host: localhost
|
||||
port: 4443
|
||||
tls_fingerprint: ${CERT_FPR}
|
||||
api_key: '00000000000000000000000000000000'
|
||||
EOF
|
||||
/usr/bin/schleuder-api-daemon &
|
||||
sleep 5 # wait for daemons to start
|
||||
export API_DAEMON_PID=$!
|
||||
test/prepare-schleuder.sh
|
||||
pip3 install --break-system-packages -e .
|
||||
python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
|
||||
python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||
# Run a second time - should be idempotent and not trigger any new mails
|
||||
python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||
python3 -m coverage combine
|
||||
python3 -m coverage report --rcfile=setup.cfg
|
||||
sleep 5 # wait for mail delivery
|
||||
test/report.py
|
||||
kill -9 ${API_DAEMON_PID} || true
|
||||
/usr/sbin/postmulti -i - -p stop
|
||||
sleep 5 # wait for daemons to terminate
|
142
.gitlab-ci.yml
142
.gitlab-ci.yml
|
@ -1,142 +0,0 @@
|
|||
---
|
||||
image: python:3.9-bullseye
|
||||
|
||||
stages:
|
||||
- test
|
||||
- coverage
|
||||
- build
|
||||
- deploy
|
||||
|
||||
|
||||
|
||||
before_script:
|
||||
- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff
|
||||
- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
|
||||
|
||||
|
||||
|
||||
test:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
|
||||
artifacts:
|
||||
paths:
|
||||
- ".coverage*"
|
||||
|
||||
codestyle:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- pycodestyle multischleuder
|
||||
|
||||
mypy:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- mypy --install-types --non-interactive multischleuder
|
||||
- mypy multischleuder
|
||||
|
||||
schleuder:
|
||||
stage: test
|
||||
script:
|
||||
- debconf-set-selections <<<"postfix postfix/mailname string example.org"
|
||||
- debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'"
|
||||
- apt update; apt install --yes schleuder schleuder-cli postfix
|
||||
- /usr/lib/postfix/configure-instance.sh -
|
||||
- echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
|
||||
- /usr/sbin/postmulti -i - -p start
|
||||
- schleuder-cli lists list || true
|
||||
- export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
|
||||
- echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
|
||||
- |
|
||||
cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
|
||||
host: localhost
|
||||
port: 4443
|
||||
tls_fingerprint: ${CERT_FPR}
|
||||
api_key: '00000000000000000000000000000000'
|
||||
EOF
|
||||
- /usr/bin/schleuder-api-daemon &
|
||||
- sleep 5 # wait for daemons to start
|
||||
- export API_DAEMON_PID=$!
|
||||
- test/prepare-schleuder.sh
|
||||
- pip3 install -e .
|
||||
- python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
|
||||
- python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||
# Run a second time - should be idempotent and not trigger any new mails
|
||||
- python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||
- sleep 5 # wait for mail delivery
|
||||
- test/report.py
|
||||
- kill -9 ${API_DAEMON_PID} || true
|
||||
- /usr/sbin/postmulti -i - -p stop
|
||||
- sleep 5 # wait for daemons to terminate
|
||||
artifacts:
|
||||
paths:
|
||||
- ".coverage*"
|
||||
|
||||
|
||||
|
||||
coverage:
|
||||
stage: coverage
|
||||
script:
|
||||
- python3 -m coverage combine
|
||||
- python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
|
||||
|
||||
build_wheel:
|
||||
stage: build
|
||||
script:
|
||||
- python3 setup.py egg_info bdist_wheel
|
||||
- cd dist
|
||||
- sha256sum *.whl > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "dist/*.whl"
|
||||
- dist/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
build_debian:
|
||||
stage: build
|
||||
script:
|
||||
- apt update && apt install --yes lintian rsync sudo
|
||||
- echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
- |
|
||||
for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
|
||||
echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
done
|
||||
- gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||
- python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
|
||||
- cd package/debian
|
||||
- sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
|
||||
- mkdir -p multischleuder/usr/lib/python3/dist-packages/
|
||||
- rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
|
||||
- rm -rf multischleuder/usr/lib/python3.9/site-packages
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
|
||||
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
|
||||
- sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
|
||||
- find multischleuder -type f -exec chmod 0644 {} \;
|
||||
- find multischleuder -type d -exec chmod 755 {} \;
|
||||
- chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
|
||||
- dpkg-deb --build multischleuder
|
||||
- mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||
- sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||
- sha256sum *.deb > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "package/debian/*.deb"
|
||||
- package/debian/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
|
||||
release:
|
||||
stage: deploy
|
||||
script:
|
||||
- python3 package/release.py
|
||||
only:
|
||||
- tags
|
81
CHANGELOG.md
81
CHANGELOG.md
|
@ -1,5 +1,86 @@
|
|||
# MultiSchleuder Changelog
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.9 -->
|
||||
## Version 0.1.9
|
||||
|
||||
Maintenance Release
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.9 -->
|
||||
- Migrate from Woodpecker to Forgejo Actions
|
||||
<!-- END CHANGES 0.1.9 -->
|
||||
|
||||
<!-- END RELEASE v0.1.9 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.8 -->
|
||||
## Version 0.1.8
|
||||
|
||||
Maintenance Release
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.8 -->
|
||||
- Migrate from Gitlab-CI to Woodpecker
|
||||
<!-- END CHANGES 0.1.8 -->
|
||||
|
||||
<!-- END RELEASE v0.1.8 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.7 -->
|
||||
## Version 0.1.7
|
||||
|
||||
Bugfix Release
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.7 -->
|
||||
- Remove and re-import keys whose expiry date has been changed
|
||||
- Don't report keys as changed if they appear to differ, but are treated as identical by GnuPG.
|
||||
<!-- END CHANGES 0.1.7 -->
|
||||
|
||||
<!-- END RELEASE v0.1.7 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.6 -->
|
||||
## Version 0.1.6
|
||||
|
||||
Bugfix Release
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.6 -->
|
||||
- Better error handling for wrongfully configured keys
|
||||
<!-- END CHANGES 0.1.6 -->
|
||||
|
||||
<!-- END RELEASE v0.1.6 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.5 -->
|
||||
## Version 0.1.5
|
||||
|
||||
Reporting changes
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.5 -->
|
||||
- Admin reports show the source sublist for each new subscriber
|
||||
- Add static code analysis CI jobs
|
||||
<!-- END CHANGES 0.1.5 -->
|
||||
|
||||
<!-- END RELEASE v0.1.5 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.4 -->
|
||||
## Version 0.1.4
|
||||
|
||||
Better error handling
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1.4 -->
|
||||
- Processing failure in one list won't affect other lists
|
||||
- Admin reports are never sent unencrypted
|
||||
<!-- END CHANGES 0.1.4 -->
|
||||
|
||||
<!-- END RELEASE v0.1.4 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.1.3 -->
|
||||
## Version 0.1.3
|
||||
|
||||
|
|
13
README.md
13
README.md
|
@ -1,10 +1,5 @@
|
|||
# MultiSchleuder
|
||||
|
||||
[![pipeline status](https://gitlab.com/s3lph/multischleuder/badges/main/pipeline.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
|
||||
[![coverage report](https://gitlab.com/s3lph/multischleuder/badges/main/coverage.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
|
||||
[![latest release](https://gitlab.com/s3lph/multischleuder/-/badges/release.svg)](https://gitlab.com/s3lph/multischleuder/-/releases)
|
||||
[![license](https://img.shields.io/badge/License-MIT-yellow.svg)](https://gitlab.com/s3lph/multischleuder/-/blob/main/LICENSE)
|
||||
|
||||
Automatically and periodically merge subscribers and keys of multiple [Schleuder][schleuder] lists into one.
|
||||
|
||||
## Dependencies
|
||||
|
@ -15,7 +10,7 @@ Automatically and periodically merge subscribers and keys of multiple [Schleuder
|
|||
|
||||
## Installation
|
||||
|
||||
You can find Debian packages and Python wheels over at [Releases][releases].
|
||||
You can find Debian packages and Python wheels over at [Packages][packages].
|
||||
|
||||
## Configuration
|
||||
|
||||
|
@ -178,11 +173,11 @@ MultiSchleuder resolves conflicts in a simple, but primitive manner:
|
|||
1. Then it checks whether a subscriber has more than one key. If so, the key used by the oldest subscription wins.
|
||||
|
||||
This is by no means a perfect solution.
|
||||
It does however yield consisitent results.
|
||||
It does however yield consistent results.
|
||||
In both cases, if configured to do so, MultiSchleuder will send a notification message to all subscribers involved in a conflict, encrypting it with all keys involved in the conflict.
|
||||
If one or more keys are - for whatever reason - unusable, the message will not be encrypted.
|
||||
This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibilty would be to not notify a subscriber when something potentially malicious is going on.
|
||||
This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibility would be to not notify a subscriber when something potentially malicious is going on.
|
||||
|
||||
|
||||
[schleuder]: https://schleuder.org/
|
||||
[releases]: https://gitlab.com/s3lph/multischleuder/-/releases
|
||||
[packages]: https://git.kabelsalat.ch/s3lph/multischleuder/packages
|
||||
|
|
|
@ -1,2 +1,2 @@
|
|||
|
||||
__version__ = '0.1.3'
|
||||
__version__ = '0.1.9'
|
||||
|
|
|
@ -3,8 +3,10 @@ from typing import List, Optional
|
|||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
|
||||
|
@ -49,7 +51,7 @@ class SchleuderApi:
|
|||
context = None
|
||||
# Perform the actual request
|
||||
req = urllib.request.Request(url, data=payload, method=method, headers=self._headers)
|
||||
resp = urllib.request.urlopen(req, context=context)
|
||||
resp = urllib.request.urlopen(req, context=context) # nosec B310 baseurl is trusted
|
||||
respdata: str = resp.read().decode()
|
||||
if len(respdata) > 0:
|
||||
return json.loads(respdata)
|
||||
|
@ -129,8 +131,12 @@ class SchleuderApi:
|
|||
|
||||
# Key Management
|
||||
|
||||
def get_key(self, fpr: str, schleuder: SchleuderList) -> SchleuderKey:
|
||||
def get_key(self, fpr: str, schleuder: SchleuderList) -> Optional[SchleuderKey]:
|
||||
try:
|
||||
key = self.__request('keys/{}.json', list_id=schleuder.id, fmt=[fpr])
|
||||
except urllib.error.HTTPError as e:
|
||||
logging.exception(e)
|
||||
return None
|
||||
return SchleuderKey.from_api(schleuder.id, **key)
|
||||
|
||||
def post_key(self, key: SchleuderKey, schleuder: SchleuderList):
|
||||
|
|
|
@ -31,8 +31,7 @@ class KeyConflictResolution:
|
|||
target: str,
|
||||
mail_from: str,
|
||||
subscriptions: List[SchleuderSubscriber],
|
||||
sources: List[SchleuderList]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
|
||||
sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
|
||||
sourcemap: Dict[int, str]) -> Tuple[List[SchleuderSubscriber], List[Optional[Message]]]:
|
||||
conflicts: List[Optional[Message]] = []
|
||||
|
||||
# First check for keys that are being used by more than one subscriber
|
||||
|
@ -174,7 +173,7 @@ class KeyConflictResolution:
|
|||
# Sort so the hash stays the same if the set of subscriptions is the same.
|
||||
# There is no guarantee that the subs are in any specific order.
|
||||
subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
|
||||
h = hashlib.new('sha1')
|
||||
h = hashlib.new('sha1') # nosec B324
|
||||
# Include the chosen email an source sub-list
|
||||
h.update(struct.pack('!sd',
|
||||
chosen.email.encode(),
|
||||
|
@ -191,7 +190,7 @@ class KeyConflictResolution:
|
|||
# Sort so the hash stays the same if the set of subscriptions is the same.
|
||||
# There is no guarantee that the subs are in any specific order.
|
||||
subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
|
||||
h = hashlib.new('sha1')
|
||||
h = hashlib.new('sha1') # nosec B324
|
||||
assert chosen.key is not None # Make mypy happy; it can't know that chosen.key can't be None
|
||||
# Include the chosen email an source sub-list
|
||||
h.update(struct.pack('!ssd',
|
||||
|
|
|
@ -74,5 +74,8 @@ def main():
|
|||
logging.debug('Verbose logging enabled')
|
||||
lists, smtp = parse_config(ns)
|
||||
for lst in lists:
|
||||
try:
|
||||
lst.process(ns.dry_run)
|
||||
except BaseException:
|
||||
logging.exception(f'An error occurred while processing {lst._target}')
|
||||
smtp.send_messages(Reporter.get_messages())
|
||||
|
|
|
@ -32,6 +32,7 @@ class MultiList:
|
|||
def process(self, dry_run: bool = False):
|
||||
logging.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
|
||||
target_list, sources = self._lists_by_name()
|
||||
sourcemap: Dict[int, str] = {s.id: s.name for s in sources}
|
||||
target_admins = self._api.get_list_admins(target_list)
|
||||
# Get current subs, except for unmanaged adresses
|
||||
current_subs: Set[SchleuderSubscriber] = set()
|
||||
|
@ -53,7 +54,7 @@ class MultiList:
|
|||
continue
|
||||
all_subs.append(s)
|
||||
# ... which is taken care of by the key conflict resolution routine
|
||||
resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sources)
|
||||
resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs, sourcemap)
|
||||
self._reporter.add_messages(conflicts)
|
||||
intended_subs: Set[SchleuderSubscriber] = set(resolved)
|
||||
intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None}
|
||||
|
@ -62,8 +63,14 @@ class MultiList:
|
|||
to_unsubscribe = current_subs.difference(intended_subs)
|
||||
to_remove = current_keys.difference(intended_keys)
|
||||
to_add = intended_keys.difference(current_keys)
|
||||
# Already present keys that are being updated have to be removed and re-imported for convergence
|
||||
to_pre_remove = {k for k in to_add if k.fingerprint in {o.fingerprint for o in to_remove}}
|
||||
to_remove = {k for k in to_remove if k.fingerprint not in {o.fingerprint for o in to_pre_remove}}
|
||||
to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
|
||||
# Perform the actual list modifications in an order which avoids race conditions
|
||||
for key in to_pre_remove:
|
||||
self._api.delete_key(key, target_list)
|
||||
logging.info(f'Pre-removed key: {key}')
|
||||
for key in to_add:
|
||||
self._api.post_key(key, target_list)
|
||||
logging.info(f'Added key: {key}')
|
||||
|
@ -80,15 +87,38 @@ class MultiList:
|
|||
self._api.delete_key(key, target_list)
|
||||
logging.info(f'Removed key: {key}')
|
||||
|
||||
# Workaround for quirky gpg behaviour where some key signatures are exported from a sublist, but dropped on
|
||||
# import into the target list, leading to a situation where the same key is imported over and over again.
|
||||
new_subs = set()
|
||||
# Get the new list of subscribers
|
||||
for s in self._api.get_subscribers(target_list):
|
||||
if s.email in self._unmanaged or s.email == self._target:
|
||||
continue
|
||||
if s.key is None or s.key.fingerprint == target_list.fingerprint:
|
||||
continue
|
||||
new_subs.add(s)
|
||||
# Compare the key blobs to the ones present before this run
|
||||
old_keys = {s.key.blob for s in current_subs if s.key is not None}
|
||||
unchanged_subs = {s for s in new_subs if s.key is not None and s.key.blob in old_keys}
|
||||
unchanged_fprs = {s.key.fingerprint for s in unchanged_subs if s.key is not None}
|
||||
# Remove the unchanged keys from the changesets so that they are not included in the admin report
|
||||
to_subscribe = {s for s in to_subscribe if s not in unchanged_subs}
|
||||
to_update = {s for s in to_update if s not in unchanged_subs}
|
||||
# need to compare by fpr because == includes the (potentially different) blob
|
||||
to_add = {k for k in to_add if k.fingerprint not in unchanged_fprs}
|
||||
|
||||
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
|
||||
logging.info(f'No changes for {self._target}')
|
||||
else:
|
||||
for admin in target_admins:
|
||||
try:
|
||||
report = AdminReport(self._target, admin.email, self._mail_from,
|
||||
admin.key.blob if admin.key is not None else None,
|
||||
to_subscribe, to_unsubscribe, to_update, to_add, to_remove,
|
||||
conflicts)
|
||||
conflicts, sourcemap)
|
||||
self._reporter.add_message(report)
|
||||
except BaseException:
|
||||
logging.exception(f'Encryption to {admin.email} failed, not sending report')
|
||||
logging.info(f'Finished processing: {self._target}')
|
||||
|
||||
def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]:
|
||||
|
|
|
@ -27,24 +27,29 @@ class Message(abc.ABC):
|
|||
mail_from: str,
|
||||
mail_to: str,
|
||||
content: str,
|
||||
encrypt_to: List[str]):
|
||||
encrypt_to: List[str],
|
||||
encrypt_may_fail: bool = False):
|
||||
self._schleuder: str = schleuder
|
||||
self._from: str = mail_from
|
||||
self._to: str = mail_to
|
||||
self._keys: List[str] = encrypt_to
|
||||
self._mime: email.mime.base.MIMEBase = self._make_mime(content)
|
||||
self._mime: email.mime.base.MIMEBase = self._make_mime(content, encrypt_may_fail)
|
||||
|
||||
@property
|
||||
def mime(self) -> email.mime.base.MIMEBase:
|
||||
return self._mime
|
||||
|
||||
def _make_mime(self, content: str) -> email.mime.base.MIMEBase:
|
||||
def _make_mime(self, content: str, encrypt_may_fail: bool) -> email.mime.base.MIMEBase:
|
||||
# Encrypt to all keys, if possible. Fall back to unencrypted otherwise
|
||||
try:
|
||||
self._mime = self._encrypt_message(content)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
if encrypt_may_fail:
|
||||
logging.exception('Encryption failed; falling back to unencrypted message')
|
||||
self._mime = email.mime.text.MIMEText(content, _subtype='plain', _charset='utf-8')
|
||||
else:
|
||||
logging.exception('Encryption failed; Not sending this message')
|
||||
raise e
|
||||
# Set all the email headers
|
||||
self._mime['From'] = self._from
|
||||
self._mime['Reply-To'] = self._from
|
||||
|
@ -124,7 +129,8 @@ class KeyConflictMessage(Message):
|
|||
mail_from=mail_from,
|
||||
mail_to=chosen.email,
|
||||
content=content,
|
||||
encrypt_to=[s.key.blob for s in affected if s.key is not None]
|
||||
encrypt_to=[s.key.blob for s in affected if s.key is not None],
|
||||
encrypt_may_fail=True # Permit unencrypted fallback so the user gets notified of the conflict anyway
|
||||
)
|
||||
self.mime['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
|
||||
self.mime['X-MultiSchleuder-Digest'] = digest
|
||||
|
@ -174,7 +180,8 @@ class UserConflictMessage(Message):
|
|||
mail_from=mail_from,
|
||||
mail_to=subscriber,
|
||||
content=content,
|
||||
encrypt_to=[chosen.key.blob]
|
||||
encrypt_to=[chosen.key.blob],
|
||||
encrypt_may_fail=True # Permit unencrypted fallback so the user gets notified of the conflict anyway
|
||||
)
|
||||
self.mime['Subject'] = f'MultiSchleuder {self._schleuder} - Subscriber Conflict'
|
||||
self.mime['X-MultiSchleuder-Digest'] = digest
|
||||
|
@ -201,7 +208,8 @@ class AdminReport(Message):
|
|||
updated: Set[SchleuderSubscriber],
|
||||
added: Set[SchleuderKey],
|
||||
removed: Set[SchleuderKey],
|
||||
conflicts: List[Optional[Message]]):
|
||||
conflicts: List[Optional[Message]],
|
||||
sourcemap: Dict[int, str]):
|
||||
if len(subscribed) == 0 and len(unsubscribed) == 0 and len(removed) == 0 \
|
||||
and len(added) == 0 and len(updated) == 0 and len(conflicts) == 0:
|
||||
raise ValueError('No changes, not creating admin report')
|
||||
|
@ -226,15 +234,14 @@ class AdminReport(Message):
|
|||
>>> Subscribed:
|
||||
'''
|
||||
for s in subscribed:
|
||||
fpr = 'no key' if s.key is None else s.key.fingerprint
|
||||
content += f'{s.email} ({fpr})\n'
|
||||
sschleuder: str = sourcemap.get(s.schleuder, 'unknown')
|
||||
content += f'{s.email} ({sschleuder})\n'
|
||||
if len(unsubscribed) > 0:
|
||||
content += '''
|
||||
>>> Unsubscribed:
|
||||
'''
|
||||
for s in unsubscribed:
|
||||
fpr = 'no key' if s.key is None else s.key.fingerprint
|
||||
content += f'{s.email} ({fpr})\n'
|
||||
content += f'{s.email}\n'
|
||||
if len(updated) > 0:
|
||||
content += '''
|
||||
>>> Subscriber keys changed:
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
|
||||
import unittest
|
||||
import urllib.error
|
||||
from datetime import datetime
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
@ -111,7 +112,7 @@ _KEY_RESPONSE = '''
|
|||
|
||||
class TestSchleuderApi(unittest.TestCase):
|
||||
|
||||
def _mock_api(self, mock, nokey=False):
|
||||
def _mock_api(self, mock, nokey=False, error=None):
|
||||
m = MagicMock()
|
||||
m.getcode.return_value = 200
|
||||
|
||||
|
@ -133,8 +134,14 @@ class TestSchleuderApi(unittest.TestCase):
|
|||
m.read = read
|
||||
m.__enter__.return_value = m
|
||||
mock.return_value = m
|
||||
return SchleuderApi('https://localhost:4443',
|
||||
api = SchleuderApi('https://localhost:4443',
|
||||
'86cf2676d065dc902548e563ab22b57868ed2eb6')
|
||||
if error is not None:
|
||||
def __request(*args, **kwargs):
|
||||
raise error
|
||||
api._SchleuderApi__request = __request
|
||||
|
||||
return api
|
||||
|
||||
@patch('urllib.request.urlopen')
|
||||
def test_get_lists(self, mock):
|
||||
|
@ -282,6 +289,13 @@ class TestSchleuderApi(unittest.TestCase):
|
|||
self.assertEqual(42, key.schleuder)
|
||||
self.assertIn('-----BEGIN PGP PUBLIC KEY BLOCK-----', key.blob)
|
||||
|
||||
@patch('urllib.request.urlopen')
|
||||
def test_get_key_404(self, mock):
|
||||
url = 'https://localhost:4443/keys/ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA.json?list_id=42'
|
||||
api = self._mock_api(mock, error=urllib.error.HTTPError(url=url, code=404, msg='Not Found', hdrs={}, fp=None))
|
||||
key = api.get_key('ADB9BC679FF53CC8EF66FAC39348FDAB7A7663FA', SchleuderList(42, '', ''))
|
||||
self.assertIsNone(key)
|
||||
|
||||
@patch('urllib.request.urlopen')
|
||||
def test_post_key(self, mock):
|
||||
api = self._mock_api(mock)
|
||||
|
|
|
@ -123,7 +123,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
contents.seek(io.SEEK_END) # Opened with 'a+'
|
||||
with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
|
||||
mock_statefile().__enter__.return_value = contents
|
||||
resolved, messages = kcr.resolve('', '', [], [])
|
||||
resolved, messages = kcr.resolve('', '', [], {})
|
||||
self.assertEqual(0, len(resolved))
|
||||
self.assertEqual(0, len(messages))
|
||||
|
||||
|
@ -133,7 +133,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
|
||||
sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
|
||||
kcr = KeyConflictResolution(3600, '/tmp/state.json', _KEY_TEMPLATE, _USER_TEMPLATE)
|
||||
resolved, messages = kcr.resolve('', '', [sub1], [sch1])
|
||||
resolved, messages = kcr.resolve('', '', [sub1], {42: sch1})
|
||||
self.assertEqual(1, len(resolved))
|
||||
self.assertEqual(sub1, resolved[0])
|
||||
self.assertEqual(0, len(messages))
|
||||
|
@ -159,7 +159,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
|
||||
self.assertEqual(1, len(resolved))
|
||||
self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
|
||||
|
@ -187,7 +187,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
|
||||
self.assertEqual(1, len(resolved))
|
||||
self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint)
|
||||
|
@ -225,7 +225,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
|
||||
self.assertEqual(1, len(resolved))
|
||||
self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
|
||||
|
@ -246,7 +246,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1],
|
||||
sources=[sch1])
|
||||
sourcemap={42: sch1})
|
||||
self.assertEqual(0, len(resolved))
|
||||
self.assertEqual(0, len(messages))
|
||||
|
||||
|
@ -271,7 +271,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
|
||||
self.assertEqual(1, len(resolved))
|
||||
self.assertEqual('bar@example.org', resolved[0].email)
|
||||
|
@ -324,7 +324,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2, sub3],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
|
||||
self.assertEqual(2, len(resolved))
|
||||
foo, bar = resolved
|
||||
|
@ -376,7 +376,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(0, len(msgs))
|
||||
|
||||
def test_send_messages_brokenstate(self):
|
||||
|
@ -399,7 +399,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(0, len(msgs))
|
||||
|
||||
def test_send_messages_emptystate(self):
|
||||
|
@ -421,7 +421,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(1, len(msgs))
|
||||
|
||||
now = datetime.utcnow().timestamp()
|
||||
|
@ -452,7 +452,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(1, len(msgs))
|
||||
|
||||
now = datetime.utcnow().timestamp()
|
||||
|
@ -483,7 +483,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(1, len(messages))
|
||||
msg = messages[0]
|
||||
|
||||
|
@ -515,7 +515,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(0, len(messages))
|
||||
|
||||
now = datetime.utcnow().timestamp()
|
||||
|
@ -549,7 +549,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2],
|
||||
sources=[sch1, sch2])
|
||||
sourcemap={42: sch1, 23: sch2})
|
||||
self.assertEqual(1, len(messages))
|
||||
|
||||
now = datetime.utcnow().timestamp()
|
||||
|
@ -582,7 +582,7 @@ class TestKeyConflictResolution(unittest.TestCase):
|
|||
target='test@schleuder.example.org',
|
||||
mail_from='test-owner@schleuder.example.org',
|
||||
subscriptions=[sub1, sub2, sub3],
|
||||
sources=[sch1, sch2, sch3])
|
||||
sourcemap={42: sch1, 23: sch2, 7: sch1})
|
||||
self.assertEqual(1, len(messages))
|
||||
msg = messages[0].mime
|
||||
pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))
|
||||
|
|
|
@ -9,6 +9,7 @@ from dateutil.tz import tzutc
|
|||
|
||||
from multischleuder.processor import MultiList
|
||||
from multischleuder.reporting import Message
|
||||
from multischleuder.test.test_conflict import _PRIVKEY_1
|
||||
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
|
||||
|
||||
|
||||
|
@ -38,7 +39,7 @@ def _list_lists():
|
|||
|
||||
def _get_key(fpr: str, schleuder: SchleuderList):
|
||||
key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
|
||||
'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
|
||||
'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
|
||||
key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
|
||||
'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
|
||||
key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',
|
||||
|
@ -72,7 +73,7 @@ def _get_admins(schleuder: SchleuderList):
|
|||
if schleuder.id != 2:
|
||||
return []
|
||||
key = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
|
||||
'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
|
||||
'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
|
||||
date = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
|
||||
admin = SchleuderSubscriber(0, 'admin@example.org', key, schleuder.id, date)
|
||||
return [admin]
|
||||
|
@ -80,7 +81,7 @@ def _get_admins(schleuder: SchleuderList):
|
|||
|
||||
def _get_subs(schleuder: SchleuderList):
|
||||
key1 = SchleuderKey('966842467B3254143F994D5E5C408C012D216471',
|
||||
'admin@example.org', 'BEGIN PGP 2D216471', schleuder.id)
|
||||
'admin@example.org', str(_PRIVKEY_1.pubkey), schleuder.id)
|
||||
key2 = SchleuderKey('6449FFB6EE68187962FA013B5CA2F4F51791BAF6',
|
||||
'ada.lovelace@example.org', 'BEGIN PGP 1791BAF6', schleuder.id)
|
||||
key3 = SchleuderKey('414D3960D34730F63C74D5190EBC5A16716DEC79',
|
||||
|
|
|
@ -3,11 +3,27 @@ import unittest
|
|||
|
||||
from datetime import datetime
|
||||
|
||||
import pgpy.errors # type: ignore
|
||||
|
||||
from multischleuder.reporting import KeyConflictMessage, AdminReport, Reporter, UserConflictMessage
|
||||
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
|
||||
from multischleuder.test.test_conflict import _PRIVKEY_1
|
||||
|
||||
|
||||
BROKENKEY = '''
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mDMEYmcMbxYJKwYBBAHaRw8BAQdAKUohRdnuTSldKwawfLdwwUvOJjz/pHx3fXS2
|
||||
v2dUQx+0SU11bHRpc2NobGV1ZGVyIEJyb2tlbiBBZG1pbiBLZXkgKFRFU1QgS0VZ
|
||||
IERPIE5PVCBVU0UpIDxhZG1pbkBleGFtcGxlLm9yZz6IkAQTFggAOBYhBGtuFOnz
|
||||
PJOCOdfv6OuAwhfh1Uj8BQJiZwxvAhsBBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA
|
||||
AAoJEOuAwhfh1Uj8PnkBAM6PfYUZbvvYEkSdwzmZXDwhPRsSA0bhjL5aVwIeCCdp
|
||||
AQDeImNI6czSLVAuwObKv8FnpmbFi3HxTNzakp44DoD8Aw==
|
||||
=JtdI
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
'''
|
||||
|
||||
|
||||
def one_of_each_kind():
|
||||
sub = SchleuderSubscriber(1, 'foo@example.org', None, 1, datetime.utcnow())
|
||||
key = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), 1)
|
||||
|
@ -24,13 +40,14 @@ def one_of_each_kind():
|
|||
schleuder='test@example.org',
|
||||
mail_to='admin@example.org',
|
||||
mail_from='test-owner@example.org',
|
||||
encrypt_to=None,
|
||||
encrypt_to=str(_PRIVKEY_1.pubkey),
|
||||
subscribed={},
|
||||
unsubscribed={sub},
|
||||
updated={},
|
||||
added={},
|
||||
removed={},
|
||||
conflicts=[])
|
||||
conflicts=[],
|
||||
sourcemap={1: 'test@example.org'})
|
||||
msg3 = UserConflictMessage(
|
||||
schleuder='test@example.org',
|
||||
subscriber='bar@example.org',
|
||||
|
@ -94,3 +111,19 @@ class TestReporting(unittest.TestCase):
|
|||
r.add_messages([None])
|
||||
self.assertEqual(0, len(Reporter.get_messages()))
|
||||
Reporter.clear_messages()
|
||||
|
||||
def test_admin_report_nokey(self):
|
||||
sub = SchleuderSubscriber(1, 'foo@example.org', None, 1, datetime.utcnow())
|
||||
with self.assertRaises(pgpy.errors.PGPError):
|
||||
AdminReport(
|
||||
schleuder='test@example.org',
|
||||
mail_to='admin@example.org',
|
||||
mail_from='test-owner@example.org',
|
||||
encrypt_to=BROKENKEY,
|
||||
subscribed={sub},
|
||||
unsubscribed={},
|
||||
updated={},
|
||||
added={},
|
||||
removed={},
|
||||
conflicts=[],
|
||||
sourcemap={1: 'test@example.org'})
|
||||
|
|
|
@ -188,7 +188,8 @@ class TestSmtpClient(unittest.TestCase):
|
|||
updated={},
|
||||
added={},
|
||||
removed={},
|
||||
conflicts=[])
|
||||
conflicts=[],
|
||||
sourcemap={1: 'foo@example.org'})
|
||||
client.send_messages([msg1, msg2])
|
||||
ctrl.stop()
|
||||
self.assertTrue(ctrl.handler.connected)
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
Package: multischleuder
|
||||
Version: __MULTISCHLEUDER_VERSION__
|
||||
Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
|
||||
Version: __VERSION__
|
||||
Maintainer: s3lph <s3lph@kabelsalat.ch>
|
||||
Section: web
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
|
|
|
@ -1,183 +0,0 @@
|
|||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib.request
|
||||
import http.client
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def parse_changelog(tag: str) -> Optional[str]:
|
||||
release_changelog: str = ''
|
||||
with open('CHANGELOG.md', 'r') as f:
|
||||
in_target: bool = False
|
||||
done: bool = False
|
||||
for line in f.readlines():
|
||||
if in_target:
|
||||
if f'<!-- END RELEASE {tag} -->' in line:
|
||||
done = True
|
||||
break
|
||||
release_changelog += line
|
||||
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
|
||||
in_target = True
|
||||
continue
|
||||
if not done:
|
||||
return None
|
||||
return release_changelog
|
||||
|
||||
|
||||
def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
|
||||
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'User-Agent': 'curl/7.70.0'
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.read()
|
||||
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
|
||||
|
||||
jobidmap: Dict[str, str] = {}
|
||||
for job in joblist:
|
||||
name: str = job['name']
|
||||
job_id: str = job['id']
|
||||
jobidmap[name] = job_id
|
||||
return jobidmap
|
||||
|
||||
|
||||
def fetch_single_shafile(url: str, api_token: str) -> str:
|
||||
headers: Dict[str, str] = {
|
||||
'User-Agent': 'curl/7.70.0',
|
||||
'Private-Token': api_token
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.readline()
|
||||
shafile: str = resp_data.decode()
|
||||
filename: str = shafile.strip().split(' ')[-1].strip()
|
||||
return filename
|
||||
|
||||
|
||||
def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
|
||||
wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
|
||||
'/artifacts/dist/SHA256SUMS'
|
||||
wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
|
||||
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
|
||||
return wheel_url, wheel_sha_url
|
||||
|
||||
|
||||
def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
|
||||
debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
|
||||
'/artifacts/package/debian/SHA256SUMS'
|
||||
debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
|
||||
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
|
||||
return debian_url, debian_sha_url
|
||||
|
||||
|
||||
def main():
|
||||
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
|
||||
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
|
||||
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
|
||||
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
|
||||
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
|
||||
if api_token is None:
|
||||
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if release_tag is None:
|
||||
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_name is None:
|
||||
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_id is None:
|
||||
print('CI_PROJECT_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if pipeline_id is None:
|
||||
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
changelog: Optional[str] = parse_changelog(release_tag)
|
||||
if changelog is None:
|
||||
print('Changelog could not be parsed.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
|
||||
|
||||
base_url: str = f'https://gitlab.com/{project_name}/-'
|
||||
|
||||
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
|
||||
debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
|
||||
|
||||
augmented_changelog = f'''{changelog.strip()}
|
||||
|
||||
### Download
|
||||
|
||||
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
|
||||
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
|
||||
# Docker currently not working
|
||||
# - Docker image: registry.gitlab.com/{project_name}:{release_tag}
|
||||
|
||||
post_body: str = json.dumps({
|
||||
'tag_name': release_tag,
|
||||
'description': augmented_changelog,
|
||||
'assets': {
|
||||
'links': [
|
||||
{
|
||||
'name': 'Python Wheel',
|
||||
'url': wheel_url,
|
||||
'link_type': 'package'
|
||||
},
|
||||
{
|
||||
'name': 'Debian Package',
|
||||
'url': debian_url,
|
||||
'link_type': 'package'
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
gitlab_release_api_url: str = \
|
||||
f'https://gitlab.com/api/v4/projects/{project_id}/releases'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'User-Agent': 'curl/7.70.0'
|
||||
}
|
||||
|
||||
request = urllib.request.Request(
|
||||
gitlab_release_api_url,
|
||||
post_body.encode('utf-8'),
|
||||
headers=headers,
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
response: http.client.HTTPResponse = urllib.request.urlopen(request)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
response_bytes: bytes = response.read()
|
||||
response_str: str = response_bytes.decode()
|
||||
response_data: Dict[str, Any] = json.loads(response_str)
|
||||
|
||||
if response_data['tag_name'] != release_tag:
|
||||
print('Something went wrong...', file=sys.stderr)
|
||||
print(response_str, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(response_data['description'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
14
setup.py
14
setup.py
|
@ -7,17 +7,27 @@ setup(
|
|||
name='multischleuder',
|
||||
version=__version__,
|
||||
author='s3lph',
|
||||
author_email='1375407-s3lph@users.noreply.gitlab.com',
|
||||
author_email='s3lph@kabelsalat.ch',
|
||||
description='Merge subscribers and keys of multiple Schleuder lists into one',
|
||||
license='MIT',
|
||||
keywords='schleuder,pgp',
|
||||
url='https://gitlab.com/s3lph/multischleuder',
|
||||
url='https://git.kabelsalat.ch/s3lph/multischleuder',
|
||||
packages=find_packages(exclude=['*.test']),
|
||||
install_requires=[
|
||||
'python-dateutil',
|
||||
'PyYAML',
|
||||
'PGPy',
|
||||
],
|
||||
extras_require={
|
||||
'test': [
|
||||
'aiosmtpd',
|
||||
'coverage',
|
||||
'pycodestyle',
|
||||
'mypy',
|
||||
'deepdiff',
|
||||
'twine'
|
||||
]
|
||||
},
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'multischleuder = multischleuder.main:main'
|
||||
|
|
Loading…
Reference in a new issue