Compare commits
No commits in common. "main" and "fix-key-404" have entirely different histories.
main
...
fix-key-40
10 changed files with 360 additions and 213 deletions
|
@ -1,38 +0,0 @@
|
||||||
---
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- "v*"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
|
|
||||||
build_wheel:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- name: Build Python wheel
|
|
||||||
run: |
|
|
||||||
apt update; apt install -y python3-pip
|
|
||||||
pip3 install --break-system-packages -e .[test]
|
|
||||||
python3 setup.py egg_info bdist_wheel
|
|
||||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.API_USERNAME }}
|
|
||||||
password: ${{ secrets.API_PASSWORD }}
|
|
||||||
|
|
||||||
build_debian:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5
|
|
||||||
with:
|
|
||||||
python_module: multischleuder
|
|
||||||
package_name: multischleuder
|
|
||||||
package_root: package/debian/multischleuder
|
|
||||||
package_output_path: package/debian
|
|
||||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.API_USERNAME }}
|
|
||||||
password: ${{ secrets.API_PASSWORD }}
|
|
||||||
deb: "package/debian/*.deb"
|
|
|
@ -1,77 +0,0 @@
|
||||||
---
|
|
||||||
|
|
||||||
on: push
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
|
|
||||||
test:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- name: Unit Tests
|
|
||||||
run: |
|
|
||||||
apt update; apt install -y python3-pip
|
|
||||||
pip3 install --break-system-packages -e .[test]
|
|
||||||
python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
|
|
||||||
python3 -m coverage combine
|
|
||||||
python3 -m coverage report --rcfile=setup.cfg
|
|
||||||
|
|
||||||
codestyle:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- name: Code Style
|
|
||||||
run: |
|
|
||||||
apt update; apt install -y python3-pip
|
|
||||||
pip3 install --break-system-packages -e .[test]
|
|
||||||
pycodestyle multischleuder
|
|
||||||
|
|
||||||
mypy:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- name: Static Type Checks
|
|
||||||
run: |
|
|
||||||
apt update; apt install -y python3-pip
|
|
||||||
pip3 install --break-system-packages -e .[test]
|
|
||||||
pip3 install --break-system-packages types-PyYAML types-python-dateutil
|
|
||||||
mypy multischleuder
|
|
||||||
|
|
||||||
schleuder:
|
|
||||||
runs-on: docker
|
|
||||||
steps:
|
|
||||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
|
||||||
- name: Integration Test against schleuder
|
|
||||||
run: |
|
|
||||||
echo "postfix postfix/mailname string example.org" | debconf-set-selections
|
|
||||||
echo "postfix postfix/main_mailer_type string 'Local only'" | debconf-set-selections
|
|
||||||
apt update; apt install --yes python3-pip schleuder schleuder-cli postfix patch
|
|
||||||
pip3 install --break-system-packages -e .[test]
|
|
||||||
/usr/lib/postfix/configure-instance.sh -
|
|
||||||
echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
|
|
||||||
/usr/sbin/postmulti -i - -p start
|
|
||||||
schleuder-cli lists list || true
|
|
||||||
export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
|
|
||||||
echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
|
|
||||||
cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
|
|
||||||
host: localhost
|
|
||||||
port: 4443
|
|
||||||
tls_fingerprint: ${CERT_FPR}
|
|
||||||
api_key: '00000000000000000000000000000000'
|
|
||||||
EOF
|
|
||||||
/usr/bin/schleuder-api-daemon &
|
|
||||||
sleep 5 # wait for daemons to start
|
|
||||||
export API_DAEMON_PID=$!
|
|
||||||
test/prepare-schleuder.sh
|
|
||||||
pip3 install --break-system-packages -e .
|
|
||||||
python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
|
|
||||||
python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
|
||||||
# Run a second time - should be idempotent and not trigger any new mails
|
|
||||||
python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
|
||||||
python3 -m coverage combine
|
|
||||||
python3 -m coverage report --rcfile=setup.cfg
|
|
||||||
sleep 5 # wait for mail delivery
|
|
||||||
test/report.py
|
|
||||||
kill -9 ${API_DAEMON_PID} || true
|
|
||||||
/usr/sbin/postmulti -i - -p stop
|
|
||||||
sleep 5 # wait for daemons to terminate
|
|
163
.gitlab-ci.yml
Normal file
163
.gitlab-ci.yml
Normal file
|
@ -0,0 +1,163 @@
|
||||||
|
---
|
||||||
|
image: python:3.9-bullseye
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- test
|
||||||
|
- coverage
|
||||||
|
- build
|
||||||
|
- deploy
|
||||||
|
- upload
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
before_script:
|
||||||
|
- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff
|
||||||
|
- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
test:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- pip3 install -e .
|
||||||
|
- python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- ".coverage*"
|
||||||
|
|
||||||
|
codestyle:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- pip3 install -e .
|
||||||
|
- pycodestyle multischleuder
|
||||||
|
|
||||||
|
mypy:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- pip3 install -e .
|
||||||
|
- mypy --install-types --non-interactive multischleuder
|
||||||
|
- mypy multischleuder
|
||||||
|
|
||||||
|
sast:
|
||||||
|
stage: test
|
||||||
|
bandit-sast:
|
||||||
|
before_script: ['']
|
||||||
|
include:
|
||||||
|
- template: Security/SAST.gitlab-ci.yml
|
||||||
|
- template: Security/Dependency-Scanning.gitlab-ci.yml
|
||||||
|
|
||||||
|
schleuder:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- debconf-set-selections <<<"postfix postfix/mailname string example.org"
|
||||||
|
- debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'"
|
||||||
|
- apt update; apt install --yes schleuder schleuder-cli postfix
|
||||||
|
- /usr/lib/postfix/configure-instance.sh -
|
||||||
|
- echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf
|
||||||
|
- /usr/sbin/postmulti -i - -p start
|
||||||
|
- schleuder-cli lists list || true
|
||||||
|
- export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4)
|
||||||
|
- echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml
|
||||||
|
- |
|
||||||
|
cat > ~/.schleuder-cli/schleuder-cli.yml <<EOF
|
||||||
|
host: localhost
|
||||||
|
port: 4443
|
||||||
|
tls_fingerprint: ${CERT_FPR}
|
||||||
|
api_key: '00000000000000000000000000000000'
|
||||||
|
EOF
|
||||||
|
- /usr/bin/schleuder-api-daemon &
|
||||||
|
- sleep 5 # wait for daemons to start
|
||||||
|
- export API_DAEMON_PID=$!
|
||||||
|
- test/prepare-schleuder.sh
|
||||||
|
- pip3 install -e .
|
||||||
|
- python3 -c 'import os; print(os.listdir(".")); print(); print(os.listdir("test/"))'
|
||||||
|
- python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||||
|
# Run a second time - should be idempotent and not trigger any new mails
|
||||||
|
- python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
|
||||||
|
- sleep 5 # wait for mail delivery
|
||||||
|
- test/report.py
|
||||||
|
- kill -9 ${API_DAEMON_PID} || true
|
||||||
|
- /usr/sbin/postmulti -i - -p stop
|
||||||
|
- sleep 5 # wait for daemons to terminate
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- ".coverage*"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
coverage:
|
||||||
|
stage: coverage
|
||||||
|
coverage: >-
|
||||||
|
/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
|
||||||
|
script:
|
||||||
|
- python3 -m coverage combine
|
||||||
|
- python3 -m coverage report --rcfile=setup.cfg
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
build_wheel:
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- python3 setup.py egg_info bdist_wheel
|
||||||
|
- cd dist
|
||||||
|
- sha256sum *.whl > SHA256SUMS
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- "dist/*.whl"
|
||||||
|
- dist/SHA256SUMS
|
||||||
|
only:
|
||||||
|
- tags
|
||||||
|
|
||||||
|
build_debian:
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- apt update && apt install --yes lintian rsync sudo
|
||||||
|
- echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||||
|
- |
|
||||||
|
for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
|
||||||
|
echo "multischleuder (${version}-1); urgency=medium\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||||
|
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||||
|
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||||
|
done
|
||||||
|
- gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog
|
||||||
|
- python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1
|
||||||
|
- cd package/debian
|
||||||
|
- sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control
|
||||||
|
- mkdir -p multischleuder/usr/lib/python3/dist-packages/
|
||||||
|
- rsync -a multischleuder/usr/lib/python3.9/site-packages/ multischleuder/usr/lib/python3/dist-packages/
|
||||||
|
- rm -rf multischleuder/usr/lib/python3.9/site-packages
|
||||||
|
- find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
|
||||||
|
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
|
||||||
|
- find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
|
||||||
|
- sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder
|
||||||
|
- find multischleuder -type f -exec chmod 0644 {} \;
|
||||||
|
- find multischleuder -type d -exec chmod 755 {} \;
|
||||||
|
- chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm
|
||||||
|
- dpkg-deb --build multischleuder
|
||||||
|
- mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||||
|
- sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb"
|
||||||
|
- sha256sum *.deb > SHA256SUMS
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- "package/debian/*.deb"
|
||||||
|
- package/debian/SHA256SUMS
|
||||||
|
only:
|
||||||
|
- tags
|
||||||
|
|
||||||
|
|
||||||
|
release:
|
||||||
|
stage: deploy
|
||||||
|
script:
|
||||||
|
- python3 package/release.py
|
||||||
|
only:
|
||||||
|
- tags
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
repo:
|
||||||
|
stage: upload
|
||||||
|
trigger: s3lph/custom-packages
|
||||||
|
variables:
|
||||||
|
MULTIPROJECT_TRIGGER_JOBNAME: multischleuder
|
||||||
|
only:
|
||||||
|
- tags
|
53
CHANGELOG.md
53
CHANGELOG.md
|
@ -1,58 +1,5 @@
|
||||||
# MultiSchleuder Changelog
|
# MultiSchleuder Changelog
|
||||||
|
|
||||||
<!-- BEGIN RELEASE v0.1.9 -->
|
|
||||||
## Version 0.1.9
|
|
||||||
|
|
||||||
Maintenance Release
|
|
||||||
|
|
||||||
### Changes
|
|
||||||
|
|
||||||
<!-- BEGIN CHANGES 0.1.9 -->
|
|
||||||
- Migrate from Woodpecker to Forgejo Actions
|
|
||||||
<!-- END CHANGES 0.1.9 -->
|
|
||||||
|
|
||||||
<!-- END RELEASE v0.1.9 -->
|
|
||||||
|
|
||||||
<!-- BEGIN RELEASE v0.1.8 -->
|
|
||||||
## Version 0.1.8
|
|
||||||
|
|
||||||
Maintenance Release
|
|
||||||
|
|
||||||
### Changes
|
|
||||||
|
|
||||||
<!-- BEGIN CHANGES 0.1.8 -->
|
|
||||||
- Migrate from Gitlab-CI to Woodpecker
|
|
||||||
<!-- END CHANGES 0.1.8 -->
|
|
||||||
|
|
||||||
<!-- END RELEASE v0.1.8 -->
|
|
||||||
|
|
||||||
<!-- BEGIN RELEASE v0.1.7 -->
|
|
||||||
## Version 0.1.7
|
|
||||||
|
|
||||||
Bugfix Release
|
|
||||||
|
|
||||||
### Changes
|
|
||||||
|
|
||||||
<!-- BEGIN CHANGES 0.1.7 -->
|
|
||||||
- Remove and re-import keys whose expiry date has been changed
|
|
||||||
- Don't report keys as changed if they appear to differ, but are treated as identical by GnuPG.
|
|
||||||
<!-- END CHANGES 0.1.7 -->
|
|
||||||
|
|
||||||
<!-- END RELEASE v0.1.7 -->
|
|
||||||
|
|
||||||
<!-- BEGIN RELEASE v0.1.6 -->
|
|
||||||
## Version 0.1.6
|
|
||||||
|
|
||||||
Bugfix Release
|
|
||||||
|
|
||||||
### Changes
|
|
||||||
|
|
||||||
<!-- BEGIN CHANGES 0.1.6 -->
|
|
||||||
- Better error handling for wrongfully configured keys
|
|
||||||
<!-- END CHANGES 0.1.6 -->
|
|
||||||
|
|
||||||
<!-- END RELEASE v0.1.6 -->
|
|
||||||
|
|
||||||
<!-- BEGIN RELEASE v0.1.5 -->
|
<!-- BEGIN RELEASE v0.1.5 -->
|
||||||
## Version 0.1.5
|
## Version 0.1.5
|
||||||
|
|
||||||
|
|
13
README.md
13
README.md
|
@ -1,5 +1,10 @@
|
||||||
# MultiSchleuder
|
# MultiSchleuder
|
||||||
|
|
||||||
|
[![pipeline status](https://gitlab.com/s3lph/multischleuder/badges/main/pipeline.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
|
||||||
|
[![coverage report](https://gitlab.com/s3lph/multischleuder/badges/main/coverage.svg)](https://gitlab.com/s3lph/multischleuder/-/commits/main)
|
||||||
|
[![latest release](https://gitlab.com/s3lph/multischleuder/-/badges/release.svg)](https://gitlab.com/s3lph/multischleuder/-/releases)
|
||||||
|
[![license](https://img.shields.io/badge/License-MIT-yellow.svg)](https://gitlab.com/s3lph/multischleuder/-/blob/main/LICENSE)
|
||||||
|
|
||||||
Automatically and periodically merge subscribers and keys of multiple [Schleuder][schleuder] lists into one.
|
Automatically and periodically merge subscribers and keys of multiple [Schleuder][schleuder] lists into one.
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
@ -10,7 +15,7 @@ Automatically and periodically merge subscribers and keys of multiple [Schleuder
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
You can find Debian packages and Python wheels over at [Packages][packages].
|
You can find Debian packages and Python wheels over at [Releases][releases].
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
|
@ -173,11 +178,11 @@ MultiSchleuder resolves conflicts in a simple, but primitive manner:
|
||||||
1. Then it checks whether a subscriber has more than one key. If so, the key used by the oldest subscription wins.
|
1. Then it checks whether a subscriber has more than one key. If so, the key used by the oldest subscription wins.
|
||||||
|
|
||||||
This is by no means a perfect solution.
|
This is by no means a perfect solution.
|
||||||
It does however yield consistent results.
|
It does however yield consisitent results.
|
||||||
In both cases, if configured to do so, MultiSchleuder will send a notification message to all subscribers involved in a conflict, encrypting it with all keys involved in the conflict.
|
In both cases, if configured to do so, MultiSchleuder will send a notification message to all subscribers involved in a conflict, encrypting it with all keys involved in the conflict.
|
||||||
If one or more keys are - for whatever reason - unusable, the message will not be encrypted.
|
If one or more keys are - for whatever reason - unusable, the message will not be encrypted.
|
||||||
This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibility would be to not notify a subscriber when something potentially malicious is going on.
|
This is a deliberate decision, since the amount of metadata possibly leaked from such a message is fairly small, and we consider it worth taking this risk, given that the other possibilty would be to not notify a subscriber when something potentially malicious is going on.
|
||||||
|
|
||||||
|
|
||||||
[schleuder]: https://schleuder.org/
|
[schleuder]: https://schleuder.org/
|
||||||
[packages]: https://git.kabelsalat.ch/s3lph/multischleuder/packages
|
[releases]: https://gitlab.com/s3lph/multischleuder/-/releases
|
|
@ -1,2 +1,2 @@
|
||||||
|
|
||||||
__version__ = '0.1.9'
|
__version__ = '0.1.5'
|
||||||
|
|
|
@ -63,14 +63,8 @@ class MultiList:
|
||||||
to_unsubscribe = current_subs.difference(intended_subs)
|
to_unsubscribe = current_subs.difference(intended_subs)
|
||||||
to_remove = current_keys.difference(intended_keys)
|
to_remove = current_keys.difference(intended_keys)
|
||||||
to_add = intended_keys.difference(current_keys)
|
to_add = intended_keys.difference(current_keys)
|
||||||
# Already present keys that are being updated have to be removed and re-imported for convergence
|
|
||||||
to_pre_remove = {k for k in to_add if k.fingerprint in {o.fingerprint for o in to_remove}}
|
|
||||||
to_remove = {k for k in to_remove if k.fingerprint not in {o.fingerprint for o in to_pre_remove}}
|
|
||||||
to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
|
to_update = {s for s in intended_subs if s in current_subs and s.key in to_add}
|
||||||
# Perform the actual list modifications in an order which avoids race conditions
|
# Perform the actual list modifications in an order which avoids race conditions
|
||||||
for key in to_pre_remove:
|
|
||||||
self._api.delete_key(key, target_list)
|
|
||||||
logging.info(f'Pre-removed key: {key}')
|
|
||||||
for key in to_add:
|
for key in to_add:
|
||||||
self._api.post_key(key, target_list)
|
self._api.post_key(key, target_list)
|
||||||
logging.info(f'Added key: {key}')
|
logging.info(f'Added key: {key}')
|
||||||
|
@ -87,26 +81,6 @@ class MultiList:
|
||||||
self._api.delete_key(key, target_list)
|
self._api.delete_key(key, target_list)
|
||||||
logging.info(f'Removed key: {key}')
|
logging.info(f'Removed key: {key}')
|
||||||
|
|
||||||
# Workaround for quirky gpg behaviour where some key signatures are exported from a sublist, but dropped on
|
|
||||||
# import into the target list, leading to a situation where the same key is imported over and over again.
|
|
||||||
new_subs = set()
|
|
||||||
# Get the new list of subscribers
|
|
||||||
for s in self._api.get_subscribers(target_list):
|
|
||||||
if s.email in self._unmanaged or s.email == self._target:
|
|
||||||
continue
|
|
||||||
if s.key is None or s.key.fingerprint == target_list.fingerprint:
|
|
||||||
continue
|
|
||||||
new_subs.add(s)
|
|
||||||
# Compare the key blobs to the ones present before this run
|
|
||||||
old_keys = {s.key.blob for s in current_subs if s.key is not None}
|
|
||||||
unchanged_subs = {s for s in new_subs if s.key is not None and s.key.blob in old_keys}
|
|
||||||
unchanged_fprs = {s.key.fingerprint for s in unchanged_subs if s.key is not None}
|
|
||||||
# Remove the unchanged keys from the changesets so that they are not included in the admin report
|
|
||||||
to_subscribe = {s for s in to_subscribe if s not in unchanged_subs}
|
|
||||||
to_update = {s for s in to_update if s not in unchanged_subs}
|
|
||||||
# need to compare by fpr because == includes the (potentially different) blob
|
|
||||||
to_add = {k for k in to_add if k.fingerprint not in unchanged_fprs}
|
|
||||||
|
|
||||||
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
|
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
|
||||||
logging.info(f'No changes for {self._target}')
|
logging.info(f'No changes for {self._target}')
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
Package: multischleuder
|
Package: multischleuder
|
||||||
Version: __VERSION__
|
Version: __MULTISCHLEUDER_VERSION__
|
||||||
Maintainer: s3lph <s3lph@kabelsalat.ch>
|
Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com>
|
||||||
Section: web
|
Section: web
|
||||||
Priority: optional
|
Priority: optional
|
||||||
Architecture: all
|
Architecture: all
|
||||||
|
|
183
package/release.py
Executable file
183
package/release.py
Executable file
|
@ -0,0 +1,183 @@
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import http.client
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
|
||||||
|
|
||||||
|
def parse_changelog(tag: str) -> Optional[str]:
|
||||||
|
release_changelog: str = ''
|
||||||
|
with open('CHANGELOG.md', 'r') as f:
|
||||||
|
in_target: bool = False
|
||||||
|
done: bool = False
|
||||||
|
for line in f.readlines():
|
||||||
|
if in_target:
|
||||||
|
if f'<!-- END RELEASE {tag} -->' in line:
|
||||||
|
done = True
|
||||||
|
break
|
||||||
|
release_changelog += line
|
||||||
|
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
|
||||||
|
in_target = True
|
||||||
|
continue
|
||||||
|
if not done:
|
||||||
|
return None
|
||||||
|
return release_changelog
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
|
||||||
|
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
|
||||||
|
headers: Dict[str, str] = {
|
||||||
|
'Private-Token': api_token,
|
||||||
|
'User-Agent': 'curl/7.70.0'
|
||||||
|
}
|
||||||
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
try:
|
||||||
|
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||||
|
except HTTPError as e:
|
||||||
|
print(e.read().decode())
|
||||||
|
sys.exit(1)
|
||||||
|
resp_data: bytes = resp.read()
|
||||||
|
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
|
||||||
|
|
||||||
|
jobidmap: Dict[str, str] = {}
|
||||||
|
for job in joblist:
|
||||||
|
name: str = job['name']
|
||||||
|
job_id: str = job['id']
|
||||||
|
jobidmap[name] = job_id
|
||||||
|
return jobidmap
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_single_shafile(url: str, api_token: str) -> str:
|
||||||
|
headers: Dict[str, str] = {
|
||||||
|
'User-Agent': 'curl/7.70.0',
|
||||||
|
'Private-Token': api_token
|
||||||
|
}
|
||||||
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
try:
|
||||||
|
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||||
|
except HTTPError as e:
|
||||||
|
print(e.read().decode())
|
||||||
|
sys.exit(1)
|
||||||
|
resp_data: bytes = resp.readline()
|
||||||
|
shafile: str = resp_data.decode()
|
||||||
|
filename: str = shafile.strip().split(' ')[-1].strip()
|
||||||
|
return filename
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||||
|
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
|
||||||
|
wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\
|
||||||
|
'/artifacts/dist/SHA256SUMS'
|
||||||
|
wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token)
|
||||||
|
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
|
||||||
|
return wheel_url, wheel_sha_url
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]:
|
||||||
|
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
|
||||||
|
debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\
|
||||||
|
'/artifacts/package/debian/SHA256SUMS'
|
||||||
|
debian_filename: str = fetch_single_shafile(debian_sha_url, api_token)
|
||||||
|
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
|
||||||
|
return debian_url, debian_sha_url
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
|
||||||
|
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
|
||||||
|
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
|
||||||
|
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
|
||||||
|
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
|
||||||
|
if api_token is None:
|
||||||
|
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if release_tag is None:
|
||||||
|
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if project_name is None:
|
||||||
|
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if project_id is None:
|
||||||
|
print('CI_PROJECT_ID is not set.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if pipeline_id is None:
|
||||||
|
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
changelog: Optional[str] = parse_changelog(release_tag)
|
||||||
|
if changelog is None:
|
||||||
|
print('Changelog could not be parsed.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
|
||||||
|
|
||||||
|
base_url: str = f'https://gitlab.com/{project_name}/-'
|
||||||
|
|
||||||
|
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token)
|
||||||
|
debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token)
|
||||||
|
|
||||||
|
augmented_changelog = f'''{changelog.strip()}
|
||||||
|
|
||||||
|
### Download
|
||||||
|
|
||||||
|
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
|
||||||
|
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
|
||||||
|
# Docker currently not working
|
||||||
|
# - Docker image: registry.gitlab.com/{project_name}:{release_tag}
|
||||||
|
|
||||||
|
post_body: str = json.dumps({
|
||||||
|
'tag_name': release_tag,
|
||||||
|
'description': augmented_changelog,
|
||||||
|
'assets': {
|
||||||
|
'links': [
|
||||||
|
{
|
||||||
|
'name': 'Python Wheel',
|
||||||
|
'url': wheel_url,
|
||||||
|
'link_type': 'package'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Debian Package',
|
||||||
|
'url': debian_url,
|
||||||
|
'link_type': 'package'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
gitlab_release_api_url: str = \
|
||||||
|
f'https://gitlab.com/api/v4/projects/{project_id}/releases'
|
||||||
|
headers: Dict[str, str] = {
|
||||||
|
'Private-Token': api_token,
|
||||||
|
'Content-Type': 'application/json; charset=utf-8',
|
||||||
|
'User-Agent': 'curl/7.70.0'
|
||||||
|
}
|
||||||
|
|
||||||
|
request = urllib.request.Request(
|
||||||
|
gitlab_release_api_url,
|
||||||
|
post_body.encode('utf-8'),
|
||||||
|
headers=headers,
|
||||||
|
method='POST'
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
response: http.client.HTTPResponse = urllib.request.urlopen(request)
|
||||||
|
except HTTPError as e:
|
||||||
|
print(e.read().decode())
|
||||||
|
sys.exit(1)
|
||||||
|
response_bytes: bytes = response.read()
|
||||||
|
response_str: str = response_bytes.decode()
|
||||||
|
response_data: Dict[str, Any] = json.loads(response_str)
|
||||||
|
|
||||||
|
if response_data['tag_name'] != release_tag:
|
||||||
|
print('Something went wrong...', file=sys.stderr)
|
||||||
|
print(response_str, file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(response_data['description'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
14
setup.py
14
setup.py
|
@ -7,27 +7,17 @@ setup(
|
||||||
name='multischleuder',
|
name='multischleuder',
|
||||||
version=__version__,
|
version=__version__,
|
||||||
author='s3lph',
|
author='s3lph',
|
||||||
author_email='s3lph@kabelsalat.ch',
|
author_email='1375407-s3lph@users.noreply.gitlab.com',
|
||||||
description='Merge subscribers and keys of multiple Schleuder lists into one',
|
description='Merge subscribers and keys of multiple Schleuder lists into one',
|
||||||
license='MIT',
|
license='MIT',
|
||||||
keywords='schleuder,pgp',
|
keywords='schleuder,pgp',
|
||||||
url='https://git.kabelsalat.ch/s3lph/multischleuder',
|
url='https://gitlab.com/s3lph/multischleuder',
|
||||||
packages=find_packages(exclude=['*.test']),
|
packages=find_packages(exclude=['*.test']),
|
||||||
install_requires=[
|
install_requires=[
|
||||||
'python-dateutil',
|
'python-dateutil',
|
||||||
'PyYAML',
|
'PyYAML',
|
||||||
'PGPy',
|
'PGPy',
|
||||||
],
|
],
|
||||||
extras_require={
|
|
||||||
'test': [
|
|
||||||
'aiosmtpd',
|
|
||||||
'coverage',
|
|
||||||
'pycodestyle',
|
|
||||||
'mypy',
|
|
||||||
'deepdiff',
|
|
||||||
'twine'
|
|
||||||
]
|
|
||||||
},
|
|
||||||
entry_points={
|
entry_points={
|
||||||
'console_scripts': [
|
'console_scripts': [
|
||||||
'multischleuder = multischleuder.main:main'
|
'multischleuder = multischleuder.main:main'
|
||||||
|
|
Loading…
Reference in a new issue