Add CI with packaging/release process

This commit is contained in:
s3lph 2021-09-27 23:40:10 +02:00
parent 0f8ce0a476
commit 9104368792
22 changed files with 584 additions and 53 deletions

102
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,102 @@
---
image: python:3.9-bullseye
stages:
- test
- build
- deploy
before_script:
- pip3 install coverage pycodestyle
- export EASYWKS_VERSION=$(python -c 'import easywks; print(easywks.__version__)')
test:
stage: test
script:
- pip3 install -e .
- python3 -m coverage run --rcfile=setup.cfg -m unittest discover easywks
- python3 -m coverage combine
- python3 -m coverage report --rcfile=setup.cfg
codestyle:
stage: test
script:
- pip3 install -e .
- pycodestyle easywks
build_docker:
stage: build
script:
- docker build -t "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" -f package/docker/Dockerfile .
- docker tag "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_REF_NAME"
- if [[ -n "$CI_COMMIT_TAG" ]]; then docker tag "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_TAG"; fi
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD registry.gitlab.com
- docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA"
- docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_REF_NAME"
- if [[ -n "$CI_COMMIT_TAG" ]]; then docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_TAG"; fi
only:
- staging
- tags
build_wheel:
stage: build
script:
- python3 setup.py egg_info bdist_wheel
- cd dist
- sha256sum *.whl > SHA256SUMS
artifacts:
paths:
- "dist/*.whl"
- dist/SHA256SUMS
only:
- tags
build_debian:
stage: build
script:
- apt update && apt install lintian
- echo -n > package/debian/easywks/usr/share/doc/easywks/changelog
- |
for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
echo "easywks (${version}-1); urgency=medium\n" >> package/debian/easywks/usr/share/doc/easywks/changelog
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/easywks/usr/share/doc/easywks/changelog
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/easywks/usr/share/doc/easywks/changelog
done
- gzip -9n package/debian/easywks/usr/share/doc/easywks/changelog
- python3 setup.py egg_info install --root=package/debian/easywks/ --prefix=/usr --optimize=1
- cd package/debian
- sed -re "s/__EASYWKS_VERSION__/${EASYWKS_VERSION}/g" -i easywks/DEBIAN/control
- mkdir -p easywks/usr/lib/python3/dist-packages/
- rsync -a easywks/usr/lib/python3.9/site-packages/ easywks/usr/lib/python3/dist-packages/
- rm -rf easywks/usr/lib/python3.9/site-packages
- find easywks/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
- find easywks/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
- find easywks/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \;
- sed -re 's$#!/usr/local/bin/python3.9$#!/usr/bin/python3$' -i easywks/usr/bin/easywks
- find easywks -type f -exec chmod 0644 {} \;
- find easywks -type d -exec chmod 755 {} \;
- chmod +x easywks/usr/bin/easywks easywks/DEBIAN/postinst easywks/DEBIAN/prerm easywks/DEBIAN/postrm
- dpkg-deb --build easywks
- mv easywks.deb "easywks_${EASTWKS_VERSION}-1_all.deb"
- sudo -u nobody lintian "easywks_${EASTWKS_VERSION}-1_all.deb"
- sha256sum *.deb > SHA256SUMS
artifacts:
paths:
- "package/debian/*.deb"
- package/debian/SHA256SUMS
only:
- tags
release:
stage: deploy
script:
- python3 package/release.py
only:
- tags

14
CHANGELOG.md Normal file
View file

@ -0,0 +1,14 @@
# Matemat Changelog
<!-- BEGIN RELEASE v0.1 -->
## Version 0.1
First release.
### Changes
<!-- BEGIN CHANGES 0.1 -->
- First somewhat stable version.
<!-- END CHANGES 0.1 -->
<!-- END RELEASE v0.1 -->

View file

@ -99,7 +99,7 @@ smtp:
username: webkey
password: SuperS3curePassword123
# Configure the LMTP server
lmtp:
lmtpds:
host: "::1"
port: 8024
# Every domain served by EasyWKS must be listed here
@ -108,7 +108,7 @@ domains:
# Users send their requests to this address. It's up to
# you to make sure that the mails sent their get handed
# to EasyWKS.
submission_address: webkey@example.com
submission_address: webkey@example.org
# If you want the PGP key for this domain to be
# password-protected, or if you're supplying your own
# password-protected key, set the passphrase here:

View file

@ -1,7 +1,7 @@
from .config import Config
from .files import init_working_directory, clean_stale_requests
from .process import process_mail
from .process import process_mail_from_stdin
from .server import run_server
from .lmtpd import run_lmtpd
@ -24,7 +24,7 @@ def parse_arguments():
process = sp.add_parser('process', help='Read an incoming mail from stdin and write the response to stdout. '
'Hook this up to your MTA. Also see lmtpd.')
process.set_defaults(fn=process_mail)
process.set_defaults(fn=process_mail_from_stdin)
server = sp.add_parser('webserver', help='Run a WKD web server. Put this behind a HTTPS-terminating reverse proxy.')
server.set_defaults(fn=run_server)

View file

@ -3,10 +3,10 @@ from typing import List, Dict
from .crypto import pgp_decrypt
from .mailing import get_mailing_method
from .util import xloop_header
from .config import Config
from .files import read_pending_key, write_public_key, remove_pending_key, write_pending_key
from .types import SubmissionRequest, ConfirmationResponse, PublishResponse, ConfirmationRequest, EasyWksError
from .types import SubmissionRequest, ConfirmationResponse, PublishResponse, ConfirmationRequest, EasyWksError,\
XLOOP_HEADER
from email.message import MIMEPart, Message
from email.parser import BytesParser
@ -114,21 +114,20 @@ def _parse_confirmation_response(pgp: PGPMessage, submission: str, sender: str):
return ConfirmationResponse(rdict['sender'], submission, rdict['nonce'], pgp)
def process_mail(mail: bytes = None):
if not mail:
mail = sys.stdin.read().encode()
def process_mail(mail: bytes):
try:
msg: Message = BytesParser(policy=default).parsebytes(mail)
_, sender_mail = getaddresses([msg['from']])[0]
try:
local, sender_domain = sender_mail.split('@', 1)
except ValueError:
raise EasyWksError('Sender mail is not a valid mail address')
if sender_domain not in Config.domains:
raise EasyWksError(f'Domain {sender_domain} not supported')
if msg.get('x-loop', '') == xloop_header(sender_domain) or 'auto-submitted' in msg:
if msg.get('x-loop', '') == XLOOP_HEADER or 'auto-submitted' in msg:
# Mail has somehow looped back to us, discard
return
submission_address: str = Config[sender_domain].submission_address
try:
rcpt = getaddresses(msg.get_all('to', []) + msg.get_all('cc', []))
if len(rcpt) != 1:
raise EasyWksError('Message has more than one recipients')
@ -144,8 +143,8 @@ def process_mail(mail: bytes = None):
try:
key = read_pending_key(sender_domain, request.nonce)
except FileNotFoundError:
# silently ignore non-existing requests
return
raise EasyWksError('There is no submission request for this email address, or it has expired. '
'Please resubmit your submission request.')
# this throws an error if signature verification fails
response: PublishResponse = request.verify_signature(key)
rmsg = response.create_signed_message()
@ -156,7 +155,12 @@ def process_mail(mail: bytes = None):
response: ConfirmationRequest = request.confirmation_request()
rmsg = response.create_signed_message()
write_pending_key(sender_domain, response.nonce, request.key)
# Finally send out the response
if rmsg:
except EasyWksError as e:
rmsg = e.create_message(sender_mail, submission_address)
method = get_mailing_method(Config.mailing_method)
method(rmsg)
def process_mail_from_stdin():
mail = sys.stdin.read().encode()
process_mail(mail)

View file

@ -10,7 +10,10 @@ from pgpy import PGPKey, PGPMessage, PGPUID
from pgpy.types import SignatureVerification
from .crypto import pgp_sign
from .util import create_nonce, fingerprint, xloop_header
from .util import create_nonce, fingerprint
XLOOP_HEADER = 'EasyWKS'
class SubmissionRequest:
@ -124,7 +127,7 @@ Encrypt live everybody is.
email['Date'] = format_datetime(datetime.utcnow())
email['Wks-Draft-Version'] = '3'
email['Wks-Phase'] = 'confirm'
email['X-Loop'] = xloop_header(self.domain)
email['X-Loop'] = XLOOP_HEADER
email['Auto-Submitted'] = 'auto-replied'
return email
@ -226,16 +229,54 @@ Encrypt live everybody is.
email['Date'] = format_datetime(datetime.utcnow())
email['Wks-Draft-Version'] = '3'
email['Wks-Phase'] = 'done'
email['X-Loop'] = xloop_header(self.domain)
email['X-Loop'] = XLOOP_HEADER
email['Auto-Submitted'] = 'auto-replied'
return email
class EasyWksError(BaseException):
def __init__(self, msg):
MAIL_TEXT = '''Hi there!
This is the EasyWKS system at {domain}.
An error has occurred while processing your request.
{message}
If this error persists, please contact your administrator for help.
For more information on WKD and WKS see:
https://gnupg.org/faq/wkd.html
https://gnupg.org/faq/wks.html
Regards
EasyWKS
--
Dance like nobody is watching.
Encrypt live everybody is.
'''
def __init__(self, msg: str, ):
super().__init__()
self._msg = msg
def __str__(self) -> str:
return self._msg
def create_message(self, submitter_addr: str, submission_addr: str) -> MIMEText:
domain = submission_addr.split('@', 1)[1]
payload = EasyWksError.MAIL_TEXT.format(domain=domain, message=self._msg)
email = MIMEText(payload)
email['Subject'] = 'An error has occurred while processing your request'
email['From'] = submission_addr
email['To'] = submitter_addr
email['Date'] = format_datetime(datetime.utcnow())
email['Wks-Draft-Version'] = '3'
email['Wks-Phase'] = 'error'
email['X-Loop'] = XLOOP_HEADER
email['Auto-Submitted'] = 'auto-replied'
return email

View file

@ -36,9 +36,3 @@ def create_nonce(n: int = 32) -> str:
def fingerprint(key: PGPKey) -> str:
return key.fingerprint.upper().replace(' ', '')
def xloop_header(domain: str) -> str:
components = list(reversed(domain.split('.')))
components.append('easywks')
return '.'.join(components)

View file

@ -0,0 +1 @@
/etc/easywks.yml

View file

@ -0,0 +1,11 @@
Package: easywks
Version: __EASYWKS_VERSION__
Maintainer: s3lph <account-gitlab-ideynizv@kernelpanic.lol>
Section: web
Priority: optional
Architecture: all
Depends: python3 (>= 3.6), python3-pgpy, python3-bottle, python3-yaml, python3-aiosmtpd
Description: OpenPGP WKS for Human Beings
EasyWKS is a drop-in replacement for gpg-wks-server that aims to be
much easyier to use manually, while maintaing compatibility with the
WKS standard.

View file

@ -0,0 +1,20 @@
#!/bin/bash
set -e
if [[ "$1" == "configure" ]]; then
if ! getent group easywks >/dev/null; then
groupadd --system easywks
fi
if ! getent passwd easywks >/dev/null; then
useradd --system --create-home --gid easywks --home-dir /var/lib/easywks --shell /usr/sbin/nologin easywks
fi
chown easywks:easywks /var/lib/easywks
chmod 0750 /var/lib/easywks
systemctl daemon-reload || true
fi

View file

@ -0,0 +1,9 @@
#!/bin/bash
set -e
if [[ "$1" == "remove" ]]; then
systemctl daemon-reload || true
fi

View file

@ -0,0 +1,9 @@
#!/bin/bash
set -e
if [[ "$1" == "remove" ]]; then
userdel easywks
fi

View file

@ -0,0 +1,40 @@
---
# EasyWKS works inside this directory. Its PGP keys as well
# as all the submitted and published keys are stored here.
#directory: /var/lib/easywks
# Number of seconds after which a pending submission request
# is considered stale and should be removed by easywks clean.
#pending_lifetime: 604800
# Port configuration for the webserver. Put this behind a
# HTTPS-terminating reverse proxy!
host: "::1"
port: 8080
# Defaults to stdout, supported: stdout, smtp
mailing_method: smtp
# Configure smtp client options
smtp:
# Connect to this SMTP server to send a mail.
host: localhost
port: 25
# if tls=True, starttls is ignored
#tls: false
#starttls: false
# Omit username/password if authentication is not needed.
#username: webkey
#password: SuperS3curePassword123
# Configure the LMTP server
lmtpd:
host: "::1"
port: 8024
# Every domain served by EasyWKS must be listed here
domains:
# Defaults are gpgwks@<domain> and no password protection.
example.org:
# Users send their requests to this address. It's up to
# you to make sure that the mails sent their get handed
# to EasyWKS.
submission_address: webkey@example.org
# If you want the PGP key for this domain to be
# password-protected, or if you're supplying your own
# password-protected key, set the passphrase here:
#passphrase: "Correct Horse Battery Staple"

View file

@ -0,0 +1,13 @@
[Unit]
Description=OpenPGP WKS for Human Beings - HTTP Server
[Service]
Type=simple
ExecStart=/usr/bin/easywks webserver
Restart=on-failure
User=easywks
Group=easywks
WorkingDirectory=/var/lib/easywks
[Install]
WantedBy=multi-user.target

View file

@ -0,0 +1,13 @@
[Unit]
Description=OpenPGP WKS for Human Beings - LMTP Server
[Service]
Type=simple
ExecStart=/usr/bin/easywks lmtpd
Restart=on-failure
User=easywks
Group=easywks
WorkingDirectory=/var/lib/easywks
[Install]
WantedBy=multi-user.target

View file

@ -0,0 +1,11 @@
[Matemat]
StaticPath=/usr/lib/matemat/static
TemplatePath=/usr/lib/matemat/templates
LogTarget=stdout
[Pagelets]
UploadDir=/var/lib/matemat/upload
DatabaseFile=/var/lib/matemat/matemat.db

View file

@ -0,0 +1,16 @@
Copyright 2018 s3lph
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge, publish, distribute,
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

17
package/docker/Dockerfile Normal file
View file

@ -0,0 +1,17 @@
FROM python:3.9-alpine
ADD . /
RUN mkdir -p /var/lib/easywks \
&& chown 1000:0 -R /var/lib/easywks \
&& chmod 0700 /var/lib/easywks \
&& pip3 install -e . \
&& mv /package/docker/entrypoint.sh /entrypoint.sh \
&& mv /package/docker/easywks.yml /etc/easywks.yml \
&& rm -rf /package
USER 1000
EXPOSE 80/tcp
EXPOSE 24/tcp
CMD [ "/entrypoint.sh" ]

View file

@ -0,0 +1,12 @@
---
directory: /var/lib/easywks
mailing_method: smtp
smtp:
host: localhost
port: 25
lmtpd:
host: "::"
port: 24
host: "::"
port: 80
domains: {}

3
package/docker/entrypoint.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/sh
/usr/local/bin/python3 -m easywks

179
package/release.py Executable file
View file

@ -0,0 +1,179 @@
from typing import Any, Dict, List, Optional, Tuple
import os
import sys
import json
import urllib.request
import http.client
from urllib.error import HTTPError
def parse_changelog(tag: str) -> Optional[str]:
release_changelog: str = ''
with open('CHANGELOG.md', 'r') as f:
in_target: bool = False
done: bool = False
for line in f.readlines():
if in_target:
if f'<!-- END RELEASE {tag} -->' in line:
done = True
break
release_changelog += line
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
in_target = True
continue
if not done:
return None
return release_changelog
def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]:
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
headers: Dict[str, str] = {
'Private-Token': api_token,
'User-Agent': 'curl/7.70.0'
}
req = urllib.request.Request(url, headers=headers)
try:
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
except HTTPError as e:
print(e.read().decode())
sys.exit(1)
resp_data: bytes = resp.read()
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
jobidmap: Dict[str, str] = {}
for job in joblist:
name: str = job['name']
job_id: str = job['id']
jobidmap[name] = job_id
return jobidmap
def fetch_single_shafile(url: str) -> str:
headers: Dict[str, str] = {
'User-Agent': 'curl/7.70.0'
}
req = urllib.request.Request(url, headers=headers)
try:
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
except HTTPError as e:
print(e.read().decode())
sys.exit(1)
resp_data: bytes = resp.readline()
shafile: str = resp_data.decode()
filename: str = shafile.strip().split(' ')[-1].strip()
return filename
def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS'
wheel_filename: str = fetch_single_shafile(wheel_sha_url)
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
return wheel_url, wheel_sha_url
def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS'
debian_filename: str = fetch_single_shafile(debian_sha_url)
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
return debian_url, debian_sha_url
def main():
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
if api_token is None:
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
sys.exit(1)
if release_tag is None:
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
sys.exit(1)
if project_name is None:
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
sys.exit(1)
if project_id is None:
print('CI_PROJECT_ID is not set.', file=sys.stderr)
sys.exit(1)
if pipeline_id is None:
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
sys.exit(1)
changelog: Optional[str] = parse_changelog(release_tag)
if changelog is None:
print('Changelog could not be parsed.', file=sys.stderr)
sys.exit(1)
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
base_url: str = f'https://gitlab.com/{project_name}/-'
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids)
debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids)
augmented_changelog = f'''{changelog.strip()}
### Download
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))
- Docker image: registry.gitlab.com/{project_name}:{release_tag}'''
post_body: str = json.dumps({
'tag_name': release_tag,
'description': augmented_changelog,
'assets': {
'links': [
{
'name': 'Python Wheel',
'url': wheel_url,
'link_type': 'package'
},
{
'name': 'Debian Package',
'url': debian_url,
'link_type': 'package'
}
]
}
})
gitlab_release_api_url: str = \
f'https://gitlab.com/api/v4/projects/{project_id}/releases'
headers: Dict[str, str] = {
'Private-Token': api_token,
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'curl/7.70.0'
}
request = urllib.request.Request(
gitlab_release_api_url,
post_body.encode('utf-8'),
headers=headers,
method='POST'
)
try:
response: http.client.HTTPResponse = urllib.request.urlopen(request)
except HTTPError as e:
print(e.read().decode())
sys.exit(1)
response_bytes: bytes = response.read()
response_str: str = response_bytes.decode()
response_data: Dict[str, Any] = json.loads(response_str)
if response_data['tag_name'] != release_tag:
print('Something went wrong...', file=sys.stderr)
print(response_str, file=sys.stderr)
sys.exit(1)
print(response_data['description'])
if __name__ == '__main__':
main()

22
setup.cfg Normal file
View file

@ -0,0 +1,22 @@
#
# PyCodestyle
#
[pycodestyle]
max-line-length = 120
statistics = True
#
# Coverage
#
[run]
branch = True
parallel = True
source = easywks/
[report]
show_missing = True
include = easywks/*
omit = */test/*.py