From 9104368792339242e6782e5b62bfa0891219f181 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 27 Sep 2021 23:40:10 +0200 Subject: [PATCH] Add CI with packaging/release process --- .gitlab-ci.yml | 102 ++++++++++ CHANGELOG.md | 14 ++ README.md | 4 +- easywks/main.py | 4 +- easywks/process.py | 82 ++++---- easywks/types.py | 49 ++++- easywks/util.py | 6 - package/debian/easywks/DEBIAN/conffiles | 1 + package/debian/easywks/DEBIAN/control | 11 ++ package/debian/easywks/DEBIAN/postinst | 20 ++ package/debian/easywks/DEBIAN/postrm | 9 + package/debian/easywks/DEBIAN/prerm | 9 + package/debian/easywks/etc/easywks.yml | 40 ++++ .../lib/systemd/system/easywks-http.service | 13 ++ .../lib/systemd/system/easywks-lmtp.service | 13 ++ .../easywks/usr/lib/matemat/matemat.conf | 11 ++ .../easywks/usr/share/doc/matemat/copyright | 16 ++ package/docker/Dockerfile | 17 ++ package/docker/easywks.yml | 12 ++ package/docker/entrypoint.sh | 3 + package/release.py | 179 ++++++++++++++++++ setup.cfg | 22 +++ 22 files changed, 584 insertions(+), 53 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 CHANGELOG.md create mode 100644 package/debian/easywks/DEBIAN/conffiles create mode 100644 package/debian/easywks/DEBIAN/control create mode 100755 package/debian/easywks/DEBIAN/postinst create mode 100755 package/debian/easywks/DEBIAN/postrm create mode 100755 package/debian/easywks/DEBIAN/prerm create mode 100644 package/debian/easywks/etc/easywks.yml create mode 100644 package/debian/easywks/lib/systemd/system/easywks-http.service create mode 100644 package/debian/easywks/lib/systemd/system/easywks-lmtp.service create mode 100644 package/debian/easywks/usr/lib/matemat/matemat.conf create mode 100644 package/debian/easywks/usr/share/doc/matemat/copyright create mode 100644 package/docker/Dockerfile create mode 100644 package/docker/easywks.yml create mode 100755 package/docker/entrypoint.sh create mode 100755 package/release.py create mode 100644 setup.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..f72c4d2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,102 @@ +--- +image: python:3.9-bullseye + +stages: +- test +- build +- deploy + + + +before_script: +- pip3 install coverage pycodestyle +- export EASYWKS_VERSION=$(python -c 'import easywks; print(easywks.__version__)') + + + +test: + stage: test + script: + - pip3 install -e . + - python3 -m coverage run --rcfile=setup.cfg -m unittest discover easywks + - python3 -m coverage combine + - python3 -m coverage report --rcfile=setup.cfg + +codestyle: + stage: test + script: + - pip3 install -e . + - pycodestyle easywks + + + +build_docker: + stage: build + script: + - docker build -t "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" -f package/docker/Dockerfile . + - docker tag "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_REF_NAME" + - if [[ -n "$CI_COMMIT_TAG" ]]; then docker tag "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_TAG"; fi + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD registry.gitlab.com + - docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_SHA" + - docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_REF_NAME" + - if [[ -n "$CI_COMMIT_TAG" ]]; then docker push "registry.gitlab.com/s3lph/easywks:$CI_COMMIT_TAG"; fi + only: + - staging + - tags + +build_wheel: + stage: build + script: + - python3 setup.py egg_info bdist_wheel + - cd dist + - sha256sum *.whl > SHA256SUMS + artifacts: + paths: + - "dist/*.whl" + - dist/SHA256SUMS + only: + - tags + +build_debian: + stage: build + script: + - apt update && apt install lintian + - echo -n > package/debian/easywks/usr/share/doc/easywks/changelog + - | + for version in "$(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/easywks/usr/share/doc/easywks/changelog + echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/easywks/usr/share/doc/easywks/changelog + done + - gzip -9n package/debian/easywks/usr/share/doc/easywks/changelog + - python3 setup.py egg_info install --root=package/debian/easywks/ --prefix=/usr --optimize=1 + - cd package/debian + - sed -re "s/__EASYWKS_VERSION__/${EASYWKS_VERSION}/g" -i easywks/DEBIAN/control + - mkdir -p easywks/usr/lib/python3/dist-packages/ + - rsync -a easywks/usr/lib/python3.9/site-packages/ easywks/usr/lib/python3/dist-packages/ + - rm -rf easywks/usr/lib/python3.9/site-packages + - find easywks/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find easywks/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - find easywks/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; + - sed -re 's$#!/usr/local/bin/python3.9$#!/usr/bin/python3$' -i easywks/usr/bin/easywks + - find easywks -type f -exec chmod 0644 {} \; + - find easywks -type d -exec chmod 755 {} \; + - chmod +x easywks/usr/bin/easywks easywks/DEBIAN/postinst easywks/DEBIAN/prerm easywks/DEBIAN/postrm + - dpkg-deb --build easywks + - mv easywks.deb "easywks_${EASTWKS_VERSION}-1_all.deb" + - sudo -u nobody lintian "easywks_${EASTWKS_VERSION}-1_all.deb" + - sha256sum *.deb > SHA256SUMS + artifacts: + paths: + - "package/debian/*.deb" + - package/debian/SHA256SUMS + only: + - tags + + +release: + stage: deploy + script: + - python3 package/release.py + only: + - tags diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..98963e6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Matemat Changelog + + +## Version 0.1 + +First release. + +### Changes + + +- First somewhat stable version. + + + diff --git a/README.md b/README.md index 7fc1cfb..71052dc 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ smtp: username: webkey password: SuperS3curePassword123 # Configure the LMTP server -lmtp: +lmtpds: host: "::1" port: 8024 # Every domain served by EasyWKS must be listed here @@ -108,7 +108,7 @@ domains: # Users send their requests to this address. It's up to # you to make sure that the mails sent their get handed # to EasyWKS. - submission_address: webkey@example.com + submission_address: webkey@example.org # If you want the PGP key for this domain to be # password-protected, or if you're supplying your own # password-protected key, set the passphrase here: diff --git a/easywks/main.py b/easywks/main.py index e71b670..8d9f0b9 100644 --- a/easywks/main.py +++ b/easywks/main.py @@ -1,7 +1,7 @@ from .config import Config from .files import init_working_directory, clean_stale_requests -from .process import process_mail +from .process import process_mail_from_stdin from .server import run_server from .lmtpd import run_lmtpd @@ -24,7 +24,7 @@ def parse_arguments(): process = sp.add_parser('process', help='Read an incoming mail from stdin and write the response to stdout. ' 'Hook this up to your MTA. Also see lmtpd.') - process.set_defaults(fn=process_mail) + process.set_defaults(fn=process_mail_from_stdin) server = sp.add_parser('webserver', help='Run a WKD web server. Put this behind a HTTPS-terminating reverse proxy.') server.set_defaults(fn=run_server) diff --git a/easywks/process.py b/easywks/process.py index 4379ee3..7b089b8 100644 --- a/easywks/process.py +++ b/easywks/process.py @@ -3,10 +3,10 @@ from typing import List, Dict from .crypto import pgp_decrypt from .mailing import get_mailing_method -from .util import xloop_header from .config import Config from .files import read_pending_key, write_public_key, remove_pending_key, write_pending_key -from .types import SubmissionRequest, ConfirmationResponse, PublishResponse, ConfirmationRequest, EasyWksError +from .types import SubmissionRequest, ConfirmationResponse, PublishResponse, ConfirmationRequest, EasyWksError,\ + XLOOP_HEADER from email.message import MIMEPart, Message from email.parser import BytesParser @@ -114,49 +114,53 @@ def _parse_confirmation_response(pgp: PGPMessage, submission: str, sender: str): return ConfirmationResponse(rdict['sender'], submission, rdict['nonce'], pgp) -def process_mail(mail: bytes = None): - if not mail: - mail = sys.stdin.read().encode() - msg: Message = BytesParser(policy=default).parsebytes(mail) - _, sender_mail = getaddresses([msg['from']])[0] +def process_mail(mail: bytes): try: + msg: Message = BytesParser(policy=default).parsebytes(mail) + _, sender_mail = getaddresses([msg['from']])[0] local, sender_domain = sender_mail.split('@', 1) except ValueError: raise EasyWksError('Sender mail is not a valid mail address') if sender_domain not in Config.domains: raise EasyWksError(f'Domain {sender_domain} not supported') - if msg.get('x-loop', '') == xloop_header(sender_domain) or 'auto-submitted' in msg: + if msg.get('x-loop', '') == XLOOP_HEADER or 'auto-submitted' in msg: # Mail has somehow looped back to us, discard return submission_address: str = Config[sender_domain].submission_address - rcpt = getaddresses(msg.get_all('to', []) + msg.get_all('cc', [])) - if len(rcpt) != 1: - raise EasyWksError('Message has more than one recipients') - _, rcpt_mail = rcpt[0] - if rcpt_mail != submission_address: - raise EasyWksError(f'Message not addressed to submission address {submission_address} ' - f'for domain {sender_domain}') - leafs = _get_mime_leafs(msg) - pgp: PGPMessage = _get_pgp_message(leafs) - decrypted = pgp_decrypt(sender_domain, pgp) - if decrypted.is_signed: - request: ConfirmationResponse = _parse_confirmation_response(decrypted, submission_address, sender_mail) - try: - key = read_pending_key(sender_domain, request.nonce) - except FileNotFoundError: - # silently ignore non-existing requests - return - # this throws an error if signature verification fails - response: PublishResponse = request.verify_signature(key) - rmsg = response.create_signed_message() - write_public_key(sender_domain, sender_mail, key) - remove_pending_key(sender_domain, request.nonce) - else: - request: SubmissionRequest = _parse_submission_request(decrypted, submission_address, sender_mail) - response: ConfirmationRequest = request.confirmation_request() - rmsg = response.create_signed_message() - write_pending_key(sender_domain, response.nonce, request.key) - # Finally send out the response - if rmsg: - method = get_mailing_method(Config.mailing_method) - method(rmsg) + try: + rcpt = getaddresses(msg.get_all('to', []) + msg.get_all('cc', [])) + if len(rcpt) != 1: + raise EasyWksError('Message has more than one recipients') + _, rcpt_mail = rcpt[0] + if rcpt_mail != submission_address: + raise EasyWksError(f'Message not addressed to submission address {submission_address} ' + f'for domain {sender_domain}') + leafs = _get_mime_leafs(msg) + pgp: PGPMessage = _get_pgp_message(leafs) + decrypted = pgp_decrypt(sender_domain, pgp) + if decrypted.is_signed: + request: ConfirmationResponse = _parse_confirmation_response(decrypted, submission_address, sender_mail) + try: + key = read_pending_key(sender_domain, request.nonce) + except FileNotFoundError: + raise EasyWksError('There is no submission request for this email address, or it has expired. ' + 'Please resubmit your submission request.') + # this throws an error if signature verification fails + response: PublishResponse = request.verify_signature(key) + rmsg = response.create_signed_message() + write_public_key(sender_domain, sender_mail, key) + remove_pending_key(sender_domain, request.nonce) + else: + request: SubmissionRequest = _parse_submission_request(decrypted, submission_address, sender_mail) + response: ConfirmationRequest = request.confirmation_request() + rmsg = response.create_signed_message() + write_pending_key(sender_domain, response.nonce, request.key) + except EasyWksError as e: + rmsg = e.create_message(sender_mail, submission_address) + method = get_mailing_method(Config.mailing_method) + method(rmsg) + + +def process_mail_from_stdin(): + mail = sys.stdin.read().encode() + process_mail(mail) diff --git a/easywks/types.py b/easywks/types.py index 5bee98d..fb9c46e 100644 --- a/easywks/types.py +++ b/easywks/types.py @@ -10,7 +10,10 @@ from pgpy import PGPKey, PGPMessage, PGPUID from pgpy.types import SignatureVerification from .crypto import pgp_sign -from .util import create_nonce, fingerprint, xloop_header +from .util import create_nonce, fingerprint + + +XLOOP_HEADER = 'EasyWKS' class SubmissionRequest: @@ -124,7 +127,7 @@ Encrypt live everybody is. email['Date'] = format_datetime(datetime.utcnow()) email['Wks-Draft-Version'] = '3' email['Wks-Phase'] = 'confirm' - email['X-Loop'] = xloop_header(self.domain) + email['X-Loop'] = XLOOP_HEADER email['Auto-Submitted'] = 'auto-replied' return email @@ -226,16 +229,54 @@ Encrypt live everybody is. email['Date'] = format_datetime(datetime.utcnow()) email['Wks-Draft-Version'] = '3' email['Wks-Phase'] = 'done' - email['X-Loop'] = xloop_header(self.domain) + email['X-Loop'] = XLOOP_HEADER email['Auto-Submitted'] = 'auto-replied' return email class EasyWksError(BaseException): - def __init__(self, msg): + MAIL_TEXT = '''Hi there! + +This is the EasyWKS system at {domain}. + +An error has occurred while processing your request. + +{message} + +If this error persists, please contact your administrator for help. + +For more information on WKD and WKS see: + + https://gnupg.org/faq/wkd.html + https://gnupg.org/faq/wks.html + + +Regards +EasyWKS + +-- +Dance like nobody is watching. +Encrypt live everybody is. +''' + + def __init__(self, msg: str, ): super().__init__() self._msg = msg def __str__(self) -> str: return self._msg + + def create_message(self, submitter_addr: str, submission_addr: str) -> MIMEText: + domain = submission_addr.split('@', 1)[1] + payload = EasyWksError.MAIL_TEXT.format(domain=domain, message=self._msg) + email = MIMEText(payload) + email['Subject'] = 'An error has occurred while processing your request' + email['From'] = submission_addr + email['To'] = submitter_addr + email['Date'] = format_datetime(datetime.utcnow()) + email['Wks-Draft-Version'] = '3' + email['Wks-Phase'] = 'error' + email['X-Loop'] = XLOOP_HEADER + email['Auto-Submitted'] = 'auto-replied' + return email diff --git a/easywks/util.py b/easywks/util.py index d3b17c1..039341b 100644 --- a/easywks/util.py +++ b/easywks/util.py @@ -36,9 +36,3 @@ def create_nonce(n: int = 32) -> str: def fingerprint(key: PGPKey) -> str: return key.fingerprint.upper().replace(' ', '') - - -def xloop_header(domain: str) -> str: - components = list(reversed(domain.split('.'))) - components.append('easywks') - return '.'.join(components) \ No newline at end of file diff --git a/package/debian/easywks/DEBIAN/conffiles b/package/debian/easywks/DEBIAN/conffiles new file mode 100644 index 0000000..8bba2b1 --- /dev/null +++ b/package/debian/easywks/DEBIAN/conffiles @@ -0,0 +1 @@ +/etc/easywks.yml diff --git a/package/debian/easywks/DEBIAN/control b/package/debian/easywks/DEBIAN/control new file mode 100644 index 0000000..87da654 --- /dev/null +++ b/package/debian/easywks/DEBIAN/control @@ -0,0 +1,11 @@ +Package: easywks +Version: __EASYWKS_VERSION__ +Maintainer: s3lph +Section: web +Priority: optional +Architecture: all +Depends: python3 (>= 3.6), python3-pgpy, python3-bottle, python3-yaml, python3-aiosmtpd +Description: OpenPGP WKS for Human Beings + EasyWKS is a drop-in replacement for gpg-wks-server that aims to be + much easyier to use manually, while maintaing compatibility with the + WKS standard. \ No newline at end of file diff --git a/package/debian/easywks/DEBIAN/postinst b/package/debian/easywks/DEBIAN/postinst new file mode 100755 index 0000000..493249e --- /dev/null +++ b/package/debian/easywks/DEBIAN/postinst @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "configure" ]]; then + + if ! getent group easywks >/dev/null; then + groupadd --system easywks + fi + + if ! getent passwd easywks >/dev/null; then + useradd --system --create-home --gid easywks --home-dir /var/lib/easywks --shell /usr/sbin/nologin easywks + fi + + chown easywks:easywks /var/lib/easywks + chmod 0750 /var/lib/easywks + + systemctl daemon-reload || true + +fi diff --git a/package/debian/easywks/DEBIAN/postrm b/package/debian/easywks/DEBIAN/postrm new file mode 100755 index 0000000..305068d --- /dev/null +++ b/package/debian/easywks/DEBIAN/postrm @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "remove" ]]; then + + systemctl daemon-reload || true + +fi diff --git a/package/debian/easywks/DEBIAN/prerm b/package/debian/easywks/DEBIAN/prerm new file mode 100755 index 0000000..5943003 --- /dev/null +++ b/package/debian/easywks/DEBIAN/prerm @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "remove" ]]; then + + userdel easywks + +fi diff --git a/package/debian/easywks/etc/easywks.yml b/package/debian/easywks/etc/easywks.yml new file mode 100644 index 0000000..6895a64 --- /dev/null +++ b/package/debian/easywks/etc/easywks.yml @@ -0,0 +1,40 @@ +--- +# EasyWKS works inside this directory. Its PGP keys as well +# as all the submitted and published keys are stored here. +#directory: /var/lib/easywks +# Number of seconds after which a pending submission request +# is considered stale and should be removed by easywks clean. +#pending_lifetime: 604800 +# Port configuration for the webserver. Put this behind a +# HTTPS-terminating reverse proxy! +host: "::1" +port: 8080 +# Defaults to stdout, supported: stdout, smtp +mailing_method: smtp +# Configure smtp client options +smtp: + # Connect to this SMTP server to send a mail. + host: localhost + port: 25 + # if tls=True, starttls is ignored + #tls: false + #starttls: false + # Omit username/password if authentication is not needed. + #username: webkey + #password: SuperS3curePassword123 +# Configure the LMTP server +lmtpd: + host: "::1" + port: 8024 +# Every domain served by EasyWKS must be listed here +domains: + # Defaults are gpgwks@ and no password protection. + example.org: + # Users send their requests to this address. It's up to + # you to make sure that the mails sent their get handed + # to EasyWKS. + submission_address: webkey@example.org + # If you want the PGP key for this domain to be + # password-protected, or if you're supplying your own + # password-protected key, set the passphrase here: + #passphrase: "Correct Horse Battery Staple" \ No newline at end of file diff --git a/package/debian/easywks/lib/systemd/system/easywks-http.service b/package/debian/easywks/lib/systemd/system/easywks-http.service new file mode 100644 index 0000000..602dde9 --- /dev/null +++ b/package/debian/easywks/lib/systemd/system/easywks-http.service @@ -0,0 +1,13 @@ +[Unit] +Description=OpenPGP WKS for Human Beings - HTTP Server + +[Service] +Type=simple +ExecStart=/usr/bin/easywks webserver +Restart=on-failure +User=easywks +Group=easywks +WorkingDirectory=/var/lib/easywks + +[Install] +WantedBy=multi-user.target diff --git a/package/debian/easywks/lib/systemd/system/easywks-lmtp.service b/package/debian/easywks/lib/systemd/system/easywks-lmtp.service new file mode 100644 index 0000000..ef803f7 --- /dev/null +++ b/package/debian/easywks/lib/systemd/system/easywks-lmtp.service @@ -0,0 +1,13 @@ +[Unit] +Description=OpenPGP WKS for Human Beings - LMTP Server + +[Service] +Type=simple +ExecStart=/usr/bin/easywks lmtpd +Restart=on-failure +User=easywks +Group=easywks +WorkingDirectory=/var/lib/easywks + +[Install] +WantedBy=multi-user.target diff --git a/package/debian/easywks/usr/lib/matemat/matemat.conf b/package/debian/easywks/usr/lib/matemat/matemat.conf new file mode 100644 index 0000000..5c34f56 --- /dev/null +++ b/package/debian/easywks/usr/lib/matemat/matemat.conf @@ -0,0 +1,11 @@ +[Matemat] + +StaticPath=/usr/lib/matemat/static +TemplatePath=/usr/lib/matemat/templates + +LogTarget=stdout + +[Pagelets] + +UploadDir=/var/lib/matemat/upload +DatabaseFile=/var/lib/matemat/matemat.db diff --git a/package/debian/easywks/usr/share/doc/matemat/copyright b/package/debian/easywks/usr/share/doc/matemat/copyright new file mode 100644 index 0000000..a38a9a5 --- /dev/null +++ b/package/debian/easywks/usr/share/doc/matemat/copyright @@ -0,0 +1,16 @@ +Copyright 2018 s3lph + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/package/docker/Dockerfile b/package/docker/Dockerfile new file mode 100644 index 0000000..9eb3e88 --- /dev/null +++ b/package/docker/Dockerfile @@ -0,0 +1,17 @@ + +FROM python:3.9-alpine + +ADD . / +RUN mkdir -p /var/lib/easywks \ + && chown 1000:0 -R /var/lib/easywks \ + && chmod 0700 /var/lib/easywks \ + && pip3 install -e . \ + && mv /package/docker/entrypoint.sh /entrypoint.sh \ + && mv /package/docker/easywks.yml /etc/easywks.yml \ + && rm -rf /package + +USER 1000 + +EXPOSE 80/tcp +EXPOSE 24/tcp +CMD [ "/entrypoint.sh" ] diff --git a/package/docker/easywks.yml b/package/docker/easywks.yml new file mode 100644 index 0000000..b15e62c --- /dev/null +++ b/package/docker/easywks.yml @@ -0,0 +1,12 @@ +--- +directory: /var/lib/easywks +mailing_method: smtp +smtp: + host: localhost + port: 25 +lmtpd: + host: "::" + port: 24 +host: "::" +port: 80 +domains: {} diff --git a/package/docker/entrypoint.sh b/package/docker/entrypoint.sh new file mode 100755 index 0000000..406fce7 --- /dev/null +++ b/package/docker/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/usr/local/bin/python3 -m easywks diff --git a/package/release.py b/package/release.py new file mode 100755 index 0000000..42d3284 --- /dev/null +++ b/package/release.py @@ -0,0 +1,179 @@ + +from typing import Any, Dict, List, Optional, Tuple + +import os +import sys +import json +import urllib.request +import http.client +from urllib.error import HTTPError + + +def parse_changelog(tag: str) -> Optional[str]: + release_changelog: str = '' + with open('CHANGELOG.md', 'r') as f: + in_target: bool = False + done: bool = False + for line in f.readlines(): + if in_target: + if f'' in line: + done = True + break + release_changelog += line + elif f'' in line: + in_target = True + continue + if not done: + return None + return release_changelog + + +def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]: + url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' + headers: Dict[str, str] = { + 'Private-Token': api_token, + 'User-Agent': 'curl/7.70.0' + } + req = urllib.request.Request(url, headers=headers) + try: + resp: http.client.HTTPResponse = urllib.request.urlopen(req) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + resp_data: bytes = resp.read() + joblist: List[Dict[str, Any]] = json.loads(resp_data.decode()) + + jobidmap: Dict[str, str] = {} + for job in joblist: + name: str = job['name'] + job_id: str = job['id'] + jobidmap[name] = job_id + return jobidmap + + +def fetch_single_shafile(url: str) -> str: + headers: Dict[str, str] = { + 'User-Agent': 'curl/7.70.0' + } + req = urllib.request.Request(url, headers=headers) + try: + resp: http.client.HTTPResponse = urllib.request.urlopen(req) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + resp_data: bytes = resp.readline() + shafile: str = resp_data.decode() + filename: str = shafile.strip().split(' ')[-1].strip() + return filename + + +def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: + mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' + wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' + wheel_filename: str = fetch_single_shafile(wheel_sha_url) + wheel_url: str = f'{mybase}/dist/{wheel_filename}' + return wheel_url, wheel_sha_url + + +def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: + mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' + debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' + debian_filename: str = fetch_single_shafile(debian_sha_url) + debian_url: str = f'{mybase}/package/debian/{debian_filename}' + return debian_url, debian_sha_url + + +def main(): + api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN') + release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG') + project_name: Optional[str] = os.getenv('CI_PROJECT_PATH') + project_id: Optional[str] = os.getenv('CI_PROJECT_ID') + pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID') + if api_token is None: + print('GITLAB_API_TOKEN is not set.', file=sys.stderr) + sys.exit(1) + if release_tag is None: + print('CI_COMMIT_TAG is not set.', file=sys.stderr) + sys.exit(1) + if project_name is None: + print('CI_PROJECT_PATH is not set.', file=sys.stderr) + sys.exit(1) + if project_id is None: + print('CI_PROJECT_ID is not set.', file=sys.stderr) + sys.exit(1) + if pipeline_id is None: + print('CI_PIPELINE_ID is not set.', file=sys.stderr) + sys.exit(1) + + changelog: Optional[str] = parse_changelog(release_tag) + if changelog is None: + print('Changelog could not be parsed.', file=sys.stderr) + sys.exit(1) + + job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token) + + base_url: str = f'https://gitlab.com/{project_name}/-' + + wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) + debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) + + augmented_changelog = f'''{changelog.strip()} + +### Download + +- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) +- [Debian Package]({debian_url}) ([sha256]({debian_sha_url})) +- Docker image: registry.gitlab.com/{project_name}:{release_tag}''' + + post_body: str = json.dumps({ + 'tag_name': release_tag, + 'description': augmented_changelog, + 'assets': { + 'links': [ + { + 'name': 'Python Wheel', + 'url': wheel_url, + 'link_type': 'package' + }, + { + 'name': 'Debian Package', + 'url': debian_url, + 'link_type': 'package' + } + ] + } + }) + + gitlab_release_api_url: str = \ + f'https://gitlab.com/api/v4/projects/{project_id}/releases' + headers: Dict[str, str] = { + 'Private-Token': api_token, + 'Content-Type': 'application/json; charset=utf-8', + 'User-Agent': 'curl/7.70.0' + } + + request = urllib.request.Request( + gitlab_release_api_url, + post_body.encode('utf-8'), + headers=headers, + method='POST' + ) + try: + response: http.client.HTTPResponse = urllib.request.urlopen(request) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + response_bytes: bytes = response.read() + response_str: str = response_bytes.decode() + response_data: Dict[str, Any] = json.loads(response_str) + + if response_data['tag_name'] != release_tag: + print('Something went wrong...', file=sys.stderr) + print(response_str, file=sys.stderr) + sys.exit(1) + + print(response_data['description']) + + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b6d543d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ + +# +# PyCodestyle +# + +[pycodestyle] +max-line-length = 120 +statistics = True + +# +# Coverage +# + +[run] +branch = True +parallel = True +source = easywks/ + +[report] +show_missing = True +include = easywks/* +omit = */test/*.py