diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 6e394cb..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- -image: python:3.11-bookworm - -stages: -- test -- coverage -- build -- deploy -- upload - - - -before_script: -- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff -- export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)') - - - -test: - stage: test - script: - - pip3 install -e . - - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder - artifacts: - paths: - - ".coverage*" - -codestyle: - stage: test - script: - - pip3 install -e . - - pycodestyle multischleuder - -mypy: - stage: test - script: - - pip3 install -e . - - mypy --install-types --non-interactive multischleuder - - mypy multischleuder - -sast: - stage: test -bandit-sast: - before_script: [''] -include: - - template: Security/SAST.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - -schleuder: - stage: test - script: - - debconf-set-selections <<<"postfix postfix/mailname string example.org" - - debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'" - - apt update; apt install --yes schleuder schleuder-cli postfix patch - - patch -d/ -p1 < test/01-ruby3cgi.patch # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1040257 - - /usr/lib/postfix/configure-instance.sh - - - echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf - - /usr/sbin/postmulti -i - -p start - - schleuder-cli lists list || true - - export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4) - - echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml - - | - cat > ~/.schleuder-cli/schleuder-cli.yml <- - /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/ - script: - - python3 -m coverage combine - - python3 -m coverage report --rcfile=setup.cfg - - - -build_wheel: - stage: build - script: - - python3 setup.py egg_info bdist_wheel - - cd dist - - sha256sum *.whl > SHA256SUMS - artifacts: - paths: - - "dist/*.whl" - - dist/SHA256SUMS - only: - - tags - -build_debian: - stage: build - script: - - apt update && apt install --yes lintian rsync sudo - - echo -n > package/debian/multischleuder/usr/share/doc/multischleuder/changelog - - | - for version in "$(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog - echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/multischleuder/usr/share/doc/multischleuder/changelog - done - - gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog - - python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1 - - cd package/debian - - sed -re "s/__MULTISCHLEUDER_VERSION__/${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control - - mkdir -p multischleuder/usr/lib/python3 - - mv multischleuder/usr/lib/python3.11/site-packages multischleuder/usr/lib/python3/dist-packages - - rm -rf multischleuder/usr/lib/python3.11 - - find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true - - find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; - - find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; - - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder - - find multischleuder -type f -exec chmod 0644 {} \; - - find multischleuder -type d -exec chmod 755 {} \; - - chmod +x multischleuder/usr/bin/multischleuder multischleuder/DEBIAN/postinst multischleuder/DEBIAN/prerm multischleuder/DEBIAN/postrm - - dpkg-deb --build multischleuder - - mv multischleuder.deb "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb" - - sudo -u nobody lintian "multischleuder_${MULTISCHLEUDER_VERSION}-1_all.deb" - - sha256sum *.deb > SHA256SUMS - artifacts: - paths: - - "package/debian/*.deb" - - package/debian/SHA256SUMS - only: - - tags - - -release: - stage: deploy - script: - - python3 package/release.py - only: - - tags - - - -repo: - stage: upload - trigger: s3lph/custom-packages - variables: - MULTIPROJECT_TRIGGER_JOBNAME: multischleuder - only: - - tags diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..f4a4275 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,139 @@ +--- + +steps: + + test: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - python3 -m coverage run --rcfile=setup.cfg -m unittest discover multischleuder + + codestyle: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - pycodestyle multischleuder + + mypy: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - mypy --install-types --non-interactive multischleuder + - mypy multischleuder + + schleuder: + image: python:3.11-bookworm + group: test + commands: + - debconf-set-selections <<<"postfix postfix/mailname string example.org" + - debconf-set-selections <<<"postfix postfix/main_mailer_type string 'Local only'" + - apt update; apt install --yes schleuder schleuder-cli postfix patch + - patch -d/ -p1 < test/01-ruby3cgi.patch # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1040257 + - /usr/lib/postfix/configure-instance.sh - + - echo "virtual_alias_maps = static:root" >> /etc/postfix/main.cf + - /usr/sbin/postmulti -i - -p start + - schleuder-cli lists list || true + - export CERT_FPR=$(schleuder cert fingerprint | cut -d' ' -f4) + - echo " - '00000000000000000000000000000000'" >> /etc/schleuder/schleuder.yml + - | + cat > ~/.schleuder-cli/schleuder-cli.yml < ~/.pypirc <" | grep -B 1000 "<"'!'"-- END CHANGES $${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' + echo -e "\n -- s3lph $(date -R)\n" + done) > package/debian/multischleuder/usr/share/doc/multischleuder/changelog + - gzip -9n package/debian/multischleuder/usr/share/doc/multischleuder/changelog + - python3 setup.py egg_info install --root=package/debian/multischleuder/ --prefix=/usr --optimize=1 + - cd package/debian + - sed -re "s/__MULTISCHLEUDER_VERSION__/$${MULTISCHLEUDER_VERSION}/g" -i multischleuder/DEBIAN/control + - mkdir -p multischleuder/usr/lib/python3/dist-packages/ + - rsync -a multischleuder/usr/lib/python3.11/site-packages/ multischleuder/usr/lib/python3/dist-packages/ + - rm -rf multischleuder/usr/lib/python3.11/site-packages + - find multischleuder/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find multischleuder/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - find multischleuder/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; + - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i multischleuder/usr/bin/multischleuder + - find multischleuder -type f -exec chmod 0644 {} \; + - find multischleuder -type d -exec chmod 755 {} \; + - >- + chmod +x + multischleuder/usr/bin/multischleuder + multischleuder/DEBIAN/postinst + multischleuder/DEBIAN/prerm + multischleuder/DEBIAN/postrm + - dpkg-deb --build multischleuder + - mv multischleuder.deb "multischleuder_$${MULTISCHLEUDER_VERSION}-1_all.deb" + - sudo -u nobody lintian "multischleuder_$${MULTISCHLEUDER_VERSION}-1_all.deb" || true + - >- + curl + --user "$${GITEA_API_USERNAME}:$${GITEA_API_PASSWORD}" + --upload-file "multischleuder_$${MULTISCHLEUDER_VERSION}-1_all.deb" + $${GITEA_API_REPOSITORY_DEB} diff --git a/package/debian/multischleuder/DEBIAN/control b/package/debian/multischleuder/DEBIAN/control index d557a96..a224d13 100644 --- a/package/debian/multischleuder/DEBIAN/control +++ b/package/debian/multischleuder/DEBIAN/control @@ -1,6 +1,6 @@ Package: multischleuder Version: __MULTISCHLEUDER_VERSION__ -Maintainer: s3lph <1375407-s3lph@users.noreply.gitlab.com> +Maintainer: s3lph Section: web Priority: optional Architecture: all diff --git a/package/release.py b/package/release.py deleted file mode 100755 index 4f09ba3..0000000 --- a/package/release.py +++ /dev/null @@ -1,183 +0,0 @@ - -from typing import Any, Dict, List, Optional, Tuple - -import os -import sys -import json -import urllib.request -import http.client -from urllib.error import HTTPError - - -def parse_changelog(tag: str) -> Optional[str]: - release_changelog: str = '' - with open('CHANGELOG.md', 'r') as f: - in_target: bool = False - done: bool = False - for line in f.readlines(): - if in_target: - if f'' in line: - done = True - break - release_changelog += line - elif f'' in line: - in_target = True - continue - if not done: - return None - return release_changelog - - -def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]: - url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'User-Agent': 'curl/7.70.0' - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.read() - joblist: List[Dict[str, Any]] = json.loads(resp_data.decode()) - - jobidmap: Dict[str, str] = {} - for job in joblist: - name: str = job['name'] - job_id: str = job['id'] - jobidmap[name] = job_id - return jobidmap - - -def fetch_single_shafile(url: str, api_token: str) -> str: - headers: Dict[str, str] = { - 'User-Agent': 'curl/7.70.0', - 'Private-Token': api_token - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.readline() - shafile: str = resp_data.decode() - filename: str = shafile.strip().split(' ')[-1].strip() - return filename - - -def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' - wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\ - '/artifacts/dist/SHA256SUMS' - wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token) - wheel_url: str = f'{mybase}/dist/{wheel_filename}' - return wheel_url, wheel_sha_url - - -def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' - debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\ - '/artifacts/package/debian/SHA256SUMS' - debian_filename: str = fetch_single_shafile(debian_sha_url, api_token) - debian_url: str = f'{mybase}/package/debian/{debian_filename}' - return debian_url, debian_sha_url - - -def main(): - api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN') - release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG') - project_name: Optional[str] = os.getenv('CI_PROJECT_PATH') - project_id: Optional[str] = os.getenv('CI_PROJECT_ID') - pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID') - if api_token is None: - print('GITLAB_API_TOKEN is not set.', file=sys.stderr) - sys.exit(1) - if release_tag is None: - print('CI_COMMIT_TAG is not set.', file=sys.stderr) - sys.exit(1) - if project_name is None: - print('CI_PROJECT_PATH is not set.', file=sys.stderr) - sys.exit(1) - if project_id is None: - print('CI_PROJECT_ID is not set.', file=sys.stderr) - sys.exit(1) - if pipeline_id is None: - print('CI_PIPELINE_ID is not set.', file=sys.stderr) - sys.exit(1) - - changelog: Optional[str] = parse_changelog(release_tag) - if changelog is None: - print('Changelog could not be parsed.', file=sys.stderr) - sys.exit(1) - - job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token) - - base_url: str = f'https://gitlab.com/{project_name}/-' - - wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token) - debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token) - - augmented_changelog = f'''{changelog.strip()} - -### Download - -- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) -- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))''' - # Docker currently not working - # - Docker image: registry.gitlab.com/{project_name}:{release_tag} - - post_body: str = json.dumps({ - 'tag_name': release_tag, - 'description': augmented_changelog, - 'assets': { - 'links': [ - { - 'name': 'Python Wheel', - 'url': wheel_url, - 'link_type': 'package' - }, - { - 'name': 'Debian Package', - 'url': debian_url, - 'link_type': 'package' - } - ] - } - }) - - gitlab_release_api_url: str = \ - f'https://gitlab.com/api/v4/projects/{project_id}/releases' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'Content-Type': 'application/json; charset=utf-8', - 'User-Agent': 'curl/7.70.0' - } - - request = urllib.request.Request( - gitlab_release_api_url, - post_body.encode('utf-8'), - headers=headers, - method='POST' - ) - try: - response: http.client.HTTPResponse = urllib.request.urlopen(request) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - response_bytes: bytes = response.read() - response_str: str = response_bytes.decode() - response_data: Dict[str, Any] = json.loads(response_str) - - if response_data['tag_name'] != release_tag: - print('Something went wrong...', file=sys.stderr) - print(response_str, file=sys.stderr) - sys.exit(1) - - print(response_data['description']) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index cd3027f..95f9f14 100755 --- a/setup.py +++ b/setup.py @@ -7,17 +7,25 @@ setup( name='multischleuder', version=__version__, author='s3lph', - author_email='1375407-s3lph@users.noreply.gitlab.com', + author_email='s3lph@kabelsalat.ch', description='Merge subscribers and keys of multiple Schleuder lists into one', license='MIT', keywords='schleuder,pgp', - url='https://gitlab.com/s3lph/multischleuder', + url='https://git.kabelsalat.ch/s3lph/multischleuder', packages=find_packages(exclude=['*.test']), install_requires=[ 'python-dateutil', 'PyYAML', 'PGPy', ], + extras_require={ + 'test': [ + 'coverage', + 'pycodestyle', + 'mypy', + 'twine' + ] + }, entry_points={ 'console_scripts': [ 'multischleuder = multischleuder.main:main'