From fd85f5b1c2aff7df4a6e41be36419aeb9afb9c54 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 12 Aug 2023 14:05:06 +0200 Subject: [PATCH] chore: migrate from gitlab-ci to woodpecker --- .gitlab-ci.yml | 119 ------------ .woodpecker.yml | 95 +++++++++ .../DEBIAN/control | 14 ++ .../DEBIAN/postinst | 8 +- .../icalendar-timeseries-server/DEBIAN/prerm | 2 +- package/release.py | 180 ------------------ setup.py | 10 +- 7 files changed, 121 insertions(+), 307 deletions(-) delete mode 100644 .gitlab-ci.yml create mode 100644 .woodpecker.yml create mode 100644 package/debian/icalendar-timeseries-server/DEBIAN/control delete mode 100755 package/release.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f62ee8a..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -image: s3lph/icalendar-timeseries-server-ci:20190820-01 - -stages: -- test -- build -- release -- upload - - - -before_script: -- export ITS_VERSION=$(python -c 'import icalendar_timeseries_server; print(icalendar_timeseries_server.__version__)') - - - -test: - stage: test - script: - - pip3 install -e . - - sudo -u its python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server - - sudo -u its python3 -m coverage combine - - sudo -u its python3 -m coverage report --rcfile=setup.cfg - -codestyle: - stage: test - script: - - pip3 install -e . - - sudo -u its pycodestyle icalendar_timeseries_server - - - -build_wheel: - stage: build - script: - - pip3 install -e . - - python3 setup.py egg_info bdist_wheel - - cd dist - - sha256sum *.whl > SHA256SUMS - artifacts: - paths: - - "dist/*.whl" - - dist/SHA256SUMS - only: - - tags - -build_debian: - stage: build - script: - # The Python package name provided by the python3-magic Debian package is "python-magic" rather than "file-magic". - - sed -re 's/file-magic/python-magic/' -i setup.py - - echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - | - cat > package/debian/icalendar-timeseries-server/DEBIAN/control <= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate - Description: Scrape iCalendar endpoints and present their data in a - timeseries format. A small service that scrapes iCalendar files - served over HTTP, parses their contents and returns a timeseries - format compatible to the /api/v1/query API endpoint of a Prometheus - server. This allows e.g. a Grafana administrator to add a Prometheus - data source pointing at this server, returning the events in the - calendars in the event metric. - EOF - - | - for version in $(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - done - - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - python3.7 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 - - cd package/debian - - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rsync -a icalendar-timeseries-server/usr/lib/python3.7/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rm -rf icalendar-timeseries-server/usr/lib/python3.7/ - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; - - mv icalendar-timeseries-server/usr/bin/icalendar-timeseries-server icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server - - rm -rf icalendar-timeseries-server/usr/bin - - sed -re 's$#!/usr/local/bin/python3.7$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server - - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; - - find icalendar-timeseries-server -type d -exec chmod 755 {} \; - - find icalendar-timeseries-server -type f -name .gitkeep -delete - - chmod +x icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server icalendar-timeseries-server/DEBIAN/postinst icalendar-timeseries-server/DEBIAN/prerm icalendar-timeseries-server/DEBIAN/postrm - - dpkg-deb --build icalendar-timeseries-server - - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" - - sudo -u nobody lintian "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" - - sha256sum *.deb > SHA256SUMS - artifacts: - paths: - - "package/debian/*.deb" - - package/debian/SHA256SUMS - only: - - tags - - - -release: - stage: release - script: - - python package/release.py - only: - - tags - - - -repo: - stage: upload - trigger: s3lph/custom-packages - variables: - MULTIPROJECT_TRIGGER_JOBNAME: icalendar-timeseries-server - only: - - tags diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..7039b14 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,95 @@ +--- + +steps: + + test: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server + + codestyle: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - pycodestyle icalendar_timeseries_server + + coverage: + image: python:3.11-bookworm + group: integration + commands: + - pip3 install -e .[test] + - python3 -m coverage combine + - python3 -m coverage report --rcfile=setup.cfg + + build_wheel: + image: python:3.11-bookworm + group: package + when: + - event: tag + secrets: + - GITEA_API_REPOSITORY_PYPI + - GITEA_API_USERNAME + - GITEA_API_PASSWORD + commands: + - pip3 install -e .[test] + - python3 setup.py egg_info bdist_wheel + - | + cat > ~/.pypirc <" | grep -B 1000 "<"'!'"-- END CHANGES $${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' + echo -e "\n -- s3lph $(date -R)\n" + done) > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - python3 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 + - cd package/debian + - sed -re "s/__ITS_VERSION__/$${ITS_VERSION}/g" -i icalendar-timeseries-server/DEBIAN/control + - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rsync -a icalendar-timeseries-server/usr/lib/python3.11/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rm -rf icalendar-timeseries-server/usr/lib/python3.11/site-packages + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; + - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/bin/icalendar-timeseries-server + - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; + - find icalendar-timeseries-server -type d -exec chmod 755 {} \; + - >- + chmod +x + icalendar-timeseries-server/usr/bin/icalendar-timeseries-server + icalendar-timeseries-server/DEBIAN/postinst + icalendar-timeseries-server/DEBIAN/prerm + icalendar-timeseries-server/DEBIAN/postrm + - dpkg-deb --build icalendar-timeseries-server + - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" + - sudo -u nobody lintian "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" || true + - >- + curl + --user "$${GITEA_API_USERNAME}:$${GITEA_API_PASSWORD}" + --upload-file "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" + $${GITEA_API_REPOSITORY_DEB} diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/control b/package/debian/icalendar-timeseries-server/DEBIAN/control new file mode 100644 index 0000000..c47eafc --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/control @@ -0,0 +1,14 @@ +Package: icalendar-timeseries-server +Version: __ITS_VERSION__ +Maintainer: s3lph +Section: web +Priority: optional +Architecture: all +Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate +Description: Scrape iCalendar endpoints and present their data in a timeseries format. + A small service that scrapes iCalendar files served over HTTP, parses + their contents and returns a timeseries format compatible to the + /api/v1/query API endpoint of a Prometheus server. This allows e.g. a + Grafana administrator to add a Prometheus data source pointing at + this server, returning the events in the calendars in the event + metric. diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/postinst b/package/debian/icalendar-timeseries-server/DEBIAN/postinst index be22d72..c132aa8 100755 --- a/package/debian/icalendar-timeseries-server/DEBIAN/postinst +++ b/package/debian/icalendar-timeseries-server/DEBIAN/postinst @@ -15,11 +15,7 @@ if [[ "$1" == "configure" ]]; then chown its:its /var/lib/its chmod 0750 /var/lib/its - systemctl daemon-reload || true - - if [[ "$2" != "" ]]; then - # Restart after upgrading the package - systemctl restart icalendar-timeseries-server.service - fi + deb-systemd-helper enable icalendar-timeseries-server.service + deb-systemd-invoke restart icalendar-timeseries-server.service fi diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/prerm b/package/debian/icalendar-timeseries-server/DEBIAN/prerm index 42c7b34..1191729 100755 --- a/package/debian/icalendar-timeseries-server/DEBIAN/prerm +++ b/package/debian/icalendar-timeseries-server/DEBIAN/prerm @@ -4,6 +4,6 @@ set -e if [[ "$1" == "remove" ]]; then - userdel its + deb-systemd-invoke stop icalendar-timeseries-server.service fi diff --git a/package/release.py b/package/release.py deleted file mode 100755 index 431c569..0000000 --- a/package/release.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 - -from typing import Any, Dict, List, Optional, Tuple - -import os -import sys -import json -import urllib.request -import http.client -from urllib.error import HTTPError - - -def parse_changelog(tag: str) -> Optional[str]: - release_changelog: str = '' - with open('CHANGELOG.md', 'r') as f: - in_target: bool = False - done: bool = False - for line in f.readlines(): - if in_target: - if f'' in line: - done = True - break - release_changelog += line - elif f'' in line: - in_target = True - continue - if not done: - return None - return release_changelog - - -def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: - url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'User-Agent': 'curl/7.70.0' - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.read() - joblist: List[Dict[str, Any]] = json.loads(resp_data.decode()) - - jobidmap: Dict[str, str] = {} - for job in joblist: - name: str = job['name'] - job_id: str = job['id'] - jobidmap[name] = job_id - return jobidmap - - -def fetch_single_shafile(url: str) -> str: - headers: Dict[str, str] = { - 'User-Agent': 'curl/7.70.0' - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.readline() - shafile: str = resp_data.decode() - filename: str = shafile.strip().split(' ')[-1].strip() - return filename - - -def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' - wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' - wheel_filename: str = fetch_single_shafile(wheel_sha_url) - wheel_url: str = f'{mybase}/dist/{wheel_filename}' - return wheel_url, wheel_sha_url - - -def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' - debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' - debian_filename: str = fetch_single_shafile(debian_sha_url) - debian_url: str = f'{mybase}/package/debian/{debian_filename}' - return debian_url, debian_sha_url - - -def main(): - api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN') - release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG') - project_name: Optional[str] = os.getenv('CI_PROJECT_PATH') - project_id: Optional[str] = os.getenv('CI_PROJECT_ID') - pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID') - if api_token is None: - print('GITLAB_API_TOKEN is not set.', file=sys.stderr) - sys.exit(1) - if release_tag is None: - print('CI_COMMIT_TAG is not set.', file=sys.stderr) - sys.exit(1) - if project_name is None: - print('CI_PROJECT_PATH is not set.', file=sys.stderr) - sys.exit(1) - if project_id is None: - print('CI_PROJECT_ID is not set.', file=sys.stderr) - sys.exit(1) - if pipeline_id is None: - print('CI_PIPELINE_ID is not set.', file=sys.stderr) - sys.exit(1) - - changelog: Optional[str] = parse_changelog(release_tag) - if changelog is None: - print('Changelog could not be parsed.', file=sys.stderr) - sys.exit(1) - - job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token) - - base_url: str = f'https://gitlab.com/{project_name}/-' - - wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) - debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) - - augmented_changelog = f'''{changelog.strip()} - -### Download - -- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) -- [Debian Package]({debian_url}) ([sha256]({debian_sha_url})) -- Docker image: registry.gitlab.com/{project_name}:{release_tag}''' - - post_body: str = json.dumps({ - 'tag_name': release_tag, - 'description': augmented_changelog, - 'assets': { - 'links': [ - { - 'name': 'Python Wheel', - 'url': wheel_url, - 'link_type': 'package' - }, - { - 'name': 'Debian Package', - 'url': debian_url, - 'link_type': 'package' - } - ] - } - }) - - gitlab_release_api_url: str = \ - f'https://gitlab.com/api/v4/projects/{project_id}/releases' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'Content-Type': 'application/json; charset=utf-8', - 'User-Agent': 'curl/7.70.0' - } - - request = urllib.request.Request( - gitlab_release_api_url, - post_body.encode('utf-8'), - headers=headers, - method='POST' - ) - try: - response: http.client.HTTPResponse = urllib.request.urlopen(request) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - response_bytes: bytes = response.read() - response_str: str = response_bytes.decode() - response_data: Dict[str, Any] = json.loads(response_str) - - if response_data['tag_name'] != release_tag: - print('Something went wrong...', file=sys.stderr) - print(response_str, file=sys.stderr) - sys.exit(1) - - print(response_data['description']) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index dfb999b..292e5c8 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( description='', license='MIT', keywords='ical,icalendar,timeseries,prometheus,grafana', - url='https://gitlab.com/s3lph/icalendar-timeseries-server', + url='https://git.kabelsalat.ch/s3lph/icalendar-timeseries-server', packages=find_packages(exclude=['*.test']), long_description='', python_requires='>=3.6', @@ -23,6 +23,14 @@ setup( 'isodate', 'jinja2' ], + extras_require={ + 'test': [ + 'coverage', + 'pycodestyle', + 'mypy', + 'twine' + ] + }, entry_points={ 'console_scripts': [ 'icalendar-timeseries-server = icalendar_timeseries_server:main'