From dec4e98a0ca71325d0e8d1cc66bf18298edf6623 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 1 Sep 2019 23:13:24 +0200 Subject: [PATCH 01/28] Use a logger rather than print statements. Unfortunately, bottle.py logs to stderr/stdout on its own. --- icalendar_timeseries_server/api.py | 15 +++----------- icalendar_timeseries_server/cal.py | 9 ++++++-- icalendar_timeseries_server/config.py | 18 +++++++++++----- icalendar_timeseries_server/main.py | 30 ++++++++++++++++++++------- icalendar_timeseries_server/query.py | 3 ++- 5 files changed, 48 insertions(+), 27 deletions(-) diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py index 5674f0e..0a22acb 100644 --- a/icalendar_timeseries_server/api.py +++ b/icalendar_timeseries_server/api.py @@ -1,8 +1,7 @@ from typing import List import json -from urllib.error import HTTPError -import traceback +import logging import bottle @@ -26,7 +25,7 @@ def prometheus_api(): 'error': str(e) } bottle.response.status = 400 - traceback.print_exc() + logging.exception('Cannot parse PromQL query') bottle.response.add_header('Content-Type', 'application/json') return json.dumps(response) @@ -42,14 +41,6 @@ def prometheus_api(): 'result': [e.serialize() for e in events] } } - except HTTPError as e: - response = { - 'status': 'error', - 'errorType': 'internal', - 'error': str(e) - } - bottle.response.status = 500 - traceback.print_exc() except BaseException: response = { 'status': 'error', @@ -57,7 +48,7 @@ def prometheus_api(): 'error': 'An internal error occurred.' } bottle.response.status = 500 - traceback.print_exc() + logging.exception('An internal error occurred') bottle.response.add_header('Content-Type', 'application/json') return json.dumps(response) diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index a2e2200..434e250 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -2,6 +2,7 @@ from typing import Dict, List, Iterable import sys import urllib.request +import logging from datetime import datetime, date, timedelta from threading import Lock, Timer @@ -54,8 +55,12 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da events = [] opener: urllib.request.OpenerDirector = config.get_url_opener() - with opener.open(config.url) as response: - data = response.read().decode('utf-8') + try: + with opener.open(config.url) as response: + data = response.read().decode('utf-8') + except BaseException: + logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ({config.url})') + return calendar = cal.Calendar.from_ical(data) for element in calendar.walk(): diff --git a/icalendar_timeseries_server/config.py b/icalendar_timeseries_server/config.py index bfbde2c..81cc795 100644 --- a/icalendar_timeseries_server/config.py +++ b/icalendar_timeseries_server/config.py @@ -6,6 +6,7 @@ from datetime import timedelta import ssl import urllib.request import sys +import logging import pytz import jinja2 @@ -155,7 +156,7 @@ def _keycheck(key: str, raise KeyError(f'Expected member "{key}" not found at path {path}') value: Any = config[key] if not isinstance(value, typ): - raise TypeError(f'Expected {typ}, not {type(value).__name__} for path {path}.{key}') + raise TypeError(f'Expected {typ.__name__}, not {type(value).__name__} for path {path}.{key}') if valid_values is not None: if value not in valid_values: raise ValueError(f'Expected one of {", ".join(valid_values)} ({typ}), not {value} for path {path}.{key}') @@ -216,10 +217,17 @@ def get_jenv() -> jinja2.Environment: def load_config(filename: str): global CONFIG, JENV - with open(filename, 'r') as f: - json_config = json.loads(f.read()) - CONFIG = Config(json_config) - JENV = jinja2.Environment() + try: + with open(filename, 'r') as f: + json_config = json.loads(f.read()) + CONFIG = Config(json_config) + JENV = jinja2.Environment() + except json.JSONDecodeError as e: + logging.exception('Cannot parse config JSON') + raise e + except Exception as e: + logging.error(e) + raise e def load_default_config(): diff --git a/icalendar_timeseries_server/main.py b/icalendar_timeseries_server/main.py index a1168f0..e056dc0 100644 --- a/icalendar_timeseries_server/main.py +++ b/icalendar_timeseries_server/main.py @@ -1,4 +1,5 @@ import sys +import logging import bottle @@ -11,17 +12,32 @@ from icalendar_timeseries_server.api import prometheus_api def main(): - if len(sys.argv) == 1: - load_default_config() - elif len(sys.argv) == 2: - load_config(sys.argv[1]) - else: - print(f'Can only read one config file, got "{" ".join(sys.argv[1:])}"') - exit(1) + # Set up logger + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter( + '%(asctime)s %(filename)s:%(lineno)d(%(funcName)s) [%(levelname)s]: %(message)s')) + logging.getLogger().addHandler(log_handler) + + # Load configuration config = get_config() + try: + if len(sys.argv) == 1: + load_default_config() + elif len(sys.argv) == 2: + load_config(sys.argv[1]) + else: + logging.log(logging.FATAL, f'Can only read one config file, got "{" ".join(sys.argv[1:])}"') + exit(1) + # Re-fetch config after parsing + config = get_config() + except: + logging.fatal('Could not parse configuration file') + exit(1) + # Schedule calendar scraping in the background for calname in config.calendars.keys(): start_scrape_calendar(calname, config.calendars[calname]) + # Start the Bottle HTTP server bottle.run(host=config.addr, port=get_config().port) diff --git a/icalendar_timeseries_server/query.py b/icalendar_timeseries_server/query.py index b266898..ce2ebfc 100644 --- a/icalendar_timeseries_server/query.py +++ b/icalendar_timeseries_server/query.py @@ -1,6 +1,7 @@ from typing import Dict import re +import logging LABEL_MATCH_OPERATORS = [ '=', @@ -64,7 +65,7 @@ class MetricQuery: self.__parse(q) def __parse(self, q: str): - print(q) + logging.debug(f'Parsing PromQL query string: {q}') # globalstate: # 0 = parsing metric name # 1 = parsing filters -- 2.39.5 From 3465e92d03e958e9fe303ea2e42bb1d2f49d0b1a Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 1 Sep 2019 23:18:23 +0200 Subject: [PATCH 02/28] Codestyle: bare except --- icalendar_timeseries_server/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icalendar_timeseries_server/main.py b/icalendar_timeseries_server/main.py index e056dc0..73e7e55 100644 --- a/icalendar_timeseries_server/main.py +++ b/icalendar_timeseries_server/main.py @@ -30,7 +30,7 @@ def main(): exit(1) # Re-fetch config after parsing config = get_config() - except: + except BaseException: logging.fatal('Could not parse configuration file') exit(1) -- 2.39.5 From 20ae18e064fe24912d30ca9b4123e5492fc5f971 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 1 Sep 2019 23:24:23 +0200 Subject: [PATCH 03/28] Fix: Ensure scrape interval is positive --- icalendar-timeseries-server.json | 32 --------------------------- icalendar_timeseries_server/config.py | 3 ++- 2 files changed, 2 insertions(+), 33 deletions(-) delete mode 100644 icalendar-timeseries-server.json diff --git a/icalendar-timeseries-server.json b/icalendar-timeseries-server.json deleted file mode 100644 index 847ee7d..0000000 --- a/icalendar-timeseries-server.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "addr": "127.0.0.1", - "port": 8090, - "start_delta": "-PT3H", - "end_delta": "P60D", - "cache": "PT3M", - "tz": "Europe/Zurich", - "calendars": { - "tlstest": { - "interval": "PT5M", - "url": "https://localhost/private.ics", - "ca": "/home/sebastian/tlstest/ca/ca/ca.crt", - "auth": { - "type": "tls", - "keyfile": "/home/sebastian/tlstest/client/combined.pem" - } - }, - "filetest": { - "interval": "PT1M", - "url": "file:///srv/http/private.ics" - } - }, - "key_replace": { - "summary": "a_summary", - "description": "b_description" - }, - "value_replace": { - "summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", - "description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", - "useless_metric": "{{ start.timestamp() + end.timestamp() }}" - } -} diff --git a/icalendar_timeseries_server/config.py b/icalendar_timeseries_server/config.py index 81cc795..c3d18df 100644 --- a/icalendar_timeseries_server/config.py +++ b/icalendar_timeseries_server/config.py @@ -28,7 +28,8 @@ class CalendarConfig: def __init__(self, config: Dict[str, Any], config_path: str) -> None: self._url: str = _keycheck('url', config, str, config_path) - self._scrape_interval: Duration = _parse_timedelta('interval', config, config_path, default_value='PT15M') + self._scrape_interval: Duration = _parse_timedelta('interval', config, config_path, default_value='PT15M', + force_positive=True) self._ca: Optional[str] = _keycheck('ca', config, str, config_path, optional=True) auth: Dict[str, Any] = _keycheck('auth', config, dict, config_path, default_value={'type': 'none'}) self._authtype: str = _keycheck('type', auth, str, f'{config_path}.auth', -- 2.39.5 From 895a72a348c7ac55619536117b067b333691dbf5 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 21 Sep 2019 14:43:48 +0200 Subject: [PATCH 04/28] Fix: Keep showing events that already started, but have not finished yet --- CHANGELOG.md | 12 ++++++++++++ icalendar_timeseries_server/cal.py | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ae123..168bb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.3 + +### Changes + + +- Keep showing events that already started, but have not finished yet + + + + + ## Version 0.2 diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index 434e250..8db89b5 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -66,12 +66,13 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da for element in calendar.walk(): if element.name == "VEVENT": dtstart = element.get('dtstart').dt - if isinstance(dtstart, date): + # Apparently datetime is a subclass of date... + if isinstance(dtstart, date) and not isinstance(dtstart, datetime): dtstart = datetime(dtstart.year, dtstart.month, dtstart.day, tzinfo=start.tzinfo) # Process either end timestamp or duration, if present if 'dtend' in element: evend = element.get('dtend').dt - if isinstance(evend, date): + if isinstance(evend, date) and not isinstance(evend, datetime): evend = datetime(evend.year, evend.month, evend.day, tzinfo=start.tzinfo) duration = evend - dtstart elif 'duration' in element: @@ -83,7 +84,7 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da else: occurences = [dtstart] for occurence in occurences: - if start <= occurence < end: + if start <= occurence + duration and occurence < end: events.append(Event(name, element, occurence, occurence + duration)) with _SCRAPE_CACHE_LOCK: _SCRAPE_CACHE[name] = events -- 2.39.5 From c7e881f6087063b1d26064998badcfa160118b39 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 21 Sep 2019 14:48:23 +0200 Subject: [PATCH 05/28] Add missing items to changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 168bb2f..97df943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ ### Changes -- Keep showing events that already started, but have not finished yet +- Replace print statements by proper logging +- Fix: Ensure scrape interval is positive +- Fix: Keep showing events that already started, but have not finished yet -- 2.39.5 From d1f6f7c2204dcf04c6d6f8400986a6d4c5dc9402 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 21 Sep 2019 15:01:54 +0200 Subject: [PATCH 06/28] Bump version number --- icalendar_timeseries_server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 373d726..d8ea336 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.1' +__version__ = '0.3' -- 2.39.5 From 3618f86bd19ae8218bd0cf6139ebb50ad3882831 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 21 Sep 2019 15:10:40 +0200 Subject: [PATCH 07/28] v0.3.1 Version number bump, because Gitlab releases are weird --- CHANGELOG.md | 12 ++++++++++++ icalendar_timeseries_server/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97df943..b29a78e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.3.1 + +### Changes + + +- Bump Version Number + + + + + ## Version 0.3 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index d8ea336..81cfc8c 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.3' +__version__ = '0.3.1' -- 2.39.5 From eaa3a31947e358def5a6ffdfcb3a1a61aca955e6 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 24 Sep 2019 16:23:35 +0200 Subject: [PATCH 08/28] Fix debian package build process: Version number and changelog --- .gitlab-ci.yml | 19 ++++++++++++++++++- .../DEBIAN/control | 14 -------------- 2 files changed, 18 insertions(+), 15 deletions(-) delete mode 100644 package/debian/icalendar-timeseries-server/DEBIAN/control diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1904415..88c7228 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,24 @@ build_debian: - sed -re 's/file-magic/python-magic/' -i setup.py - echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - | - for version in "$(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/control b/package/debian/icalendar-timeseries-server/DEBIAN/control deleted file mode 100644 index 799d2f7..0000000 --- a/package/debian/icalendar-timeseries-server/DEBIAN/control +++ /dev/null @@ -1,14 +0,0 @@ -Package: icalendar-timeseries-server -Version: 0.1 -Maintainer: s3lph -Section: web -Priority: optional -Architecture: all -Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate, python3-tz -Description: Scrape iCalendar endpoints and present their data in a - timeseries format. A small service that scrapes iCalendar files - served over HTTP, parses their contents and returns a timeseries - format compatible to the /api/v1/query API endpoint of a Prometheus - server. This allows e.g. a Grafana administrator to add a Prometheus - data source pointing at this server, returning the events in the - calendars in the event metric. -- 2.39.5 From 293b30ba59cf759240e0e0e8c1f9487c0aa990cf Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 24 Sep 2019 16:25:18 +0200 Subject: [PATCH 09/28] Bump version number to 0.3.2 --- CHANGELOG.md | 12 ++++++++++++ icalendar_timeseries_server/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b29a78e..689f8c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.3.2 + +### Changes + + +- Fix Debian package build process + + + + + ## Version 0.3.1 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 81cfc8c..8299a60 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.3.1' +__version__ = '0.3.2' -- 2.39.5 From afd4710c44f0a93b63ad23285a2f2d96d2d3bf43 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 18 Nov 2019 21:37:20 +0100 Subject: [PATCH 10/28] Debian: Add postinst restart hook --- package/debian/icalendar-timeseries-server/DEBIAN/postinst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/postinst b/package/debian/icalendar-timeseries-server/DEBIAN/postinst index b7586de..be22d72 100755 --- a/package/debian/icalendar-timeseries-server/DEBIAN/postinst +++ b/package/debian/icalendar-timeseries-server/DEBIAN/postinst @@ -17,4 +17,9 @@ if [[ "$1" == "configure" ]]; then systemctl daemon-reload || true + if [[ "$2" != "" ]]; then + # Restart after upgrading the package + systemctl restart icalendar-timeseries-server.service + fi + fi -- 2.39.5 From e29ef0ff2c58916fccb992e853057a1ab288298a Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 18 Nov 2019 21:46:41 +0100 Subject: [PATCH 11/28] Remove pytz dependency in favor of dateutil.tz --- .gitlab-ci.yml | 2 +- icalendar_timeseries_server/config.py | 12 ++++++++---- icalendar_timeseries_server/test/test_config.py | 12 ++++++------ setup.py | 5 ++--- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88c7228..546c461 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ build_debian: Section: web Priority: optional Architecture: all - Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate, python3-tz + Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate Description: Scrape iCalendar endpoints and present their data in a timeseries format. A small service that scrapes iCalendar files served over HTTP, parses their contents and returns a timeseries diff --git a/icalendar_timeseries_server/config.py b/icalendar_timeseries_server/config.py index c3d18df..b763527 100644 --- a/icalendar_timeseries_server/config.py +++ b/icalendar_timeseries_server/config.py @@ -8,9 +8,10 @@ import urllib.request import sys import logging -import pytz import jinja2 from isodate import Duration, parse_duration +from dateutil import tz +from datetime import tzinfo from icalendar_timeseries_server import __version__ @@ -93,7 +94,7 @@ class Config: config = dict() self._addr: str = _keycheck('addr', config, str, '', default_value='127.0.0.1') self._port: int = _keycheck('port', config, int, '', default_value=8090) - self._tz: pytz.tzinfo = _parse_timezone('tz', config, '', default_value='UTC') + self._tz: tzinfo = _parse_timezone('tz', config, '', default_value='UTC') self._start_delta: Duration = _parse_timedelta('start_delta', config, '', default_value='PT') self._end_delta: Duration = _parse_timedelta('end_delta', config, '', default_value='P30D') self._calendars: Dict[str, CalendarConfig] = self._parse_calendars_config('calendars', config, '') @@ -120,7 +121,7 @@ class Config: return self._port @property - def tz(self) -> pytz.tzinfo: + def tz(self) -> tzinfo: return self._tz @property @@ -185,7 +186,10 @@ def _parse_timezone(key: str, path: str, default_value: Any = None) -> Any: zonename: str = _keycheck(key, config, str, path, default_value=default_value) - return pytz.timezone(zonename) + zone: zoneinfo = tz.gettz(zonename) + if zone is None: + raise ValueError(f'Unknown timezone: {zonename}') + return zone def _parse_key_replace(key: str, diff --git a/icalendar_timeseries_server/test/test_config.py b/icalendar_timeseries_server/test/test_config.py index c0f7601..3d9be55 100644 --- a/icalendar_timeseries_server/test/test_config.py +++ b/icalendar_timeseries_server/test/test_config.py @@ -2,9 +2,9 @@ import unittest import json -import pytz -from datetime import timedelta +from datetime import timedelta, tzinfo +from dateutil import tz from isodate.duration import Duration from icalendar_timeseries_server.config import _keycheck, _parse_timedelta, _parse_timezone, Config @@ -113,10 +113,10 @@ class ConfigTest(unittest.TestCase): 'tz': 'Europe/Zurich', 'notz': 'North/Winterfell' } - self.assertEqual(_parse_timezone('tz', config, ''), pytz.timezone('Europe/Zurich')) + self.assertEqual(_parse_timezone('tz', config, ''), tz.gettz('Europe/Zurich')) self.assertEqual(_parse_timezone('def', config, '', default_value='Europe/Berlin'), - pytz.timezone('Europe/Berlin')) - with self.assertRaises(pytz.exceptions.UnknownTimeZoneError): + tz.gettz('Europe/Berlin')) + with self.assertRaises(ValueError): _parse_timezone('notz', config, '') def test_parse_full_config_valid(self): @@ -125,7 +125,7 @@ class ConfigTest(unittest.TestCase): self.assertEqual(config.port, 8090) self.assertEqual(config.start_delta, Duration(hours=-3)) self.assertEqual(config.end_delta, Duration(days=30)) - self.assertEqual(config.tz, pytz.timezone('Europe/Zurich')) + self.assertEqual(config.tz, tz.gettz('Europe/Zurich')) def test_parse_calendars(self): config = Config(json.loads(_CONFIG_VALID)) diff --git a/setup.py b/setup.py index e5aaae3..dfb999b 100755 --- a/setup.py +++ b/setup.py @@ -18,11 +18,10 @@ setup( python_requires='>=3.6', install_requires=[ 'bottle', - 'python-dateutil', + 'python-dateutil>=2.8', 'icalendar', 'isodate', - 'jinja2', - 'pytz' + 'jinja2' ], entry_points={ 'console_scripts': [ -- 2.39.5 From 58145fe178166ae1d61d724dacf77e91ff94aef1 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 19 Jun 2020 00:35:12 +0200 Subject: [PATCH 12/28] Fix type confusion bug in recurring events --- icalendar_timeseries_server/cal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index 8db89b5..e4e479a 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -26,6 +26,8 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration: occurences: List[datetime] = [] evstart = event.get('dtstart').dt + if isinstance(evstart, date) and not isinstance(evstart, datetime): + evstart = datetime(evstart.year, evstart.month, evstart.day, tzinfo=start.tzinfo) # First occurence lies in the future; no need to process further if evstart >= end: return occurences -- 2.39.5 From 7963dc7bb84c96c9b8e204bca06f8fa8943e2ba8 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 19 Jun 2020 00:38:50 +0200 Subject: [PATCH 13/28] Prepare 0.3.3 release --- CHANGELOG.md | 13 +++++++++++++ README.md | 1 - icalendar_timeseries_server/__init__.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 689f8c9..9dedff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # iCalendar Timeseries Server Changelog + +## Version 0.3.3 + +### Changes + + +- Fix type confusion bug in recurring events +- Remove pytz dependency in favor of dateutil.tz + + + + + ## Version 0.3.2 diff --git a/README.md b/README.md index 41bb59c..c762283 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ The server would transform this into the following API response: - `icalendar`: Parse iCalendar - `isodate`: Parse ISO-8601 time periods - `jinja2`: Template value replacements -- `pytz`: Work with timezones ## Configuration diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 8299a60..2fd20b5 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.3.2' +__version__ = '0.3.3' -- 2.39.5 From b5304cb8d74cf06256de328b371112e051ed7e6b Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 21 Jun 2020 05:01:55 +0200 Subject: [PATCH 14/28] release.py: Replace default user agent by that of curl --- package/release.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/package/release.py b/package/release.py index 6281bd0..fe19919 100755 --- a/package/release.py +++ b/package/release.py @@ -10,6 +10,9 @@ import http.client from urllib.error import HTTPError +USER_AGENT = 'curl/7.70.0' + + def parse_changelog(tag: str) -> Optional[str]: release_changelog: str = '' with open('CHANGELOG.md', 'r') as f: @@ -32,7 +35,8 @@ def parse_changelog(tag: str) -> Optional[str]: def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' headers: Dict[str, str] = { - 'Private-Token': api_token + 'Private-Token': api_token, + 'User-Agent': USER_AGENT } req = urllib.request.Request(url, headers=headers) try: @@ -52,7 +56,10 @@ def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str def fetch_single_shafile(url: str) -> str: - req = urllib.request.Request(url) + headers: Dict[str, str] = { + 'User-Agent': USER_AGENT + } + req = urllib.request.Request(url, headers=headers) try: resp: http.client.HTTPResponse = urllib.request.urlopen(req) except HTTPError as e: @@ -127,7 +134,8 @@ def main(): f'https://gitlab.com/api/v4/projects/{project_id}/repository/tags/{release_tag}/release' headers: Dict[str, str] = { 'Private-Token': api_token, - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'application/json; charset=utf-8', + 'User-Agent': USER_AGENT } request = urllib.request.Request( -- 2.39.5 From 395393345e0a74e6b7c35df3fcc3abf38348315c Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 6 Nov 2020 03:30:47 +0100 Subject: [PATCH 15/28] Implement todo exporting --- CHANGELOG.md | 12 +++++ README.md | 6 +-- icalendar_timeseries_server/__init__.py | 2 +- icalendar_timeseries_server/api.py | 15 ++++-- icalendar_timeseries_server/cal.py | 32 ++++++++--- icalendar_timeseries_server/event.py | 2 +- icalendar_timeseries_server/query.py | 6 ++- icalendar_timeseries_server/todo.py | 72 +++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 icalendar_timeseries_server/todo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dedff8..9df746c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.4.0 + +### Changes + + +- VTODO components are exported in a second time series, `todo` . Todo recurrence is not supported yet though. + + + + + ## Version 0.3.3 diff --git a/README.md b/README.md index c762283..60a5fe6 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ This project is a small service that scrapes iCalendar files served over HTTP, parses their contents and returns the data in a timeseries format compatible to the `/api/v1/query` API endpoint of a Prometheus server. This allows e.g. a Grafana administrator to add a Prometheus -data source pointing at this server, returning the events in the -calendars in the `event` metric. +data source pointing at this server, returning calendar events in the +`event` metric and todos in the `todo` metric. ## Example @@ -147,7 +147,7 @@ Configuration is done through a JSON config file: | `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. | | `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. | | `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). | -| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. | +| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana 6 and earlier. | | `keys(key_replace)` | string | The labels to rename. | | `key_replace.*` | string | The names to rename the labels to. | | `value_replace` | dict | Label values to postprocess. | diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 2fd20b5..652a8f4 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.3.3' +__version__ = '0.4.0' diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py index 0a22acb..fbfd60d 100644 --- a/icalendar_timeseries_server/api.py +++ b/icalendar_timeseries_server/api.py @@ -7,7 +7,7 @@ import bottle from icalendar_timeseries_server.config import get_config from icalendar_timeseries_server.event import Metric -from icalendar_timeseries_server.cal import get_calendar +from icalendar_timeseries_server.cal import get_calendar_events, get_calendar_todos from icalendar_timeseries_server.query import MetricQuery @@ -31,9 +31,16 @@ def prometheus_api(): try: for name in get_config().calendars.keys(): - events.extend(get_calendar(name)) - events = list(filter(q, events)) - events.sort(key=lambda e: e.start) + if q.name == 'event': + events.extend(get_calendar_events(name)) + events = list(filter(q, events)) + events.sort(key=lambda e: e.start) + elif q.name == 'todo': + events.extend(get_calendar_todos(name)) + events = list(filter(q, events)) + [print(e.name, e.due, e.priority) for e in events] + # Sort by due date and priority + events.sort(key=lambda e: (e.due, e.priority)) response = { 'status': 'success', 'data': { diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index e4e479a..46e2c03 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -13,9 +13,11 @@ from isodate import Duration from icalendar_timeseries_server import __version__ from icalendar_timeseries_server.config import get_config, CalendarConfig from icalendar_timeseries_server.event import Event +from icalendar_timeseries_server.todo import Todo -_SCRAPE_CACHE: Dict[str, List[Event]] = dict() +_EVENT_SCRAPE_CACHE: Dict[str, List[Event]] = dict() +_TODO_SCRAPE_CACHE: Dict[str, List[Todo]] = dict() _SCRAPE_CACHE_LOCK: Lock = Lock() __py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}' @@ -53,8 +55,9 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration: def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime): - global _SCRAPE_CACHE, _SCRAPE_CACHE_LOCK + global _EVENT_SCRAPE_CACHE, _TODO_SCRAPE_CACHE, _SCRAPE_CACHE_LOCK events = [] + todos = [] opener: urllib.request.OpenerDirector = config.get_url_opener() try: @@ -88,8 +91,19 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da for occurence in occurences: if start <= occurence + duration and occurence < end: events.append(Event(name, element, occurence, occurence + duration)) + elif element.name == "VTODO": + dtstart = element.get('dtstamp').dt + duration = timedelta(0) + if 'dtstart' in element: + dtstart = element.get('dtstart').dt + if 'duration' in element: + duration = element.get('duration').dt + todos.append(Todo(name, element, dtstart, dtstart + duration)) + + with _SCRAPE_CACHE_LOCK: - _SCRAPE_CACHE[name] = events + _EVENT_SCRAPE_CACHE[name] = events + _TODO_SCRAPE_CACHE[name] = todos def scrape_calendar(name: str, config: CalendarConfig): @@ -115,7 +129,13 @@ def start_scrape_calendar(name: str, config: CalendarConfig): cron.start() -def get_calendar(name: str): - global _SCRAPE_CACHE +def get_calendar_events(name: str): + global _EVENT_SCRAPE_CACHE with _SCRAPE_CACHE_LOCK: - return _SCRAPE_CACHE.get(name, []) + return _EVENT_SCRAPE_CACHE.get(name, []) + + +def get_calendar_todos(name: str): + global _TODO_SCRAPE_CACHE + with _SCRAPE_CACHE_LOCK: + return _TODO_SCRAPE_CACHE.get(name, []) diff --git a/icalendar_timeseries_server/event.py b/icalendar_timeseries_server/event.py index 934b4f7..d03ce79 100644 --- a/icalendar_timeseries_server/event.py +++ b/icalendar_timeseries_server/event.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Set +from typing import Any, Dict, List import icalendar import jinja2 diff --git a/icalendar_timeseries_server/query.py b/icalendar_timeseries_server/query.py index ce2ebfc..d7427cc 100644 --- a/icalendar_timeseries_server/query.py +++ b/icalendar_timeseries_server/query.py @@ -172,7 +172,7 @@ class MetricQuery: elif filterstate != 0: raise ValueError('Unexpected EOF') - def __call__(self, metric: Metric): + def __call__(self, metric: Metric) -> bool: """ Applies the filter deducted from the query string to the given metric. @@ -188,3 +188,7 @@ class MetricQuery: return False # Return True if all filters matched return True + + @property + def name(self) -> str: + return self._metric_name diff --git a/icalendar_timeseries_server/todo.py b/icalendar_timeseries_server/todo.py new file mode 100644 index 0000000..64ad7ad --- /dev/null +++ b/icalendar_timeseries_server/todo.py @@ -0,0 +1,72 @@ +from typing import Any, Dict, List + +import icalendar +import jinja2 +from datetime import datetime, timedelta + +from icalendar_timeseries_server.config import get_config, get_jenv +from icalendar_timeseries_server.query import Metric + +_ATTRIBUTES: List[str] = [ + 'class', + 'description', + 'geo', + 'location', + 'organizer', + 'percent-complete', + 'priority', + 'status', + 'summary', + 'url', + 'due', + 'attach' +] + + +class Todo(Metric): + + def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime): + self.calendar: str = cname + self.start = start + # self.attributes: Dict[str, str] = dict() + attributes: Dict[str, str] = dict() + tmp: Dict[str, Any] = { + 'calendar': cname, + 'start': start, + 'end': end + } + for attr in _ATTRIBUTES: + tmp[attr] = todo.get(attr, '') + substitution_keys = set(_ATTRIBUTES) + substitution_keys.update(tmp.keys()) + substitution_keys.update(get_config().key_replace.keys()) + substitution_keys.update(get_config().value_replace.keys()) + for attr in substitution_keys: + newkey: str = get_config().key_replace.get(attr, attr) + value: str = tmp.get(attr, '') + newval_template: str = get_config().value_replace.get(attr, str(value)) + jtemplate: jinja2.Template = get_jenv().from_string(newval_template) + newvalue: str = jtemplate.render(**tmp) + attributes[newkey] = newvalue + self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}' + due = todo.get('due', None) + if due: + self.due = due.dt + else: + self.due = datetime.now(get_config().tz) + timedelta(days=36500) + self.priority = todo.get('priority', '0') + super().__init__('todo', attributes) + + def serialize(self) -> Dict[str, Any]: + todo: Dict[str, Any] = { + 'metric': { + '__name__': 'todo', + 'calendar': self.calendar + }, + 'value': [ + self.start.timestamp(), + 1 + ] + } + todo['metric'].update(self._labels) + return todo -- 2.39.5 From fae85de8cab8e73b0e8934db02bbf34e53a26b54 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 6 Nov 2020 03:35:22 +0100 Subject: [PATCH 16/28] Fix codestyle --- icalendar_timeseries_server/cal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index 46e2c03..8a6e11c 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -100,7 +100,6 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da duration = element.get('duration').dt todos.append(Todo(name, element, dtstart, dtstart + duration)) - with _SCRAPE_CACHE_LOCK: _EVENT_SCRAPE_CACHE[name] = events _TODO_SCRAPE_CACHE[name] = todos -- 2.39.5 From f99496c497aaf628b5ab1dcb527d28c3e48c95f2 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 24 May 2021 05:33:32 +0200 Subject: [PATCH 17/28] 0.4.1 --- CHANGELOG.md | 13 +++++++++++++ README.md | 6 ++++++ icalendar_timeseries_server/api.py | 3 +-- icalendar_timeseries_server/todo.py | 19 ++++++++++++------- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df746c..202c38b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # iCalendar Timeseries Server Changelog + +## Version 0.4.1 + +### Changes + + +- Fix todo sorting by due date. +- Update README regarding `todo` time series. + + + + + ## Version 0.4.0 diff --git a/README.md b/README.md index 60a5fe6..b3af7e8 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,12 @@ In addition, PromQL label filters can be used. event{calendar="public",foo=~".*"} ``` +Alongside with events, todos are exported in a second time series: + +``` +todo{status!="COMPLETED"} +``` + ## Why Prometheus API - It's JSON. A JSON generator is builtin in Python, so no further dependency. diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py index fbfd60d..694a467 100644 --- a/icalendar_timeseries_server/api.py +++ b/icalendar_timeseries_server/api.py @@ -38,9 +38,8 @@ def prometheus_api(): elif q.name == 'todo': events.extend(get_calendar_todos(name)) events = list(filter(q, events)) - [print(e.name, e.due, e.priority) for e in events] # Sort by due date and priority - events.sort(key=lambda e: (e.due, e.priority)) + events.sort(key=lambda e: (e.due is None, e.due, e.priority)) response = { 'status': 'success', 'data': { diff --git a/icalendar_timeseries_server/todo.py b/icalendar_timeseries_server/todo.py index 64ad7ad..ff5e403 100644 --- a/icalendar_timeseries_server/todo.py +++ b/icalendar_timeseries_server/todo.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List import icalendar import jinja2 -from datetime import datetime, timedelta +from datetime import datetime, date, timedelta from icalendar_timeseries_server.config import get_config, get_jenv from icalendar_timeseries_server.query import Metric @@ -18,7 +18,6 @@ _ATTRIBUTES: List[str] = [ 'status', 'summary', 'url', - 'due', 'attach' ] @@ -28,6 +27,15 @@ class Todo(Metric): def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime): self.calendar: str = cname self.start = start + due = todo.get('due', None) + if due: + if isinstance(due.dt, datetime): + self.due = due.dt + elif isinstance(due.dt, date): + self.due = datetime.combine(due.dt, datetime.min.time()) + self.due = self.due.replace(tzinfo=get_config().tz) + else: + self.due = None # self.attributes: Dict[str, str] = dict() attributes: Dict[str, str] = dict() tmp: Dict[str, Any] = { @@ -35,6 +43,8 @@ class Todo(Metric): 'start': start, 'end': end } + if self.due: + tmp['due'] = str(self.due) for attr in _ATTRIBUTES: tmp[attr] = todo.get(attr, '') substitution_keys = set(_ATTRIBUTES) @@ -49,11 +59,6 @@ class Todo(Metric): newvalue: str = jtemplate.render(**tmp) attributes[newkey] = newvalue self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}' - due = todo.get('due', None) - if due: - self.due = due.dt - else: - self.due = datetime.now(get_config().tz) + timedelta(days=36500) self.priority = todo.get('priority', '0') super().__init__('todo', attributes) -- 2.39.5 From 28c2b1d425f7d88ec0d0d852c08459e2dae09fe6 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 24 May 2021 05:34:52 +0200 Subject: [PATCH 18/28] Bump version to 0.4.1 --- icalendar_timeseries_server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 652a8f4..489c7e1 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.4.0' +__version__ = '0.4.1' -- 2.39.5 From 90b80ae5299502da749dfacf5d543bcfff65c1aa Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 19 Feb 2022 03:29:27 +0100 Subject: [PATCH 19/28] v0.5: Exponential backoff for retrys --- CHANGELOG.md | 12 ++++++++++ icalendar_timeseries_server/__init__.py | 2 +- icalendar_timeseries_server/cal.py | 30 ++++++++++++++----------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202c38b..73d9cd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.5 + +### Changes + + +- Retry calendar scraping with exponential backoff. + + + + + ## Version 0.4.1 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 489c7e1..27f4493 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.4.1' +__version__ = '0.5' diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index 8a6e11c..925fa09 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -60,12 +60,8 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da todos = [] opener: urllib.request.OpenerDirector = config.get_url_opener() - try: - with opener.open(config.url) as response: - data = response.read().decode('utf-8') - except BaseException: - logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ({config.url})') - return + with opener.open(config.url) as response: + data = response.read().decode('utf-8') calendar = cal.Calendar.from_ical(data) for element in calendar.walk(): @@ -105,26 +101,34 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da _TODO_SCRAPE_CACHE[name] = todos -def scrape_calendar(name: str, config: CalendarConfig): +def scrape_calendar(name: str, config: CalendarConfig, retry: int): # Get current time in configured timezone tz = get_config().tz now: datetime = datetime.now(tz) - # Reschedule calendar scraping - cron = Timer(config.interval.totimedelta(start=now).total_seconds(), - lambda: scrape_calendar(name, config)) - cron.start() + # Only scrape at most once a minute + interval = max(int(config.interval.totimedelta(start=now).total_seconds()), 60) # Compute interval for which to return events start_delta: Duration = get_config().start_delta end_delta: Duration = get_config().end_delta start: datetime = now + start_delta end: datetime = now + end_delta # Scrape and parse the calendar - _scrape_calendar(name, config, start, end) + try: + _scrape_calendar(name, config, start, end) + # Reschedule calendar scraping + cron = Timer(interval, lambda: scrape_calendar(name, config, 0)) + except BaseException: + # reschedule with exponential backoff, but no more than the regular scrape interval + backoff_seconds = min(60 * 2**retry, interval) + logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ' + f'({config.url}), retrying in {backoff_seconds}s.') + cron = Timer(backoff_seconds, lambda: scrape_calendar(name, config, retry+1)) + cron.start() def start_scrape_calendar(name: str, config: CalendarConfig): # Schedule first calendar scraping - cron = Timer(0, lambda: scrape_calendar(name, config)) + cron = Timer(0, lambda: scrape_calendar(name, config, retry=0)) cron.start() -- 2.39.5 From 396e0e5c5f7afb9f8aa7d6248b25c16fc6903332 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 19 Feb 2022 03:48:26 +0100 Subject: [PATCH 20/28] Fix release script --- package/release.py | 56 ++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/package/release.py b/package/release.py index fe19919..4f09ba3 100755 --- a/package/release.py +++ b/package/release.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 from typing import Any, Dict, List, Optional, Tuple @@ -10,9 +9,6 @@ import http.client from urllib.error import HTTPError -USER_AGENT = 'curl/7.70.0' - - def parse_changelog(tag: str) -> Optional[str]: release_changelog: str = '' with open('CHANGELOG.md', 'r') as f: @@ -32,11 +28,11 @@ def parse_changelog(tag: str) -> Optional[str]: return release_changelog -def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: +def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]: url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' headers: Dict[str, str] = { 'Private-Token': api_token, - 'User-Agent': USER_AGENT + 'User-Agent': 'curl/7.70.0' } req = urllib.request.Request(url, headers=headers) try: @@ -55,9 +51,10 @@ def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str return jobidmap -def fetch_single_shafile(url: str) -> str: +def fetch_single_shafile(url: str, api_token: str) -> str: headers: Dict[str, str] = { - 'User-Agent': USER_AGENT + 'User-Agent': 'curl/7.70.0', + 'Private-Token': api_token } req = urllib.request.Request(url, headers=headers) try: @@ -71,18 +68,20 @@ def fetch_single_shafile(url: str) -> str: return filename -def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: +def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' - wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' - wheel_filename: str = fetch_single_shafile(wheel_sha_url) + wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\ + '/artifacts/dist/SHA256SUMS' + wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token) wheel_url: str = f'{mybase}/dist/{wheel_filename}' return wheel_url, wheel_sha_url -def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: +def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' - debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' - debian_filename: str = fetch_single_shafile(debian_sha_url) + debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\ + '/artifacts/package/debian/SHA256SUMS' + debian_filename: str = fetch_single_shafile(debian_sha_url, api_token) debian_url: str = f'{mybase}/package/debian/{debian_filename}' return debian_url, debian_sha_url @@ -118,8 +117,8 @@ def main(): base_url: str = f'https://gitlab.com/{project_name}/-' - wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) - debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) + wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token) + debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token) augmented_changelog = f'''{changelog.strip()} @@ -127,15 +126,34 @@ def main(): - [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) - [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))''' + # Docker currently not working + # - Docker image: registry.gitlab.com/{project_name}:{release_tag} - post_body: str = json.dumps({'description': augmented_changelog}) + post_body: str = json.dumps({ + 'tag_name': release_tag, + 'description': augmented_changelog, + 'assets': { + 'links': [ + { + 'name': 'Python Wheel', + 'url': wheel_url, + 'link_type': 'package' + }, + { + 'name': 'Debian Package', + 'url': debian_url, + 'link_type': 'package' + } + ] + } + }) gitlab_release_api_url: str = \ - f'https://gitlab.com/api/v4/projects/{project_id}/repository/tags/{release_tag}/release' + f'https://gitlab.com/api/v4/projects/{project_id}/releases' headers: Dict[str, str] = { 'Private-Token': api_token, 'Content-Type': 'application/json; charset=utf-8', - 'User-Agent': USER_AGENT + 'User-Agent': 'curl/7.70.0' } request = urllib.request.Request( -- 2.39.5 From dc3f7101f0930d50ccc415b52ec822b4714533ea Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jul 2022 03:01:46 +0200 Subject: [PATCH 21/28] Fix: A specific API field has to be a string rather than float, and recent Grafana versions validate this. --- CHANGELOG.md | 13 +++++++++++++ icalendar_timeseries_server/__init__.py | 2 +- icalendar_timeseries_server/event.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73d9cd9..4446064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # iCalendar Timeseries Server Changelog + +## Version 0.6 + +### Changes + + +- Fix: A specific API field has to be a string rather than float, and recent Grafana versions validate this. + + + + + + ## Version 0.5 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 27f4493..80fd09f 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.5' +__version__ = '0.6' diff --git a/icalendar_timeseries_server/event.py b/icalendar_timeseries_server/event.py index d03ce79..46b25d7 100644 --- a/icalendar_timeseries_server/event.py +++ b/icalendar_timeseries_server/event.py @@ -59,7 +59,7 @@ class Event(Metric): }, 'value': [ self.start.timestamp(), - 1 + "1" ] } event['metric'].update(self._labels) -- 2.39.5 From 675e391ea5b27bc8bc659450299864094a1b98e8 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jul 2022 03:02:36 +0200 Subject: [PATCH 22/28] release.py: Use new Gitlab releases API --- package/release.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/package/release.py b/package/release.py index 4f09ba3..431c569 100755 --- a/package/release.py +++ b/package/release.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from typing import Any, Dict, List, Optional, Tuple @@ -28,7 +29,7 @@ def parse_changelog(tag: str) -> Optional[str]: return release_changelog -def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str, str]: +def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' headers: Dict[str, str] = { 'Private-Token': api_token, @@ -51,10 +52,9 @@ def fetch_job_ids(project_id: str, pipeline_id: str, api_token: str) -> Dict[str return jobidmap -def fetch_single_shafile(url: str, api_token: str) -> str: +def fetch_single_shafile(url: str) -> str: headers: Dict[str, str] = { - 'User-Agent': 'curl/7.70.0', - 'Private-Token': api_token + 'User-Agent': 'curl/7.70.0' } req = urllib.request.Request(url, headers=headers) try: @@ -68,20 +68,18 @@ def fetch_single_shafile(url: str, api_token: str) -> str: return filename -def fetch_wheel_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: +def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' - wheel_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_wheel"]}'\ - '/artifacts/dist/SHA256SUMS' - wheel_filename: str = fetch_single_shafile(wheel_sha_url, api_token) + wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' + wheel_filename: str = fetch_single_shafile(wheel_sha_url) wheel_url: str = f'{mybase}/dist/{wheel_filename}' return wheel_url, wheel_sha_url -def fetch_debian_url(base_url: str, project_id: str, job_ids: Dict[str, str], api_token: str) -> Optional[Tuple[str, str]]: +def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' - debian_sha_url: str = f'https://gitlab.com/api/v4/projects/{project_id}/jobs/{job_ids["build_debian"]}'\ - '/artifacts/package/debian/SHA256SUMS' - debian_filename: str = fetch_single_shafile(debian_sha_url, api_token) + debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' + debian_filename: str = fetch_single_shafile(debian_sha_url) debian_url: str = f'{mybase}/package/debian/{debian_filename}' return debian_url, debian_sha_url @@ -117,17 +115,16 @@ def main(): base_url: str = f'https://gitlab.com/{project_name}/-' - wheel_url, wheel_sha_url = fetch_wheel_url(base_url, project_id, job_ids, api_token) - debian_url, debian_sha_url = fetch_debian_url(base_url, project_id, job_ids, api_token) + wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) + debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) augmented_changelog = f'''{changelog.strip()} ### Download - [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) -- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))''' - # Docker currently not working - # - Docker image: registry.gitlab.com/{project_name}:{release_tag} +- [Debian Package]({debian_url}) ([sha256]({debian_sha_url})) +- Docker image: registry.gitlab.com/{project_name}:{release_tag}''' post_body: str = json.dumps({ 'tag_name': release_tag, -- 2.39.5 From 879bd5b1b74587073d1f8742696d791c3188f04b Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jul 2022 03:39:46 +0200 Subject: [PATCH 23/28] 0.6.1 --- CHANGELOG.md | 13 +++++++++++++ icalendar_timeseries_server/__init__.py | 2 +- icalendar_timeseries_server/todo.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4446064..d9a9830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # iCalendar Timeseries Server Changelog + +## Version 0.6.1 + +### Changes + + +- Same fix, but for todo as well as events. + + + + + + ## Version 0.6 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 80fd09f..7a2d0cd 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.6' +__version__ = '0.6.1' diff --git a/icalendar_timeseries_server/todo.py b/icalendar_timeseries_server/todo.py index ff5e403..1493c24 100644 --- a/icalendar_timeseries_server/todo.py +++ b/icalendar_timeseries_server/todo.py @@ -70,7 +70,7 @@ class Todo(Metric): }, 'value': [ self.start.timestamp(), - 1 + "1" ] } todo['metric'].update(self._labels) -- 2.39.5 From 69315c80e84dce85261192a6e0f162c450edc4ce Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jul 2022 23:29:07 +0200 Subject: [PATCH 24/28] Add CI task to trigger a build of the repository pipeline --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 546c461..d683710 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ stages: - test - build - release +- upload @@ -106,3 +107,13 @@ release: - python package/release.py only: - tags + + + +repo: + stage: upload + trigger: s3lph/custom-packages + variables: + MULTIPROJECT_TRIGGER_JOBNAME: icalendar-timeseries-server + #only: + #- tags -- 2.39.5 From c94003cba77b5920433f1856b5d291134a23a628 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jul 2022 23:33:03 +0200 Subject: [PATCH 25/28] Works, restricting to tags only --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d683710..f62ee8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,5 +115,5 @@ repo: trigger: s3lph/custom-packages variables: MULTIPROJECT_TRIGGER_JOBNAME: icalendar-timeseries-server - #only: - #- tags + only: + - tags -- 2.39.5 From fd85f5b1c2aff7df4a6e41be36419aeb9afb9c54 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 12 Aug 2023 14:05:06 +0200 Subject: [PATCH 26/28] chore: migrate from gitlab-ci to woodpecker --- .gitlab-ci.yml | 119 ------------ .woodpecker.yml | 95 +++++++++ .../DEBIAN/control | 14 ++ .../DEBIAN/postinst | 8 +- .../icalendar-timeseries-server/DEBIAN/prerm | 2 +- package/release.py | 180 ------------------ setup.py | 10 +- 7 files changed, 121 insertions(+), 307 deletions(-) delete mode 100644 .gitlab-ci.yml create mode 100644 .woodpecker.yml create mode 100644 package/debian/icalendar-timeseries-server/DEBIAN/control delete mode 100755 package/release.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f62ee8a..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -image: s3lph/icalendar-timeseries-server-ci:20190820-01 - -stages: -- test -- build -- release -- upload - - - -before_script: -- export ITS_VERSION=$(python -c 'import icalendar_timeseries_server; print(icalendar_timeseries_server.__version__)') - - - -test: - stage: test - script: - - pip3 install -e . - - sudo -u its python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server - - sudo -u its python3 -m coverage combine - - sudo -u its python3 -m coverage report --rcfile=setup.cfg - -codestyle: - stage: test - script: - - pip3 install -e . - - sudo -u its pycodestyle icalendar_timeseries_server - - - -build_wheel: - stage: build - script: - - pip3 install -e . - - python3 setup.py egg_info bdist_wheel - - cd dist - - sha256sum *.whl > SHA256SUMS - artifacts: - paths: - - "dist/*.whl" - - dist/SHA256SUMS - only: - - tags - -build_debian: - stage: build - script: - # The Python package name provided by the python3-magic Debian package is "python-magic" rather than "file-magic". - - sed -re 's/file-magic/python-magic/' -i setup.py - - echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - | - cat > package/debian/icalendar-timeseries-server/DEBIAN/control <= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate - Description: Scrape iCalendar endpoints and present their data in a - timeseries format. A small service that scrapes iCalendar files - served over HTTP, parses their contents and returns a timeseries - format compatible to the /api/v1/query API endpoint of a Prometheus - server. This allows e.g. a Grafana administrator to add a Prometheus - data source pointing at this server, returning the events in the - calendars in the event metric. - EOF - - | - for version in $(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - done - - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - python3.7 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 - - cd package/debian - - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rsync -a icalendar-timeseries-server/usr/lib/python3.7/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rm -rf icalendar-timeseries-server/usr/lib/python3.7/ - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; - - mv icalendar-timeseries-server/usr/bin/icalendar-timeseries-server icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server - - rm -rf icalendar-timeseries-server/usr/bin - - sed -re 's$#!/usr/local/bin/python3.7$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server - - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; - - find icalendar-timeseries-server -type d -exec chmod 755 {} \; - - find icalendar-timeseries-server -type f -name .gitkeep -delete - - chmod +x icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server icalendar-timeseries-server/DEBIAN/postinst icalendar-timeseries-server/DEBIAN/prerm icalendar-timeseries-server/DEBIAN/postrm - - dpkg-deb --build icalendar-timeseries-server - - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" - - sudo -u nobody lintian "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" - - sha256sum *.deb > SHA256SUMS - artifacts: - paths: - - "package/debian/*.deb" - - package/debian/SHA256SUMS - only: - - tags - - - -release: - stage: release - script: - - python package/release.py - only: - - tags - - - -repo: - stage: upload - trigger: s3lph/custom-packages - variables: - MULTIPROJECT_TRIGGER_JOBNAME: icalendar-timeseries-server - only: - - tags diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..7039b14 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,95 @@ +--- + +steps: + + test: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server + + codestyle: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - pycodestyle icalendar_timeseries_server + + coverage: + image: python:3.11-bookworm + group: integration + commands: + - pip3 install -e .[test] + - python3 -m coverage combine + - python3 -m coverage report --rcfile=setup.cfg + + build_wheel: + image: python:3.11-bookworm + group: package + when: + - event: tag + secrets: + - GITEA_API_REPOSITORY_PYPI + - GITEA_API_USERNAME + - GITEA_API_PASSWORD + commands: + - pip3 install -e .[test] + - python3 setup.py egg_info bdist_wheel + - | + cat > ~/.pypirc <" | grep -B 1000 "<"'!'"-- END CHANGES $${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' + echo -e "\n -- s3lph $(date -R)\n" + done) > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - python3 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 + - cd package/debian + - sed -re "s/__ITS_VERSION__/$${ITS_VERSION}/g" -i icalendar-timeseries-server/DEBIAN/control + - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rsync -a icalendar-timeseries-server/usr/lib/python3.11/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rm -rf icalendar-timeseries-server/usr/lib/python3.11/site-packages + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; + - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/bin/icalendar-timeseries-server + - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; + - find icalendar-timeseries-server -type d -exec chmod 755 {} \; + - >- + chmod +x + icalendar-timeseries-server/usr/bin/icalendar-timeseries-server + icalendar-timeseries-server/DEBIAN/postinst + icalendar-timeseries-server/DEBIAN/prerm + icalendar-timeseries-server/DEBIAN/postrm + - dpkg-deb --build icalendar-timeseries-server + - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" + - sudo -u nobody lintian "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" || true + - >- + curl + --user "$${GITEA_API_USERNAME}:$${GITEA_API_PASSWORD}" + --upload-file "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" + $${GITEA_API_REPOSITORY_DEB} diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/control b/package/debian/icalendar-timeseries-server/DEBIAN/control new file mode 100644 index 0000000..c47eafc --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/control @@ -0,0 +1,14 @@ +Package: icalendar-timeseries-server +Version: __ITS_VERSION__ +Maintainer: s3lph +Section: web +Priority: optional +Architecture: all +Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate +Description: Scrape iCalendar endpoints and present their data in a timeseries format. + A small service that scrapes iCalendar files served over HTTP, parses + their contents and returns a timeseries format compatible to the + /api/v1/query API endpoint of a Prometheus server. This allows e.g. a + Grafana administrator to add a Prometheus data source pointing at + this server, returning the events in the calendars in the event + metric. diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/postinst b/package/debian/icalendar-timeseries-server/DEBIAN/postinst index be22d72..c132aa8 100755 --- a/package/debian/icalendar-timeseries-server/DEBIAN/postinst +++ b/package/debian/icalendar-timeseries-server/DEBIAN/postinst @@ -15,11 +15,7 @@ if [[ "$1" == "configure" ]]; then chown its:its /var/lib/its chmod 0750 /var/lib/its - systemctl daemon-reload || true - - if [[ "$2" != "" ]]; then - # Restart after upgrading the package - systemctl restart icalendar-timeseries-server.service - fi + deb-systemd-helper enable icalendar-timeseries-server.service + deb-systemd-invoke restart icalendar-timeseries-server.service fi diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/prerm b/package/debian/icalendar-timeseries-server/DEBIAN/prerm index 42c7b34..1191729 100755 --- a/package/debian/icalendar-timeseries-server/DEBIAN/prerm +++ b/package/debian/icalendar-timeseries-server/DEBIAN/prerm @@ -4,6 +4,6 @@ set -e if [[ "$1" == "remove" ]]; then - userdel its + deb-systemd-invoke stop icalendar-timeseries-server.service fi diff --git a/package/release.py b/package/release.py deleted file mode 100755 index 431c569..0000000 --- a/package/release.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 - -from typing import Any, Dict, List, Optional, Tuple - -import os -import sys -import json -import urllib.request -import http.client -from urllib.error import HTTPError - - -def parse_changelog(tag: str) -> Optional[str]: - release_changelog: str = '' - with open('CHANGELOG.md', 'r') as f: - in_target: bool = False - done: bool = False - for line in f.readlines(): - if in_target: - if f'' in line: - done = True - break - release_changelog += line - elif f'' in line: - in_target = True - continue - if not done: - return None - return release_changelog - - -def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: - url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'User-Agent': 'curl/7.70.0' - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.read() - joblist: List[Dict[str, Any]] = json.loads(resp_data.decode()) - - jobidmap: Dict[str, str] = {} - for job in joblist: - name: str = job['name'] - job_id: str = job['id'] - jobidmap[name] = job_id - return jobidmap - - -def fetch_single_shafile(url: str) -> str: - headers: Dict[str, str] = { - 'User-Agent': 'curl/7.70.0' - } - req = urllib.request.Request(url, headers=headers) - try: - resp: http.client.HTTPResponse = urllib.request.urlopen(req) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - resp_data: bytes = resp.readline() - shafile: str = resp_data.decode() - filename: str = shafile.strip().split(' ')[-1].strip() - return filename - - -def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' - wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' - wheel_filename: str = fetch_single_shafile(wheel_sha_url) - wheel_url: str = f'{mybase}/dist/{wheel_filename}' - return wheel_url, wheel_sha_url - - -def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: - mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' - debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' - debian_filename: str = fetch_single_shafile(debian_sha_url) - debian_url: str = f'{mybase}/package/debian/{debian_filename}' - return debian_url, debian_sha_url - - -def main(): - api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN') - release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG') - project_name: Optional[str] = os.getenv('CI_PROJECT_PATH') - project_id: Optional[str] = os.getenv('CI_PROJECT_ID') - pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID') - if api_token is None: - print('GITLAB_API_TOKEN is not set.', file=sys.stderr) - sys.exit(1) - if release_tag is None: - print('CI_COMMIT_TAG is not set.', file=sys.stderr) - sys.exit(1) - if project_name is None: - print('CI_PROJECT_PATH is not set.', file=sys.stderr) - sys.exit(1) - if project_id is None: - print('CI_PROJECT_ID is not set.', file=sys.stderr) - sys.exit(1) - if pipeline_id is None: - print('CI_PIPELINE_ID is not set.', file=sys.stderr) - sys.exit(1) - - changelog: Optional[str] = parse_changelog(release_tag) - if changelog is None: - print('Changelog could not be parsed.', file=sys.stderr) - sys.exit(1) - - job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token) - - base_url: str = f'https://gitlab.com/{project_name}/-' - - wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) - debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) - - augmented_changelog = f'''{changelog.strip()} - -### Download - -- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) -- [Debian Package]({debian_url}) ([sha256]({debian_sha_url})) -- Docker image: registry.gitlab.com/{project_name}:{release_tag}''' - - post_body: str = json.dumps({ - 'tag_name': release_tag, - 'description': augmented_changelog, - 'assets': { - 'links': [ - { - 'name': 'Python Wheel', - 'url': wheel_url, - 'link_type': 'package' - }, - { - 'name': 'Debian Package', - 'url': debian_url, - 'link_type': 'package' - } - ] - } - }) - - gitlab_release_api_url: str = \ - f'https://gitlab.com/api/v4/projects/{project_id}/releases' - headers: Dict[str, str] = { - 'Private-Token': api_token, - 'Content-Type': 'application/json; charset=utf-8', - 'User-Agent': 'curl/7.70.0' - } - - request = urllib.request.Request( - gitlab_release_api_url, - post_body.encode('utf-8'), - headers=headers, - method='POST' - ) - try: - response: http.client.HTTPResponse = urllib.request.urlopen(request) - except HTTPError as e: - print(e.read().decode()) - sys.exit(1) - response_bytes: bytes = response.read() - response_str: str = response_bytes.decode() - response_data: Dict[str, Any] = json.loads(response_str) - - if response_data['tag_name'] != release_tag: - print('Something went wrong...', file=sys.stderr) - print(response_str, file=sys.stderr) - sys.exit(1) - - print(response_data['description']) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index dfb999b..292e5c8 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( description='', license='MIT', keywords='ical,icalendar,timeseries,prometheus,grafana', - url='https://gitlab.com/s3lph/icalendar-timeseries-server', + url='https://git.kabelsalat.ch/s3lph/icalendar-timeseries-server', packages=find_packages(exclude=['*.test']), long_description='', python_requires='>=3.6', @@ -23,6 +23,14 @@ setup( 'isodate', 'jinja2' ], + extras_require={ + 'test': [ + 'coverage', + 'pycodestyle', + 'mypy', + 'twine' + ] + }, entry_points={ 'console_scripts': [ 'icalendar-timeseries-server = icalendar_timeseries_server:main' -- 2.39.5 From 5f8653797fbf6e4b013e15275ad641d295424dfe Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 12 Aug 2023 14:09:06 +0200 Subject: [PATCH 27/28] chore: bump version number --- CHANGELOG.md | 13 +++++++++++++ icalendar_timeseries_server/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a9830..910bb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # iCalendar Timeseries Server Changelog + +## Version 0.6.2 + +### Changes + + +- Migration from Gitlab-CI to Woodpecker + + + + + + ## Version 0.6.1 diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 7a2d0cd..6808c70 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.6.1' +__version__ = '0.6.2' -- 2.39.5 From 5e52dffc27e56178e176e1672621c6bf05cc1810 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Dec 2023 05:25:00 +0100 Subject: [PATCH 28/28] feat: migrate from woodpecker to forgejo actions --- .forgejo/workflows/package.yml | 38 ++++++++ .forgejo/workflows/test.yml | 27 ++++++ .woodpecker.yml | 95 ------------------- CHANGELOG.md | 12 ++- icalendar_timeseries_server/__init__.py | 2 +- .../DEBIAN/control | 2 +- 6 files changed, 78 insertions(+), 98 deletions(-) create mode 100644 .forgejo/workflows/package.yml create mode 100644 .forgejo/workflows/test.yml diff --git a/.forgejo/workflows/package.yml b/.forgejo/workflows/package.yml new file mode 100644 index 0000000..63af42a --- /dev/null +++ b/.forgejo/workflows/package.yml @@ -0,0 +1,38 @@ +--- + +on: + push: + tags: + - "v*" + +jobs: + + build_wheel: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - name: Build Python wheel + run: | + apt update; apt install -y python3-pip + pip3 install --break-system-packages -e .[test] + python3 setup.py egg_info bdist_wheel + - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3 + with: + username: ${{ secrets.API_USERNAME }} + password: ${{ secrets.API_PASSWORD }} + + build_debian: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5 + with: + python_module: icalendar_timeseries_server + package_name: icalendar-timeseries-server + package_root: package/debian/icalendar-timeseries-server + package_output_path: package/debian + - uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2 + with: + username: ${{ secrets.API_USERNAME }} + password: ${{ secrets.API_PASSWORD }} + deb: "package/debian/*.deb" diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml new file mode 100644 index 0000000..238f155 --- /dev/null +++ b/.forgejo/workflows/test.yml @@ -0,0 +1,27 @@ +--- + +on: push + +jobs: + + test: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - name: test + run: | + apt update; apt install --yes python3-pip + pip3 install --break-system-packages -e .[test] + python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server + python3 -m coverage combine + python3 -m coverage report --rcfile=setup.cfg + + codestyle: + runs-on: docker + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - name: codestyle + run: | + apt update; apt install --yes python3-pip + pip3 install --break-system-packages -e .[test] + pycodestyle icalendar_timeseries_server diff --git a/.woodpecker.yml b/.woodpecker.yml index 7039b14..e69de29 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,95 +0,0 @@ ---- - -steps: - - test: - image: python:3.11-bookworm - group: test - commands: - - pip3 install -e .[test] - - python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server - - codestyle: - image: python:3.11-bookworm - group: test - commands: - - pip3 install -e .[test] - - pycodestyle icalendar_timeseries_server - - coverage: - image: python:3.11-bookworm - group: integration - commands: - - pip3 install -e .[test] - - python3 -m coverage combine - - python3 -m coverage report --rcfile=setup.cfg - - build_wheel: - image: python:3.11-bookworm - group: package - when: - - event: tag - secrets: - - GITEA_API_REPOSITORY_PYPI - - GITEA_API_USERNAME - - GITEA_API_PASSWORD - commands: - - pip3 install -e .[test] - - python3 setup.py egg_info bdist_wheel - - | - cat > ~/.pypirc <" | grep -B 1000 "<"'!'"-- END CHANGES $${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' - echo -e "\n -- s3lph $(date -R)\n" - done) > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog - - python3 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 - - cd package/debian - - sed -re "s/__ITS_VERSION__/$${ITS_VERSION}/g" -i icalendar-timeseries-server/DEBIAN/control - - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rsync -a icalendar-timeseries-server/usr/lib/python3.11/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ - - rm -rf icalendar-timeseries-server/usr/lib/python3.11/site-packages - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; - - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyo' -exec rm {} \; - - sed -re 's$#!/usr/local/bin/python3$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/bin/icalendar-timeseries-server - - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; - - find icalendar-timeseries-server -type d -exec chmod 755 {} \; - - >- - chmod +x - icalendar-timeseries-server/usr/bin/icalendar-timeseries-server - icalendar-timeseries-server/DEBIAN/postinst - icalendar-timeseries-server/DEBIAN/prerm - icalendar-timeseries-server/DEBIAN/postrm - - dpkg-deb --build icalendar-timeseries-server - - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" - - sudo -u nobody lintian "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" || true - - >- - curl - --user "$${GITEA_API_USERNAME}:$${GITEA_API_PASSWORD}" - --upload-file "icalendar-timeseries-server_$${ITS_VERSION}-1_all.deb" - $${GITEA_API_REPOSITORY_DEB} diff --git a/CHANGELOG.md b/CHANGELOG.md index 910bb28..f33a665 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ # iCalendar Timeseries Server Changelog + +## Version 0.6.3 + +### Changes + + +- Migration from Woodpecker to Forgejo Actions. + + + + ## Version 0.6.2 @@ -8,7 +19,6 @@ - Migration from Gitlab-CI to Woodpecker - diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 6808c70..6df2f49 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.6.2' +__version__ = '0.6.3' diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/control b/package/debian/icalendar-timeseries-server/DEBIAN/control index c47eafc..18b9a7d 100644 --- a/package/debian/icalendar-timeseries-server/DEBIAN/control +++ b/package/debian/icalendar-timeseries-server/DEBIAN/control @@ -1,5 +1,5 @@ Package: icalendar-timeseries-server -Version: __ITS_VERSION__ +Version: __VERSION__ Maintainer: s3lph Section: web Priority: optional -- 2.39.5