commit 5722fd7c98818d1cab5a954d32040e7e9f9f13ee Author: s3lph Date: Tue Aug 20 00:24:51 2019 +0200 Rename project to icalendar-timeseries-server diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a01865 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +**/.idea/ +*.iml + +**/__pycache__/ +*.pyc +**/*.egg-info/ +*.coverage +**/.mypy_cache/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..1904415 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,91 @@ +--- +image: s3lph/icalendar-timeseries-server-ci:20190820-01 + +stages: +- test +- build +- release + + + +before_script: +- export ITS_VERSION=$(python -c 'import icalendar_timeseries_server; print(icalendar_timeseries_server.__version__)') + + + +test: + stage: test + script: + - pip3 install -e . + - sudo -u its python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server + - sudo -u its python3 -m coverage combine + - sudo -u its python3 -m coverage report --rcfile=setup.cfg + +codestyle: + stage: test + script: + - pip3 install -e . + - sudo -u its pycodestyle icalendar_timeseries_server + + + +build_wheel: + stage: build + script: + - pip3 install -e . + - python3 setup.py egg_info bdist_wheel + - cd dist + - sha256sum *.whl > SHA256SUMS + artifacts: + paths: + - "dist/*.whl" + - dist/SHA256SUMS + only: + - tags + +build_debian: + stage: build + script: + # The Python package name provided by the python3-magic Debian package is "python-magic" rather than "file-magic". + - sed -re 's/file-magic/python-magic/' -i setup.py + - echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - | + for version in "$(cat CHANGELOG.md | grep '" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + done + - gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog + - python3.7 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1 + - cd package/debian + - mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rsync -a icalendar-timeseries-server/usr/lib/python3.7/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/ + - rm -rf icalendar-timeseries-server/usr/lib/python3.7/ + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - mv icalendar-timeseries-server/usr/bin/icalendar-timeseries-server icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server + - rm -rf icalendar-timeseries-server/usr/bin + - sed -re 's$#!/usr/local/bin/python3.7$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server + - find icalendar-timeseries-server -type f -exec chmod 0644 {} \; + - find icalendar-timeseries-server -type d -exec chmod 755 {} \; + - find icalendar-timeseries-server -type f -name .gitkeep -delete + - chmod +x icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server icalendar-timeseries-server/DEBIAN/postinst icalendar-timeseries-server/DEBIAN/prerm icalendar-timeseries-server/DEBIAN/postrm + - dpkg-deb --build icalendar-timeseries-server + - mv icalendar-timeseries-server.deb "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" + - sudo -u nobody lintian "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb" + - sha256sum *.deb > SHA256SUMS + artifacts: + paths: + - "package/debian/*.deb" + - package/debian/SHA256SUMS + only: + - tags + + + +release: + stage: release + script: + - python package/release.py + only: + - tags diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4b14593 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# iCalendar Timeseries Server Changelog + + +## Version 0.1 + +First pre-1.0 release of iCalendar Timeseries Server. + +### Changes + + +- Configure multiple scrape sources +- Authorization: HTTP Basic Auth or TLS Client Certificates +- Offer API endpoints /api/v1/query and /api/v1/query_range +- Support simple PromQL label filters + + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3dea343 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.7-stretch + +RUN useradd -d /home/its -m its \ + && apt-get update -qy \ + && apt-get install -y --no-install-recommends sudo build-essential lintian rsync \ + && python3.7 -m pip install coverage wheel pycodestyle mypy + +WORKDIR /home/its \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8bb43fb --- /dev/null +++ b/README.md @@ -0,0 +1,176 @@ +# iCalendar Timeseries Server + +This project is a small service that scrapes iCalendar files served +over HTTP, parses their contents and returns the data in a timeseries +format compatible to the `/api/v1/query` API endpoint of a Prometheus +server. This allows e.g. a Grafana administrator to add a Prometheus +data source pointing at this server, returning the events in the +calendars in the `event` metric. + +## Example + +Consider the following iCalendar file: + +``` +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//ACME//NONSGML Rocket Powered Anvil//EN +BEGIN:VEVENT +UID:20190603T032500CEST-foo +SUMMARY:Foo +DESCRIPTION:An example event +DTSTART;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T032500 +DTEND;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T040000 +END:VEVENT +BEGIN:VEVENT +UID:20190603T032500CEST-bar +SUMMARY:Bar +DESCRIPTION:Another example event +DTSTART;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T032500 +DTEND;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T040000 +END:VEVENT +END:VCALENDAR +``` + +The server would transform this into the following API response: + +```json +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "event", + "calendar": "0", + "uid": "20190603T032500CEST-foo", + "summary": "Foo", + "description": "An example event" + }, + "value": [ + 1560043497, + 1 + ] + }, + { + "metric": { + "__name__": "event", + "calendar": "1", + "uid": "20190603T032500CEST-bar", + "summary": "Bar", + "description": "Another example event" + }, + "value": [ + 1560043497, + 1 + ] + } + ] + } +} +``` + +## Dependencies + +- `bottle`: Serve the results +- `dateutil`: Work with recurring events +- `icalendar`: Parse iCalendar +- `isodate`: Parse ISO-8601 time periods +- `jinja2`: Template value replacements +- `pytz`: Work with timezones + +## Configuration + +Configuration is done through a JSON config file: + +### Example + +```json +{ + "addr": "127.0.0.1", + "port": 8090, + "start_delta": "-PT3H", + "end_delta": "P30D", + "cache": "PT15M", + "tz": "Europe/Zurich", + "calendars": { + "private": { + "url": "https://example.cloud/dav/me/private.ics", + "auth": { + "type": "basic", + "username": "me", + "password": "mysupersecurepassword" + } + }, + "public": { + "url": "https://example.cloud/dav/me/public.ics" + }, + "confidential": { + "url": "https://example.cloud/dav/me/confidential.ics", + "ca": "/etc/ssl/ca.pem", + "auth": { + "type": "tls", + "keyfile": "/etc/ssl/client.pem", + "passphrase": "mysupersecurepassword" + } + } + }, + "key_replace": { + "summary": "01_summary", + "description": "02_description" + }, + "value_replace": { + "summary": "{{ summary|truncate(100) }}", + "description": "{{ description|truncate(100) }}" + } +} +``` + +### Explanation + + +| JMESPath | Type | Description | +|----------|------|-------------| +| `addr` | string | The address to listen on. | +| `port` | int | The port to listen on. | +| `start_delta` | string | A signed ISO 8601 duration string, describing the event range start offset relative to the current time. | +| `end_delta` | string | An unsigned ISO 8601 duration string, describing the event range end offset relative to the current time. | +| `cache` | string | An unsigned ISO 8601 duration string, describing the cache timeout duration. | +| `tz` | string | The local timezone. | +| `calendars` | dict | The calendars to scrape. | +| `keys(calendars)` | string | Name of the calendar. | +| `calendars.*.url` | string | The HTTP or HTTPS URL to scrape. | +| `calendars.*.ca` | string | Path to the CA certificate file to validate the server's TLS certificate against, in PEM format (optional). | +| `calendars.*.auth` | dict | Authorization config for the calendar. | +| `calendars.*.auth[].type` | string | Authorization type, one of `none` (no authorization), `basic` (HTTP Basic Authentication), `tls` (TLS client certificate). | +| `calendars.*.auth[?type=='basic'].username` | string | The Basic Auth username to authenticate with. | +| `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. | +| `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. | +| `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). | +| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. | +| `keys(key_replace)` | string | The labels to rename. | +| `key_replace.*` | string | The names to rename the labels to. | +| `value_replace` | dict | Label values to postprocess. | +| `keys(value_replace)` | string | Original label name to postprocess, also may introduce new labels. | +| `value_replace.*` | string | The new value for the label. Supports Jinja2 templates. All original label values can be accessed by using their original name as a Jinja2 variable name. | + +## Queries + +The most basic query is simply the metric name: + +``` +event +``` + +In addition, PromQL label filters can be used. + +``` +event{calendar="public",foo=~".*"} +``` + +## Why Prometheus API + +- It's JSON. A JSON generator is builtin in Python, so no further dependency. +- The Prometheus Data Source is builtin into Grafana. +- The API is simple. \ No newline at end of file diff --git a/icalendar-timeseries-server.json b/icalendar-timeseries-server.json new file mode 100644 index 0000000..d24809c --- /dev/null +++ b/icalendar-timeseries-server.json @@ -0,0 +1,29 @@ +{ + "addr": "127.0.0.1", + "port": 8090, + "start_delta": "-PT3H", + "end_delta": "P60D", + "cache": "PT3M", + "tz": "Europe/Zurich", + "calendars": { + "tlstest": { + "url": "https://localhost/private.ics", + "ca": "/home/sebastian/tlstest/ca/ca/ca.crt", + "auth": { + "type": "tls", + "keyfile": "/home/sebastian/tlstest/client/combined.pem" + } + } + }, + "key_replace": { + "summary": "a_summary", + "description": "b_description", + "calendar": "c_calendar" + }, + "value_replace": { + "summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", + "description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", + "calendar": "{{ 0 if calendar == 'private' else 1 }}", + "useless_metric": "{{ start.timestamp() + end.timestamp() }}" + } +} diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py new file mode 100644 index 0000000..373d726 --- /dev/null +++ b/icalendar_timeseries_server/__init__.py @@ -0,0 +1,2 @@ + +__version__ = '0.1' diff --git a/icalendar_timeseries_server/__main__.py b/icalendar_timeseries_server/__main__.py new file mode 100644 index 0000000..80e5dd3 --- /dev/null +++ b/icalendar_timeseries_server/__main__.py @@ -0,0 +1,4 @@ + +from icalendar_timeseries_server.main import main + +main() diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py new file mode 100644 index 0000000..0132574 --- /dev/null +++ b/icalendar_timeseries_server/api.py @@ -0,0 +1,71 @@ +from typing import List + +import json +from datetime import datetime +from urllib.error import HTTPError +import traceback + +import bottle +from isodate import Duration + +from icalendar_timeseries_server.config import get_config +from icalendar_timeseries_server.event import Event +from icalendar_timeseries_server.cal import scrape_calendar +from icalendar_timeseries_server.query import MetricQuery + + +@bottle.route('/api/v1/query') +@bottle.route('/api/v1/query_range') +def prometheus_api(): + tz = get_config().tz + now: datetime = datetime.now(tz) + start_delta: Duration = get_config().start_delta + end_delta: Duration = get_config().end_delta + start: datetime = now + start_delta + end: datetime = now + end_delta + events: List[Event] = [] + + try: + q = MetricQuery(bottle.request.query['query']) + except ValueError as e: + response = { + 'status': 'error', + 'errorType': 'bad_data', + 'error': str(e) + } + bottle.response.status = 400 + traceback.print_exc() + bottle.response.add_header('Content-Type', 'application/json') + return json.dumps(response) + + try: + for name, caldef in get_config().calendars.items(): + events.extend(scrape_calendar(name, caldef, start, end)) + events = list(filter(q, events)) + events.sort(key=lambda e: e.start) + response = { + 'status': 'success', + 'data': { + 'resultType': 'vector', + 'result': [e.serialize() for e in events] + } + } + except HTTPError as e: + response = { + 'status': 'error', + 'errorType': 'internal', + 'error': str(e) + } + bottle.response.status = 500 + traceback.print_exc() + except BaseException: + response = { + 'status': 'error', + 'errorType': 'internal', + 'error': 'An internal error occurred.' + } + bottle.response.status = 500 + traceback.print_exc() + + bottle.response.add_header('Content-Type', 'application/json') + return json.dumps(response) diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py new file mode 100644 index 0000000..c17dd77 --- /dev/null +++ b/icalendar_timeseries_server/cal.py @@ -0,0 +1,92 @@ +from typing import Dict, List, Iterable, Tuple + +import sys +import urllib.request +from datetime import datetime, date, timedelta + +from dateutil import rrule +from icalendar import cal + +from icalendar_timeseries_server import __version__ +from icalendar_timeseries_server.config import get_config, CalendarConfig +from icalendar_timeseries_server.event import Event + + +_SCRAPE_CACHE: Dict[str, Tuple[datetime, List[Event]]] = dict() + +__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}' +USER_AGENT: str = f'icalendar-timeseries-server/{__version__} (Python/{__py_version})' + + +def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration: timedelta) -> List[datetime]: + occurences: List[datetime] = [] + + evstart = event.get('dtstart').dt + # First occurence lies in the future; no need to process further + if evstart >= end: + return occurences + + # Extract recurrence rules from ical + ical_lines = event.to_ical().decode('utf-8').split('\r\n') + recurrence = '\n'.join( + [x for x in ical_lines + if x.startswith('RRULE') or x.startswith('RDATE') or x.startswith('EXRULE') or x.startswith('EXDATE')]) + # Create a generator that yields a timestamp for each recurrence + generator = rrule.rrulestr(recurrence, dtstart=evstart) + + # Generate an event entry for each occurence of the event + for dt in generator: + # Skip past occurences and break once the the event lies too far in the future + if dt + duration < start: + continue + if dt > end: + break + # Create an event entry + occurences.append(dt) + return occurences + + +def _parse_calendar(name: str, calendar: cal.Calendar, start: datetime, end: datetime) -> List[Event]: + events = [] + for element in calendar.walk(): + if element.name == "VEVENT": + dtstart = element.get('dtstart').dt + if isinstance(dtstart, date): + dtstart = datetime(dtstart.year, dtstart.month, dtstart.day, tzinfo=start.tzinfo) + # Process either end timestamp or duration, if present + if 'dtend' in element: + evend = element.get('dtend').dt + if isinstance(evend, date): + evend = datetime(evend.year, evend.month, evend.day, tzinfo=start.tzinfo) + duration = evend - dtstart + elif 'duration' in element: + duration = element.get('duration').dt + else: + duration = timedelta(0) + if element.get('rrule') is not None or element.get('rdate') is not None: + occurences: Iterable[datetime] = _parse_recurring(element, start, end, duration) + else: + occurences = [dtstart] + for occurence in occurences: + if start <= occurence < end: + events.append(Event(name, element, occurence, occurence + duration)) + return events + + +def scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime) -> List[Event]: + global _SCRAPE_CACHE + now: datetime = datetime.now(tz=get_config().tz) + if get_config().cache.total_seconds() > 0 and name in _SCRAPE_CACHE: + cache_timeout, cached = _SCRAPE_CACHE[name] + if now < cache_timeout: + print('serving cached') + return cached + print('doing request') + + opener: urllib.request.OpenerDirector = config.get_url_opener() + with opener.open(config.url) as response: + data = response.read().decode('utf-8') + calendar = cal.Calendar.from_ical(data) + parsed: List[Event] = _parse_calendar(name, calendar, start, end) + _SCRAPE_CACHE[name] = now + get_config().cache, parsed + return parsed diff --git a/icalendar_timeseries_server/config.py b/icalendar_timeseries_server/config.py new file mode 100644 index 0000000..5ce34ea --- /dev/null +++ b/icalendar_timeseries_server/config.py @@ -0,0 +1,228 @@ +from typing import Any, Dict, List, Optional, Type, Union + +import base64 +import json +from datetime import timedelta +import ssl +import urllib.request +import sys + +import pytz +import jinja2 +from isodate import Duration, parse_duration + +from icalendar_timeseries_server import __version__ + + +CONFIG: Optional['Config'] = None + +JENV: Optional[jinja2.Environment] = None + + +__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}' +USER_AGENT: str = f'icalendar-timeseries-server/{__version__} (Python/{__py_version})' + + +class CalendarConfig: + + def __init__(self, config: Dict[str, Any], config_path: str) -> None: + self._url: str = _keycheck('url', config, str, config_path) + self._ca: Optional[str] = _keycheck('ca', config, str, config_path, optional=True) + auth: Dict[str, Any] = _keycheck('auth', config, dict, config_path, default_value={'type': 'none'}) + self._authtype: str = _keycheck('type', auth, str, f'{config_path}.auth', + default_value='none', + valid_values=['none', 'basic', 'tls']) + self._request_headers: Dict[str, str] = { + 'User-Agent': USER_AGENT + } + + if self._authtype == 'none': + # No auth, nothing to do + pass + + elif self._authtype == 'basic': + # Build the Authorization header required for HTTP Basic Auth + username: str = _keycheck('username', auth, str, f'{config_path}.auth') + password: str = _keycheck('password', auth, str, f'{config_path}.auth') + token: str = base64.b64encode(f'{username}:{password}'.encode()).decode() + self._request_headers['Authorization'] = f'Basic {token}' + + elif self._authtype == 'tls': + # Store TLS config for later use + self._tls_keyfile: str = _keycheck('keyfile', auth, str, f'{config_path}.auth') + self._tls_passphrase: str = _keycheck('passphrase', auth, str, f'{config_path}.auth', optional=True) + + @property + def url(self) -> str: + return self._url + + def get_url_opener(self) -> urllib.request.OpenerDirector: + + if self._authtype == 'tls': + # Create an OpenSSL context and load the CA and client certificates + context: ssl.SSLContext = ssl.create_default_context(cafile=self._ca) + context.load_cert_chain(certfile=self._tls_keyfile, password=self._tls_passphrase) + opener: urllib.request.OpenerDirector = \ + urllib.request.build_opener(urllib.request.HTTPSHandler(context=context)) + + elif self._ca is not None: + # No TLS Client Certificate, but load the CA certificate + context: ssl.SSLContext = ssl.create_default_context(cafile=self._ca) + opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=context)) + + else: + # No TLS config, return default opener + opener = urllib.request.build_opener() + + # Set request headers, at least User-Agent, optionally Authorization + opener.addheaders = [(k, v) for k, v in self._request_headers.items()] + return opener + + +class Config: + + def __init__(self, config: Dict[str, Any] = None) -> None: + if config is None: + config = dict() + self._addr: str = _keycheck('addr', config, str, '', default_value='127.0.0.1') + self._port: int = _keycheck('port', config, int, '', default_value=8090) + self._tz: pytz.tzinfo = _parse_timezone('tz', config, '', default_value='UTC') + self._start_delta: Duration = _parse_timedelta('start_delta', config, '', default_value='PT') + self._end_delta: Duration = _parse_timedelta('end_delta', config, '', default_value='P30D') + self._cache: Duration = _parse_timedelta('cache', config, '', default_value='PT', force_positive=True) + self._calendars: Dict[str, CalendarConfig] = self._parse_calendars_config('calendars', config, '') + self._key_replace = _parse_key_replace('key_replace', config, '') + self._value_replace = _parse_value_replace('value_replace', config, '') + + @staticmethod + def _parse_calendars_config(key: str, + config: Dict[str, Any], + path: str) -> Dict[str, CalendarConfig]: + cdef: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={}) + calendars: Dict[str, CalendarConfig] = dict() + for name, c in cdef.items(): + calendar: CalendarConfig = CalendarConfig(c, f'{path}.{name}') + calendars[name] = calendar + return calendars + + @property + def addr(self) -> str: + return self._addr + + @property + def port(self) -> int: + return self._port + + @property + def tz(self) -> pytz.tzinfo: + return self._tz + + @property + def start_delta(self) -> Duration: + return self._start_delta + + @property + def end_delta(self) -> Duration: + return self._end_delta + + @property + def cache(self) -> Duration: + return self._cache + + @property + def calendars(self) -> Dict[str, CalendarConfig]: + return self._calendars + + @property + def key_replace(self) -> Dict[str, str]: + return self._key_replace + + @property + def value_replace(self) -> Dict[str, str]: + return self._value_replace + + +def _keycheck(key: str, + config: Dict[str, Any], + typ: Type, + path: str, + default_value: Any = None, + optional: bool = False, + valid_values: Optional[List[Any]] = None) -> Any: + if key not in config: + if default_value is not None or optional: + return default_value + raise KeyError(f'Expected member "{key}" not found at path {path}') + value: Any = config[key] + if not isinstance(value, typ): + raise TypeError(f'Expected {typ}, not {type(value).__name__} for path {path}.{key}') + if valid_values is not None: + if value not in valid_values: + raise ValueError(f'Expected one of {", ".join(valid_values)} ({typ}), not {value} for path {path}.{key}') + return value + + +def _parse_timedelta(key: str, + config: Dict[str, Any], + path: str, + default_value: Any = None, + force_positive: bool = False) -> Duration: + tdstring: str = _keycheck(key, config, str, path, default_value=default_value) + duration: Union[Duration, timedelta] = parse_duration(tdstring) + if isinstance(duration, timedelta): + duration = Duration(days=duration.days, + seconds=duration.seconds, + microseconds=duration.microseconds) + if force_positive and duration.total_seconds() < 0: + raise ValueError(f'Duration must be positive for path {path}.{key}') + return duration + + +def _parse_timezone(key: str, + config: Dict[str, Any], + path: str, + default_value: Any = None) -> Any: + zonename: str = _keycheck(key, config, str, path, default_value=default_value) + return pytz.timezone(zonename) + + +def _parse_key_replace(key: str, + config: Dict[str, Any], + path: str) -> Dict[str, str]: + key_replace: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={}) + for label in key_replace.keys(): + _keycheck(label, key_replace, str, f'{path}.{key}') + return key_replace + + +def _parse_value_replace(key: str, + config: Dict[str, Any], + path: str) -> Dict[str, str]: + value_replace: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={}) + for label in value_replace.keys(): + _keycheck(label, value_replace, str, f'{path}.{key}') + return value_replace + + +def get_config() -> Config: + global CONFIG + return CONFIG + + +def get_jenv() -> jinja2.Environment: + global JENV + return JENV + + +def load_config(filename: str): + global CONFIG, JENV + with open(filename, 'r') as f: + json_config = json.loads(f.read()) + CONFIG = Config(json_config) + JENV = jinja2.Environment() + + +def load_default_config(): + global CONFIG, JENV + CONFIG = Config({}) + JENV = jinja2.Environment() diff --git a/icalendar_timeseries_server/event.py b/icalendar_timeseries_server/event.py new file mode 100644 index 0000000..dff8a9e --- /dev/null +++ b/icalendar_timeseries_server/event.py @@ -0,0 +1,66 @@ +from typing import Any, Dict, List, Set + +import icalendar +import jinja2 +from datetime import datetime + +from icalendar_timeseries_server.config import get_config, get_jenv +from icalendar_timeseries_server.query import Metric + +_ATTRIBUTES: List[str] = [ + 'attachs', + 'categories', + 'class', + 'comment', + 'description', + 'geo', + 'location', + 'percent-complete', + 'priority', + 'resources', + 'status', + 'summary' +] + + +class Event(Metric): + + def __init__(self, cname: str, event: icalendar.cal.Event, start: datetime, end: datetime): + self.start: datetime = start + self.calendar: str = cname + # self.attributes: Dict[str, str] = dict() + attributes: Dict[str, str] = dict() + tmp: Dict[str, Any] = { + 'calendar': cname, + 'start': start, + 'end': end + } + for attr in _ATTRIBUTES: + tmp[attr] = event.get(attr, '') + substitution_keys = set(_ATTRIBUTES) + substitution_keys.update(['start', 'end']) + substitution_keys.update(get_config().key_replace.keys()) + substitution_keys.update(get_config().value_replace.keys()) + for attr in substitution_keys: + newkey: str = get_config().key_replace.get(attr, attr) + value: str = tmp.get(attr, '') + newval_template: str = get_config().value_replace.get(attr, str(value)) + jtemplate: jinja2.Template = get_jenv().from_string(newval_template) + newvalue: str = jtemplate.render(**tmp) + attributes[newkey] = newvalue + self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}' + super().__init__('event', attributes) + + def serialize(self) -> Dict[str, Any]: + event: Dict[str, Any] = { + 'metric': { + '__name__': 'event', + 'calendar': self.calendar + }, + 'value': [ + self.start.timestamp(), + 1 + ] + } + event['metric'].update(self._labels) + return event diff --git a/icalendar_timeseries_server/main.py b/icalendar_timeseries_server/main.py new file mode 100644 index 0000000..cf549cb --- /dev/null +++ b/icalendar_timeseries_server/main.py @@ -0,0 +1,24 @@ +import sys + +import bottle + +from icalendar_timeseries_server.config import load_config, load_default_config, get_config + +# Contains decorated bottle handler function for /api/v1/query +# noinspection PyUnresolvedReferences +from icalendar_timeseries_server.api import prometheus_api + + +def main(): + if len(sys.argv) == 1: + load_default_config() + elif len(sys.argv) == 2: + load_config(sys.argv[1]) + else: + print(f'Can only read one config file, got "{" ".join(sys.argv[1:])}"') + exit(1) + bottle.run(host=get_config().addr, port=get_config().port) + + +if __name__ == '__main__': + main() diff --git a/icalendar_timeseries_server/query.py b/icalendar_timeseries_server/query.py new file mode 100644 index 0000000..b266898 --- /dev/null +++ b/icalendar_timeseries_server/query.py @@ -0,0 +1,189 @@ +from typing import Dict + +import re + +LABEL_MATCH_OPERATORS = [ + '=', + '!=', + '=~', + '!~' +] + + +class Metric: + + def __init__(self, name: str, labels: Dict[str, str]): + self._name: str = name + self._labels: Dict[str, str] = dict() + for k, v in labels.items(): + self._labels[k] = v + + @property + def name(self) -> str: + return self._name + + def keys(self): + return self._labels.keys() + + def __getitem__(self, item: str) -> str: + return self._labels.get(item, None) + + def __getattr__(self, item: str) -> str: + return self._labels.get(item, None) + + +class LabelFilter: + + def __init__(self, op: str, label: str, value: str): + if op not in LABEL_MATCH_OPERATORS: + raise ValueError() + self._op = op + self._label = label + self._value = value + + def __call__(self, metric: Metric) -> bool: + if self._label not in metric.keys(): + return False + if self._op == '=': + return metric[self._label] == self._value + elif self._op == '!=': + return metric[self._label] != self._value + elif self._op == '=~': + return re.match(self._value, metric[self._label]) is not None + elif self._op == '!~': + return re.match(self._value, metric[self._label]) is None + + +class MetricQuery: + + def __init__(self, q: str) -> None: + self._filters = [] + # The query '1+1' is used by Grafana to test the connection. + if q == "1+1": + return + self.__parse(q) + + def __parse(self, q: str): + print(q) + # globalstate: + # 0 = parsing metric name + # 1 = parsing filters + # 2 = fully parsed query + globalstate = 0 + self._metric_name = '' + # buffer for the name of the filter that is being parsed + cfiltername = '' + # buffer for the filter operator that is being parsed + coperator = '' + # buffer for the filter value that is being parsed + cfiltervalue = '' + # filterstate: + # 0 = parsing label name + # 1 = parsing operator + # 2 = parsing filter value + filterstate = 0 + # Whether currently inside a quoted string + quoted = False + # Whether the last character started an escape sequence + escaping = False + # query string index + i = 0 + while i < len(q): + c = q[i] + i += 1 + if globalstate == 0: + # Parsing the metric name: append the character to the name buffer, + # or move on to the filter parsing step when "{" is encountered. + if c == '{': + globalstate = 1 + else: + self._metric_name += c + elif globalstate == 1: + # Parsing the query's filters, until an unescaped "}" is encountered. + if filterstate == 0: + # Parsing the filter's label name. + # When a character is not accepted, re-loop over it in the next mode. + if c in '_: 0123456789 ' or ('a' <= c <= 'z') or ('A' <= c <= 'Z'): + cfiltername += c + else: + filterstate = 1 + i -= 1 + elif filterstate == 1: + # Parsing the filter's operator, one of =, !=, =~, !~. + # When a character is not accepted, re-loop over it in the next mode. + if c in '!~= ': + coperator += c + else: + filterstate = 2 + i -= 1 + elif filterstate == 2: + # Parsing the filter's value. + # When a "," is encountered, start with the next filter. When a "}" is encountered, parsing is done. + if not quoted: + if c.strip() == '': + continue + if c == '"': + quoted = True + elif c == ',' or c == '}': + # Done parsing the filter. Validate label name and operator. + if coperator not in LABEL_MATCH_OPERATORS: + raise ValueError(f'Invalid operator: {coperator.strip()}') + if not re.match('^[a-zA-Z_:][a-zA-Z0-9_:]*$', cfiltername.strip()): + raise ValueError(f'Invalid label name: {cfiltername.strip()}') + self._filters.append(LabelFilter(coperator.strip(), cfiltername.strip(), cfiltervalue)) + # Reset filter buffers + cfiltername = '' + coperator = '' + cfiltervalue = '' + filterstate = 0 + if c == '}': + globalstate = 2 + else: + raise ValueError() + else: + # Quoting and escaping + if c == '"' and not escaping: + quoted = False + else: + if escaping: + escaping = False + cfiltervalue += c + else: + if c == '\\': + escaping = True + else: + cfiltervalue += c + else: + if c.strip() == '': + # Trim remaining whitespace + continue + else: + raise ValueError('Garbage after metric query') + # Match metric name against the regex from the Prometheus documentation + if not re.match('^[a-zA-Z_:][a-zA-Z0-9_:]*$', self._metric_name.strip()): + raise ValueError(f'Invalid metric name: {self._metric_name.strip()}') + if escaping: + raise ValueError('Expected escape sequence, got EOF') + elif quoted: + raise ValueError('Expected \'\"\', got EOF') + elif globalstate == 1: + raise ValueError('Expected \'}\', got EOF') + elif filterstate != 0: + raise ValueError('Unexpected EOF') + + def __call__(self, metric: Metric): + """ + Applies the filter deducted from the query string to the given metric. + + :param metric: The metric to apply the filter to. + :return: True if the filter matches, False otherwise. + """ + # Never match a metric with a different name + if metric.name != self._metric_name: + return False + # Iterate the single filters, return False if even one does not match + for f in self._filters: + if not f(metric): + return False + # Return True if all filters matched + return True diff --git a/icalendar_timeseries_server/test/__init__.py b/icalendar_timeseries_server/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/icalendar_timeseries_server/test/test_config.py b/icalendar_timeseries_server/test/test_config.py new file mode 100644 index 0000000..3543e0d --- /dev/null +++ b/icalendar_timeseries_server/test/test_config.py @@ -0,0 +1,128 @@ + +import unittest + +import json +import pytz +from datetime import timedelta + +from isodate.duration import Duration + +from icalendar_timeseries_server.config import _keycheck, _parse_timedelta, _parse_timezone, Config + + +_CONFIG_VALID = """ +{ + "addr": "127.0.0.1", + "port": 8090, + "start_delta": "-PT3H", + "end_delta": "P30D", + "cache": "PT15M", + "tz": "Europe/Zurich", + "calendars": { + "private": { + "url": "https://example.cloud/dav/me/private.ics", + "auth": { + "type": "basic", + "username": "me", + "password": "mysupersecurepassword" + } + }, + "public": { + "url": "https://example.cloud/dav/me/public.ics" + }, + "confidential": { + "url": "https://example.cloud/dav/me/confidential.ics", + "ca": "/etc/ssl/ca.pem", + "auth": { + "type": "tls", + "keyfile": "/etc/ssl/client.pem", + "passphrase": "mysupersecurepassword" + } + } + }, + "key_replace": { + "summary": "01_summary", + "description": "02_description" + }, + "value_replace": { + "summary": "{{ summary|truncate(100) }}", + "description": "{{ description|truncate(100) }}" + } +} +""" + + +class ConfigTest(unittest.TestCase): + + def test_keycheck_valid(self) -> None: + config = { + 'foo': 'bar', + 'bar': 42, + 'baz': [], + 'qux': {} + } + strvalue = _keycheck('foo', config, str, '', valid_values=['foo', 'bar', 'baz']) + self.assertEqual(strvalue, 'bar') + strdefvalue = _keycheck('fooo', config, str, '', default_value='baar') + self.assertEqual(strdefvalue, 'baar') + stroptvalue = _keycheck('foooo', config, str, '', optional=True) + self.assertIsNone(stroptvalue) + intvalue = _keycheck('bar', config, int, '') + self.assertEqual(intvalue, 42) + listvalue = _keycheck('baz', config, list, '') + self.assertEqual(listvalue, []) + objvalue = _keycheck('qux', config, dict, '') + self.assertEqual(objvalue, {}) + + def test_keycheck_missing(self) -> None: + config = { + 'foo': 'bar', + } + with self.assertRaises(KeyError): + _keycheck('baz', config, str, '') + + def test_keycheck_type(self) -> None: + config = { + 'foo': 'bar', + } + with self.assertRaises(TypeError): + _keycheck('foo', config, int, '') + + def test_keycheck_value(self) -> None: + config = { + 'foo': '1337', + } + with self.assertRaises(ValueError): + _keycheck('foo', config, str, '', valid_values=['foo', 'bar', 'baz']) + + def test_parse_timedelta(self) -> None: + config = { + 'pos': 'PT42S', + 'neg': '-PT42S' + } + self.assertEqual(_parse_timedelta('zero', config, '', default_value='PT'), timedelta(seconds=0)) + self.assertEqual(_parse_timedelta('pos', config, ''), timedelta(seconds=42)) + self.assertEqual(_parse_timedelta('neg', config, ''), timedelta(seconds=-42)) + self.assertEqual(_parse_timedelta('pos', config, '', force_positive=True), timedelta(seconds=42)) + with self.assertRaises(ValueError): + _parse_timedelta('neg', config, '', force_positive=True) + + def test_parse_timezone(self) -> None: + config = { + 'tz': 'Europe/Zurich', + 'notz': 'North/Winterfell' + } + self.assertEqual(_parse_timezone('tz', config, ''), pytz.timezone('Europe/Zurich')) + self.assertEqual(_parse_timezone('def', config, '', default_value='Europe/Berlin'), + pytz.timezone('Europe/Berlin')) + with self.assertRaises(pytz.exceptions.UnknownTimeZoneError): + _parse_timezone('notz', config, '') + + def test_parse_full_config_valid(self): + config = Config(json.loads(_CONFIG_VALID)) + self.assertEqual(config.addr, '127.0.0.1') + self.assertEqual(config.port, 8090) + self.assertEqual(config.start_delta, Duration(hours=-3)) + self.assertEqual(config.end_delta, Duration(days=30)) + self.assertEqual(config.cache, Duration(minutes=15)) + self.assertEqual(config.tz, pytz.timezone('Europe/Zurich')) diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/conffiles b/package/debian/icalendar-timeseries-server/DEBIAN/conffiles new file mode 100644 index 0000000..164be91 --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/conffiles @@ -0,0 +1 @@ +/etc/icalendar-timeseries-server.json diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/control b/package/debian/icalendar-timeseries-server/DEBIAN/control new file mode 100644 index 0000000..799d2f7 --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/control @@ -0,0 +1,14 @@ +Package: icalendar-timeseries-server +Version: 0.1 +Maintainer: s3lph +Section: web +Priority: optional +Architecture: all +Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate, python3-tz +Description: Scrape iCalendar endpoints and present their data in a + timeseries format. A small service that scrapes iCalendar files + served over HTTP, parses their contents and returns a timeseries + format compatible to the /api/v1/query API endpoint of a Prometheus + server. This allows e.g. a Grafana administrator to add a Prometheus + data source pointing at this server, returning the events in the + calendars in the event metric. diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/postinst b/package/debian/icalendar-timeseries-server/DEBIAN/postinst new file mode 100755 index 0000000..b7586de --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/postinst @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "configure" ]]; then + + if ! getent group its >/dev/null; then + groupadd --system its + fi + + if ! getent passwd its >/dev/null; then + useradd --system --create-home --gid its --home-dir /var/lib/its --shell /usr/sbin/nologin its + fi + + chown its:its /var/lib/its + chmod 0750 /var/lib/its + + systemctl daemon-reload || true + +fi diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/postrm b/package/debian/icalendar-timeseries-server/DEBIAN/postrm new file mode 100755 index 0000000..305068d --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/postrm @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "remove" ]]; then + + systemctl daemon-reload || true + +fi diff --git a/package/debian/icalendar-timeseries-server/DEBIAN/prerm b/package/debian/icalendar-timeseries-server/DEBIAN/prerm new file mode 100755 index 0000000..42c7b34 --- /dev/null +++ b/package/debian/icalendar-timeseries-server/DEBIAN/prerm @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "remove" ]]; then + + userdel its + +fi diff --git a/package/debian/icalendar-timeseries-server/etc/icalendar-timeseries-server.json b/package/debian/icalendar-timeseries-server/etc/icalendar-timeseries-server.json new file mode 100644 index 0000000..92f16fc --- /dev/null +++ b/package/debian/icalendar-timeseries-server/etc/icalendar-timeseries-server.json @@ -0,0 +1,18 @@ +{ + "addr": "127.0.0.1", + "port": 8090, + "start_delta": "-PT3H", + "end_delta": "P60D", + "cache": "PT3M", + "tz": "Europe/Zurich", + "calendars": {}, + "key_replace": { + "summary": "01_summary", + "description": "02_description", + "calendar": "03_calendar" + }, + "value_replace": { + "summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", + "description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}" + } +} diff --git a/package/debian/icalendar-timeseries-server/lib/systemd/system/icalendar-timeseries-server.service b/package/debian/icalendar-timeseries-server/lib/systemd/system/icalendar-timeseries-server.service new file mode 100644 index 0000000..e0dc628 --- /dev/null +++ b/package/debian/icalendar-timeseries-server/lib/systemd/system/icalendar-timeseries-server.service @@ -0,0 +1,10 @@ +[Unit] +Description=icalendar-timeseries-server +After=networking.target + +[Service] +ExecStart=/usr/bin/python3 -m icalendar_timeseries_server /etc/icalendar-timeseries-server.json +User=its + +[Install] +WantedBy=multi-user.target diff --git a/package/debian/icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/.gitkeep b/package/debian/icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/copyright b/package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/copyright new file mode 100644 index 0000000..bf560eb --- /dev/null +++ b/package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/copyright @@ -0,0 +1,16 @@ +Copyright 2019 s3lph + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/package/release.py b/package/release.py new file mode 100755 index 0000000..6281bd0 --- /dev/null +++ b/package/release.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +from typing import Any, Dict, List, Optional, Tuple + +import os +import sys +import json +import urllib.request +import http.client +from urllib.error import HTTPError + + +def parse_changelog(tag: str) -> Optional[str]: + release_changelog: str = '' + with open('CHANGELOG.md', 'r') as f: + in_target: bool = False + done: bool = False + for line in f.readlines(): + if in_target: + if f'' in line: + done = True + break + release_changelog += line + elif f'' in line: + in_target = True + continue + if not done: + return None + return release_changelog + + +def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]: + url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs' + headers: Dict[str, str] = { + 'Private-Token': api_token + } + req = urllib.request.Request(url, headers=headers) + try: + resp: http.client.HTTPResponse = urllib.request.urlopen(req) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + resp_data: bytes = resp.read() + joblist: List[Dict[str, Any]] = json.loads(resp_data.decode()) + + jobidmap: Dict[str, str] = {} + for job in joblist: + name: str = job['name'] + job_id: str = job['id'] + jobidmap[name] = job_id + return jobidmap + + +def fetch_single_shafile(url: str) -> str: + req = urllib.request.Request(url) + try: + resp: http.client.HTTPResponse = urllib.request.urlopen(req) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + resp_data: bytes = resp.readline() + shafile: str = resp_data.decode() + filename: str = shafile.strip().split(' ')[-1].strip() + return filename + + +def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: + mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw' + wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS' + wheel_filename: str = fetch_single_shafile(wheel_sha_url) + wheel_url: str = f'{mybase}/dist/{wheel_filename}' + return wheel_url, wheel_sha_url + + +def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]: + mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw' + debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS' + debian_filename: str = fetch_single_shafile(debian_sha_url) + debian_url: str = f'{mybase}/package/debian/{debian_filename}' + return debian_url, debian_sha_url + + +def main(): + api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN') + release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG') + project_name: Optional[str] = os.getenv('CI_PROJECT_PATH') + project_id: Optional[str] = os.getenv('CI_PROJECT_ID') + pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID') + if api_token is None: + print('GITLAB_API_TOKEN is not set.', file=sys.stderr) + sys.exit(1) + if release_tag is None: + print('CI_COMMIT_TAG is not set.', file=sys.stderr) + sys.exit(1) + if project_name is None: + print('CI_PROJECT_PATH is not set.', file=sys.stderr) + sys.exit(1) + if project_id is None: + print('CI_PROJECT_ID is not set.', file=sys.stderr) + sys.exit(1) + if pipeline_id is None: + print('CI_PIPELINE_ID is not set.', file=sys.stderr) + sys.exit(1) + + changelog: Optional[str] = parse_changelog(release_tag) + if changelog is None: + print('Changelog could not be parsed.', file=sys.stderr) + sys.exit(1) + + job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token) + + base_url: str = f'https://gitlab.com/{project_name}/-' + + wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids) + debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids) + + augmented_changelog = f'''{changelog.strip()} + +### Download + +- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url})) +- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))''' + + post_body: str = json.dumps({'description': augmented_changelog}) + + gitlab_release_api_url: str = \ + f'https://gitlab.com/api/v4/projects/{project_id}/repository/tags/{release_tag}/release' + headers: Dict[str, str] = { + 'Private-Token': api_token, + 'Content-Type': 'application/json; charset=utf-8' + } + + request = urllib.request.Request( + gitlab_release_api_url, + post_body.encode('utf-8'), + headers=headers, + method='POST' + ) + try: + response: http.client.HTTPResponse = urllib.request.urlopen(request) + except HTTPError as e: + print(e.read().decode()) + sys.exit(1) + response_bytes: bytes = response.read() + response_str: str = response_bytes.decode() + response_data: Dict[str, Any] = json.loads(response_str) + + if response_data['tag_name'] != release_tag: + print('Something went wrong...', file=sys.stderr) + print(response_str, file=sys.stderr) + sys.exit(1) + + print(response_data['description']) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..25cdba8 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ + +# +# PyCodestyle +# + +[pycodestyle] +max-line-length = 120 +statistics = True + +# +# Coverage +# + +[run] +branch = True +parallel = True +source = icalendar_timeseries_server/ + +[report] +show_missing = True +include = icalendar_timeseries_server/* +omit = */test/*.py diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..e5aaae3 --- /dev/null +++ b/setup.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +from setuptools import setup, find_packages + +from icalendar_timeseries_server import __version__ + +setup( + name='icalendar_timeseries_server', + version=__version__, + author='s3lph', + author_email='', + description='', + license='MIT', + keywords='ical,icalendar,timeseries,prometheus,grafana', + url='https://gitlab.com/s3lph/icalendar-timeseries-server', + packages=find_packages(exclude=['*.test']), + long_description='', + python_requires='>=3.6', + install_requires=[ + 'bottle', + 'python-dateutil', + 'icalendar', + 'isodate', + 'jinja2', + 'pytz' + ], + entry_points={ + 'console_scripts': [ + 'icalendar-timeseries-server = icalendar_timeseries_server:main' + ] + }, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Web Environment', + 'Framework :: Bottle', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: MIT License', + 'Topic :: System :: Monitoring' + ], +)