diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ae123..97df943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # iCalendar Timeseries Server Changelog + +## Version 0.3 + +### Changes + + +- Replace print statements by proper logging +- Fix: Ensure scrape interval is positive +- Fix: Keep showing events that already started, but have not finished yet + + + + + ## Version 0.2 diff --git a/icalendar-timeseries-server.json b/icalendar-timeseries-server.json deleted file mode 100644 index 847ee7d..0000000 --- a/icalendar-timeseries-server.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "addr": "127.0.0.1", - "port": 8090, - "start_delta": "-PT3H", - "end_delta": "P60D", - "cache": "PT3M", - "tz": "Europe/Zurich", - "calendars": { - "tlstest": { - "interval": "PT5M", - "url": "https://localhost/private.ics", - "ca": "/home/sebastian/tlstest/ca/ca/ca.crt", - "auth": { - "type": "tls", - "keyfile": "/home/sebastian/tlstest/client/combined.pem" - } - }, - "filetest": { - "interval": "PT1M", - "url": "file:///srv/http/private.ics" - } - }, - "key_replace": { - "summary": "a_summary", - "description": "b_description" - }, - "value_replace": { - "summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", - "description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}", - "useless_metric": "{{ start.timestamp() + end.timestamp() }}" - } -} diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py index 5674f0e..0a22acb 100644 --- a/icalendar_timeseries_server/api.py +++ b/icalendar_timeseries_server/api.py @@ -1,8 +1,7 @@ from typing import List import json -from urllib.error import HTTPError -import traceback +import logging import bottle @@ -26,7 +25,7 @@ def prometheus_api(): 'error': str(e) } bottle.response.status = 400 - traceback.print_exc() + logging.exception('Cannot parse PromQL query') bottle.response.add_header('Content-Type', 'application/json') return json.dumps(response) @@ -42,14 +41,6 @@ def prometheus_api(): 'result': [e.serialize() for e in events] } } - except HTTPError as e: - response = { - 'status': 'error', - 'errorType': 'internal', - 'error': str(e) - } - bottle.response.status = 500 - traceback.print_exc() except BaseException: response = { 'status': 'error', @@ -57,7 +48,7 @@ def prometheus_api(): 'error': 'An internal error occurred.' } bottle.response.status = 500 - traceback.print_exc() + logging.exception('An internal error occurred') bottle.response.add_header('Content-Type', 'application/json') return json.dumps(response) diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index a2e2200..8db89b5 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -2,6 +2,7 @@ from typing import Dict, List, Iterable import sys import urllib.request +import logging from datetime import datetime, date, timedelta from threading import Lock, Timer @@ -54,19 +55,24 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da events = [] opener: urllib.request.OpenerDirector = config.get_url_opener() - with opener.open(config.url) as response: - data = response.read().decode('utf-8') + try: + with opener.open(config.url) as response: + data = response.read().decode('utf-8') + except BaseException: + logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ({config.url})') + return calendar = cal.Calendar.from_ical(data) for element in calendar.walk(): if element.name == "VEVENT": dtstart = element.get('dtstart').dt - if isinstance(dtstart, date): + # Apparently datetime is a subclass of date... + if isinstance(dtstart, date) and not isinstance(dtstart, datetime): dtstart = datetime(dtstart.year, dtstart.month, dtstart.day, tzinfo=start.tzinfo) # Process either end timestamp or duration, if present if 'dtend' in element: evend = element.get('dtend').dt - if isinstance(evend, date): + if isinstance(evend, date) and not isinstance(evend, datetime): evend = datetime(evend.year, evend.month, evend.day, tzinfo=start.tzinfo) duration = evend - dtstart elif 'duration' in element: @@ -78,7 +84,7 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da else: occurences = [dtstart] for occurence in occurences: - if start <= occurence < end: + if start <= occurence + duration and occurence < end: events.append(Event(name, element, occurence, occurence + duration)) with _SCRAPE_CACHE_LOCK: _SCRAPE_CACHE[name] = events diff --git a/icalendar_timeseries_server/config.py b/icalendar_timeseries_server/config.py index bfbde2c..c3d18df 100644 --- a/icalendar_timeseries_server/config.py +++ b/icalendar_timeseries_server/config.py @@ -6,6 +6,7 @@ from datetime import timedelta import ssl import urllib.request import sys +import logging import pytz import jinja2 @@ -27,7 +28,8 @@ class CalendarConfig: def __init__(self, config: Dict[str, Any], config_path: str) -> None: self._url: str = _keycheck('url', config, str, config_path) - self._scrape_interval: Duration = _parse_timedelta('interval', config, config_path, default_value='PT15M') + self._scrape_interval: Duration = _parse_timedelta('interval', config, config_path, default_value='PT15M', + force_positive=True) self._ca: Optional[str] = _keycheck('ca', config, str, config_path, optional=True) auth: Dict[str, Any] = _keycheck('auth', config, dict, config_path, default_value={'type': 'none'}) self._authtype: str = _keycheck('type', auth, str, f'{config_path}.auth', @@ -155,7 +157,7 @@ def _keycheck(key: str, raise KeyError(f'Expected member "{key}" not found at path {path}') value: Any = config[key] if not isinstance(value, typ): - raise TypeError(f'Expected {typ}, not {type(value).__name__} for path {path}.{key}') + raise TypeError(f'Expected {typ.__name__}, not {type(value).__name__} for path {path}.{key}') if valid_values is not None: if value not in valid_values: raise ValueError(f'Expected one of {", ".join(valid_values)} ({typ}), not {value} for path {path}.{key}') @@ -216,10 +218,17 @@ def get_jenv() -> jinja2.Environment: def load_config(filename: str): global CONFIG, JENV - with open(filename, 'r') as f: - json_config = json.loads(f.read()) - CONFIG = Config(json_config) - JENV = jinja2.Environment() + try: + with open(filename, 'r') as f: + json_config = json.loads(f.read()) + CONFIG = Config(json_config) + JENV = jinja2.Environment() + except json.JSONDecodeError as e: + logging.exception('Cannot parse config JSON') + raise e + except Exception as e: + logging.error(e) + raise e def load_default_config(): diff --git a/icalendar_timeseries_server/main.py b/icalendar_timeseries_server/main.py index a1168f0..73e7e55 100644 --- a/icalendar_timeseries_server/main.py +++ b/icalendar_timeseries_server/main.py @@ -1,4 +1,5 @@ import sys +import logging import bottle @@ -11,17 +12,32 @@ from icalendar_timeseries_server.api import prometheus_api def main(): - if len(sys.argv) == 1: - load_default_config() - elif len(sys.argv) == 2: - load_config(sys.argv[1]) - else: - print(f'Can only read one config file, got "{" ".join(sys.argv[1:])}"') - exit(1) + # Set up logger + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter( + '%(asctime)s %(filename)s:%(lineno)d(%(funcName)s) [%(levelname)s]: %(message)s')) + logging.getLogger().addHandler(log_handler) + + # Load configuration config = get_config() + try: + if len(sys.argv) == 1: + load_default_config() + elif len(sys.argv) == 2: + load_config(sys.argv[1]) + else: + logging.log(logging.FATAL, f'Can only read one config file, got "{" ".join(sys.argv[1:])}"') + exit(1) + # Re-fetch config after parsing + config = get_config() + except BaseException: + logging.fatal('Could not parse configuration file') + exit(1) + # Schedule calendar scraping in the background for calname in config.calendars.keys(): start_scrape_calendar(calname, config.calendars[calname]) + # Start the Bottle HTTP server bottle.run(host=config.addr, port=get_config().port) diff --git a/icalendar_timeseries_server/query.py b/icalendar_timeseries_server/query.py index b266898..ce2ebfc 100644 --- a/icalendar_timeseries_server/query.py +++ b/icalendar_timeseries_server/query.py @@ -1,6 +1,7 @@ from typing import Dict import re +import logging LABEL_MATCH_OPERATORS = [ '=', @@ -64,7 +65,7 @@ class MetricQuery: self.__parse(q) def __parse(self, q: str): - print(q) + logging.debug(f'Parsing PromQL query string: {q}') # globalstate: # 0 = parsing metric name # 1 = parsing filters