From 395393345e0a74e6b7c35df3fcc3abf38348315c Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 6 Nov 2020 03:30:47 +0100 Subject: [PATCH] Implement todo exporting --- CHANGELOG.md | 12 +++++ README.md | 6 +-- icalendar_timeseries_server/__init__.py | 2 +- icalendar_timeseries_server/api.py | 15 ++++-- icalendar_timeseries_server/cal.py | 32 ++++++++--- icalendar_timeseries_server/event.py | 2 +- icalendar_timeseries_server/query.py | 6 ++- icalendar_timeseries_server/todo.py | 72 +++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 icalendar_timeseries_server/todo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dedff8..9df746c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # iCalendar Timeseries Server Changelog + +## Version 0.4.0 + +### Changes + + +- VTODO components are exported in a second time series, `todo` . Todo recurrence is not supported yet though. + + + + + ## Version 0.3.3 diff --git a/README.md b/README.md index c762283..60a5fe6 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ This project is a small service that scrapes iCalendar files served over HTTP, parses their contents and returns the data in a timeseries format compatible to the `/api/v1/query` API endpoint of a Prometheus server. This allows e.g. a Grafana administrator to add a Prometheus -data source pointing at this server, returning the events in the -calendars in the `event` metric. +data source pointing at this server, returning calendar events in the +`event` metric and todos in the `todo` metric. ## Example @@ -147,7 +147,7 @@ Configuration is done through a JSON config file: | `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. | | `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. | | `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). | -| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. | +| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana 6 and earlier. | | `keys(key_replace)` | string | The labels to rename. | | `key_replace.*` | string | The names to rename the labels to. | | `value_replace` | dict | Label values to postprocess. | diff --git a/icalendar_timeseries_server/__init__.py b/icalendar_timeseries_server/__init__.py index 2fd20b5..652a8f4 100644 --- a/icalendar_timeseries_server/__init__.py +++ b/icalendar_timeseries_server/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.3.3' +__version__ = '0.4.0' diff --git a/icalendar_timeseries_server/api.py b/icalendar_timeseries_server/api.py index 0a22acb..fbfd60d 100644 --- a/icalendar_timeseries_server/api.py +++ b/icalendar_timeseries_server/api.py @@ -7,7 +7,7 @@ import bottle from icalendar_timeseries_server.config import get_config from icalendar_timeseries_server.event import Metric -from icalendar_timeseries_server.cal import get_calendar +from icalendar_timeseries_server.cal import get_calendar_events, get_calendar_todos from icalendar_timeseries_server.query import MetricQuery @@ -31,9 +31,16 @@ def prometheus_api(): try: for name in get_config().calendars.keys(): - events.extend(get_calendar(name)) - events = list(filter(q, events)) - events.sort(key=lambda e: e.start) + if q.name == 'event': + events.extend(get_calendar_events(name)) + events = list(filter(q, events)) + events.sort(key=lambda e: e.start) + elif q.name == 'todo': + events.extend(get_calendar_todos(name)) + events = list(filter(q, events)) + [print(e.name, e.due, e.priority) for e in events] + # Sort by due date and priority + events.sort(key=lambda e: (e.due, e.priority)) response = { 'status': 'success', 'data': { diff --git a/icalendar_timeseries_server/cal.py b/icalendar_timeseries_server/cal.py index e4e479a..46e2c03 100644 --- a/icalendar_timeseries_server/cal.py +++ b/icalendar_timeseries_server/cal.py @@ -13,9 +13,11 @@ from isodate import Duration from icalendar_timeseries_server import __version__ from icalendar_timeseries_server.config import get_config, CalendarConfig from icalendar_timeseries_server.event import Event +from icalendar_timeseries_server.todo import Todo -_SCRAPE_CACHE: Dict[str, List[Event]] = dict() +_EVENT_SCRAPE_CACHE: Dict[str, List[Event]] = dict() +_TODO_SCRAPE_CACHE: Dict[str, List[Todo]] = dict() _SCRAPE_CACHE_LOCK: Lock = Lock() __py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}' @@ -53,8 +55,9 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration: def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime): - global _SCRAPE_CACHE, _SCRAPE_CACHE_LOCK + global _EVENT_SCRAPE_CACHE, _TODO_SCRAPE_CACHE, _SCRAPE_CACHE_LOCK events = [] + todos = [] opener: urllib.request.OpenerDirector = config.get_url_opener() try: @@ -88,8 +91,19 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da for occurence in occurences: if start <= occurence + duration and occurence < end: events.append(Event(name, element, occurence, occurence + duration)) + elif element.name == "VTODO": + dtstart = element.get('dtstamp').dt + duration = timedelta(0) + if 'dtstart' in element: + dtstart = element.get('dtstart').dt + if 'duration' in element: + duration = element.get('duration').dt + todos.append(Todo(name, element, dtstart, dtstart + duration)) + + with _SCRAPE_CACHE_LOCK: - _SCRAPE_CACHE[name] = events + _EVENT_SCRAPE_CACHE[name] = events + _TODO_SCRAPE_CACHE[name] = todos def scrape_calendar(name: str, config: CalendarConfig): @@ -115,7 +129,13 @@ def start_scrape_calendar(name: str, config: CalendarConfig): cron.start() -def get_calendar(name: str): - global _SCRAPE_CACHE +def get_calendar_events(name: str): + global _EVENT_SCRAPE_CACHE with _SCRAPE_CACHE_LOCK: - return _SCRAPE_CACHE.get(name, []) + return _EVENT_SCRAPE_CACHE.get(name, []) + + +def get_calendar_todos(name: str): + global _TODO_SCRAPE_CACHE + with _SCRAPE_CACHE_LOCK: + return _TODO_SCRAPE_CACHE.get(name, []) diff --git a/icalendar_timeseries_server/event.py b/icalendar_timeseries_server/event.py index 934b4f7..d03ce79 100644 --- a/icalendar_timeseries_server/event.py +++ b/icalendar_timeseries_server/event.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Set +from typing import Any, Dict, List import icalendar import jinja2 diff --git a/icalendar_timeseries_server/query.py b/icalendar_timeseries_server/query.py index ce2ebfc..d7427cc 100644 --- a/icalendar_timeseries_server/query.py +++ b/icalendar_timeseries_server/query.py @@ -172,7 +172,7 @@ class MetricQuery: elif filterstate != 0: raise ValueError('Unexpected EOF') - def __call__(self, metric: Metric): + def __call__(self, metric: Metric) -> bool: """ Applies the filter deducted from the query string to the given metric. @@ -188,3 +188,7 @@ class MetricQuery: return False # Return True if all filters matched return True + + @property + def name(self) -> str: + return self._metric_name diff --git a/icalendar_timeseries_server/todo.py b/icalendar_timeseries_server/todo.py new file mode 100644 index 0000000..64ad7ad --- /dev/null +++ b/icalendar_timeseries_server/todo.py @@ -0,0 +1,72 @@ +from typing import Any, Dict, List + +import icalendar +import jinja2 +from datetime import datetime, timedelta + +from icalendar_timeseries_server.config import get_config, get_jenv +from icalendar_timeseries_server.query import Metric + +_ATTRIBUTES: List[str] = [ + 'class', + 'description', + 'geo', + 'location', + 'organizer', + 'percent-complete', + 'priority', + 'status', + 'summary', + 'url', + 'due', + 'attach' +] + + +class Todo(Metric): + + def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime): + self.calendar: str = cname + self.start = start + # self.attributes: Dict[str, str] = dict() + attributes: Dict[str, str] = dict() + tmp: Dict[str, Any] = { + 'calendar': cname, + 'start': start, + 'end': end + } + for attr in _ATTRIBUTES: + tmp[attr] = todo.get(attr, '') + substitution_keys = set(_ATTRIBUTES) + substitution_keys.update(tmp.keys()) + substitution_keys.update(get_config().key_replace.keys()) + substitution_keys.update(get_config().value_replace.keys()) + for attr in substitution_keys: + newkey: str = get_config().key_replace.get(attr, attr) + value: str = tmp.get(attr, '') + newval_template: str = get_config().value_replace.get(attr, str(value)) + jtemplate: jinja2.Template = get_jenv().from_string(newval_template) + newvalue: str = jtemplate.render(**tmp) + attributes[newkey] = newvalue + self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}' + due = todo.get('due', None) + if due: + self.due = due.dt + else: + self.due = datetime.now(get_config().tz) + timedelta(days=36500) + self.priority = todo.get('priority', '0') + super().__init__('todo', attributes) + + def serialize(self) -> Dict[str, Any]: + todo: Dict[str, Any] = { + 'metric': { + '__name__': 'todo', + 'calendar': self.calendar + }, + 'value': [ + self.start.timestamp(), + 1 + ] + } + todo['metric'].update(self._labels) + return todo