Implement todo exporting

CHANGELOG.md

@@ -1,6 +1,18 @@
 # iCalendar Timeseries Server Changelog
 
 
+<!-- BEGIN RELEASE v0.4.0 -->
+## Version 0.4.0
+
+### Changes
+
+<!-- BEGIN CHANGES 0.4.0 -->
+- VTODO components are exported in a second time series, `todo` . Todo recurrence is not supported yet though.
+<!-- END CHANGES 0.4.0 -->
+
+<!-- END RELEASE v0.4.0 -->
+
+
 <!-- BEGIN RELEASE v0.3.3 -->
 ## Version 0.3.3
 

README.md

@@ -4,8 +4,8 @@ This project is a small service that scrapes iCalendar files served
 over HTTP, parses their contents and returns the data in a timeseries
 format compatible to the `/api/v1/query` API endpoint of a Prometheus
 server.  This allows e.g. a Grafana administrator to add a Prometheus
-data source pointing at this server, returning the events in the
-calendars in the `event` metric.
+data source pointing at this server, returning calendar events in the
+`event` metric and todos in the `todo` metric.
 
 ## Example
 
@@ -147,7 +147,7 @@ Configuration is done through a JSON config file:
 | `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. |
 | `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. |
 | `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). |
-| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. |
+| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana 6 and earlier. |
 | `keys(key_replace)` | string | The labels to rename. |
 | `key_replace.*` | string | The names to rename the labels to. |
 | `value_replace` | dict | Label values to postprocess. |

icalendar_timeseries_server/__init__.py

@@ -1,2 +1,2 @@
 
-__version__ = '0.3.3'
+__version__ = '0.4.0'

icalendar_timeseries_server/api.py

@@ -7,7 +7,7 @@ import bottle
 
 from icalendar_timeseries_server.config import get_config
 from icalendar_timeseries_server.event import Metric
-from icalendar_timeseries_server.cal import get_calendar
+from icalendar_timeseries_server.cal import get_calendar_events, get_calendar_todos
 from icalendar_timeseries_server.query import MetricQuery
 
 
@@ -31,9 +31,16 @@ def prometheus_api():
 
     try:
         for name in get_config().calendars.keys():
-            events.extend(get_calendar(name))
-        events = list(filter(q, events))
-        events.sort(key=lambda e: e.start)
+            if q.name == 'event':
+                events.extend(get_calendar_events(name))
+                events = list(filter(q, events))
+                events.sort(key=lambda e: e.start)
+            elif q.name == 'todo':
+                events.extend(get_calendar_todos(name))
+                events = list(filter(q, events))
+                [print(e.name, e.due, e.priority) for e in events]
+                # Sort by due date and priority
+                events.sort(key=lambda e: (e.due, e.priority))
         response = {
             'status': 'success',
             'data': {

icalendar_timeseries_server/cal.py

@@ -13,9 +13,11 @@ from isodate import Duration
 from icalendar_timeseries_server import __version__
 from icalendar_timeseries_server.config import get_config, CalendarConfig
 from icalendar_timeseries_server.event import Event
+from icalendar_timeseries_server.todo import Todo
 
 
-_SCRAPE_CACHE: Dict[str, List[Event]] = dict()
+_EVENT_SCRAPE_CACHE: Dict[str, List[Event]] = dict()
+_TODO_SCRAPE_CACHE: Dict[str, List[Todo]] = dict()
 _SCRAPE_CACHE_LOCK: Lock = Lock()
 
 __py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
@@ -53,8 +55,9 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration:
 
 
 def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime):
-    global _SCRAPE_CACHE, _SCRAPE_CACHE_LOCK
+    global _EVENT_SCRAPE_CACHE, _TODO_SCRAPE_CACHE, _SCRAPE_CACHE_LOCK
     events = []
+    todos = []
 
     opener: urllib.request.OpenerDirector = config.get_url_opener()
     try:
@@ -88,8 +91,19 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da
             for occurence in occurences:
                 if start <= occurence + duration and occurence < end:
                     events.append(Event(name, element, occurence, occurence + duration))
+        elif element.name == "VTODO":
+            dtstart = element.get('dtstamp').dt
+            duration = timedelta(0)
+            if 'dtstart' in element:
+                dtstart = element.get('dtstart').dt
+                if 'duration' in element:
+                    duration = element.get('duration').dt
+            todos.append(Todo(name, element, dtstart, dtstart + duration))
+
+
     with _SCRAPE_CACHE_LOCK:
-        _SCRAPE_CACHE[name] = events
+        _EVENT_SCRAPE_CACHE[name] = events
+        _TODO_SCRAPE_CACHE[name] = todos
 
 
 def scrape_calendar(name: str, config: CalendarConfig):
@@ -115,7 +129,13 @@ def start_scrape_calendar(name: str, config: CalendarConfig):
     cron.start()
 
 
-def get_calendar(name: str):
-    global _SCRAPE_CACHE
+def get_calendar_events(name: str):
+    global _EVENT_SCRAPE_CACHE
     with _SCRAPE_CACHE_LOCK:
-        return _SCRAPE_CACHE.get(name, [])
+        return _EVENT_SCRAPE_CACHE.get(name, [])
+
+
+def get_calendar_todos(name: str):
+    global _TODO_SCRAPE_CACHE
+    with _SCRAPE_CACHE_LOCK:
+        return _TODO_SCRAPE_CACHE.get(name, [])

icalendar_timeseries_server/event.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, List
 
 import icalendar
 import jinja2

icalendar_timeseries_server/query.py

@@ -172,7 +172,7 @@ class MetricQuery:
         elif filterstate != 0:
             raise ValueError('Unexpected EOF')
 
-    def __call__(self, metric: Metric):
+    def __call__(self, metric: Metric) -> bool:
         """
         Applies the filter deducted from the query string to the given metric.
 
@@ -188,3 +188,7 @@ class MetricQuery:
                 return False
         # Return True if all filters matched
         return True
+
+    @property
+    def name(self) -> str:
+        return self._metric_name

icalendar_timeseries_server/todo.py

@@ -0,0 +1,72 @@
+from typing import Any, Dict, List
+
+import icalendar
+import jinja2
+from datetime import datetime, timedelta
+
+from icalendar_timeseries_server.config import get_config, get_jenv
+from icalendar_timeseries_server.query import Metric
+
+_ATTRIBUTES: List[str] = [
+    'class',
+    'description',
+    'geo',
+    'location',
+    'organizer',
+    'percent-complete',
+    'priority',
+    'status',
+    'summary',
+    'url',
+    'due',
+    'attach'
+]
+
+
+class Todo(Metric):
+
+    def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime):
+        self.calendar: str = cname
+        self.start = start
+        # self.attributes: Dict[str, str] = dict()
+        attributes: Dict[str, str] = dict()
+        tmp: Dict[str, Any] = {
+            'calendar': cname,
+            'start': start,
+            'end': end
+        }
+        for attr in _ATTRIBUTES:
+            tmp[attr] = todo.get(attr, '')
+        substitution_keys = set(_ATTRIBUTES)
+        substitution_keys.update(tmp.keys())
+        substitution_keys.update(get_config().key_replace.keys())
+        substitution_keys.update(get_config().value_replace.keys())
+        for attr in substitution_keys:
+            newkey: str = get_config().key_replace.get(attr, attr)
+            value: str = tmp.get(attr, '')
+            newval_template: str = get_config().value_replace.get(attr, str(value))
+            jtemplate: jinja2.Template = get_jenv().from_string(newval_template)
+            newvalue: str = jtemplate.render(**tmp)
+            attributes[newkey] = newvalue
+        self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}'
+        due = todo.get('due', None)
+        if due:
+            self.due = due.dt
+        else:
+            self.due = datetime.now(get_config().tz) + timedelta(days=36500)
+        self.priority = todo.get('priority', '0')
+        super().__init__('todo', attributes)
+
+    def serialize(self) -> Dict[str, Any]:
+        todo: Dict[str, Any] = {
+            'metric': {
+                '__name__': 'todo',
+                'calendar': self.calendar
+            },
+            'value': [
+                self.start.timestamp(),
+                1
+            ]
+        }
+        todo['metric'].update(self._labels)
+        return todo