Implement todo exporting

This commit is contained in:
s3lph 2020-11-06 03:30:47 +01:00
parent 7f36e5b8dc
commit 395393345e
8 changed files with 131 additions and 16 deletions

View file

@ -1,6 +1,18 @@
# iCalendar Timeseries Server Changelog # iCalendar Timeseries Server Changelog
<!-- BEGIN RELEASE v0.4.0 -->
## Version 0.4.0
### Changes
<!-- BEGIN CHANGES 0.4.0 -->
- VTODO components are exported in a second time series, `todo` . Todo recurrence is not supported yet though.
<!-- END CHANGES 0.4.0 -->
<!-- END RELEASE v0.4.0 -->
<!-- BEGIN RELEASE v0.3.3 --> <!-- BEGIN RELEASE v0.3.3 -->
## Version 0.3.3 ## Version 0.3.3

View file

@ -4,8 +4,8 @@ This project is a small service that scrapes iCalendar files served
over HTTP, parses their contents and returns the data in a timeseries over HTTP, parses their contents and returns the data in a timeseries
format compatible to the `/api/v1/query` API endpoint of a Prometheus format compatible to the `/api/v1/query` API endpoint of a Prometheus
server. This allows e.g. a Grafana administrator to add a Prometheus server. This allows e.g. a Grafana administrator to add a Prometheus
data source pointing at this server, returning the events in the data source pointing at this server, returning calendar events in the
calendars in the `event` metric. `event` metric and todos in the `todo` metric.
## Example ## Example
@ -147,7 +147,7 @@ Configuration is done through a JSON config file:
| `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. | | `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. |
| `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. | | `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. |
| `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). | | `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). |
| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. | | `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana 6 and earlier. |
| `keys(key_replace)` | string | The labels to rename. | | `keys(key_replace)` | string | The labels to rename. |
| `key_replace.*` | string | The names to rename the labels to. | | `key_replace.*` | string | The names to rename the labels to. |
| `value_replace` | dict | Label values to postprocess. | | `value_replace` | dict | Label values to postprocess. |

View file

@ -1,2 +1,2 @@
__version__ = '0.3.3' __version__ = '0.4.0'

View file

@ -7,7 +7,7 @@ import bottle
from icalendar_timeseries_server.config import get_config from icalendar_timeseries_server.config import get_config
from icalendar_timeseries_server.event import Metric from icalendar_timeseries_server.event import Metric
from icalendar_timeseries_server.cal import get_calendar from icalendar_timeseries_server.cal import get_calendar_events, get_calendar_todos
from icalendar_timeseries_server.query import MetricQuery from icalendar_timeseries_server.query import MetricQuery
@ -31,9 +31,16 @@ def prometheus_api():
try: try:
for name in get_config().calendars.keys(): for name in get_config().calendars.keys():
events.extend(get_calendar(name)) if q.name == 'event':
events.extend(get_calendar_events(name))
events = list(filter(q, events)) events = list(filter(q, events))
events.sort(key=lambda e: e.start) events.sort(key=lambda e: e.start)
elif q.name == 'todo':
events.extend(get_calendar_todos(name))
events = list(filter(q, events))
[print(e.name, e.due, e.priority) for e in events]
# Sort by due date and priority
events.sort(key=lambda e: (e.due, e.priority))
response = { response = {
'status': 'success', 'status': 'success',
'data': { 'data': {

View file

@ -13,9 +13,11 @@ from isodate import Duration
from icalendar_timeseries_server import __version__ from icalendar_timeseries_server import __version__
from icalendar_timeseries_server.config import get_config, CalendarConfig from icalendar_timeseries_server.config import get_config, CalendarConfig
from icalendar_timeseries_server.event import Event from icalendar_timeseries_server.event import Event
from icalendar_timeseries_server.todo import Todo
_SCRAPE_CACHE: Dict[str, List[Event]] = dict() _EVENT_SCRAPE_CACHE: Dict[str, List[Event]] = dict()
_TODO_SCRAPE_CACHE: Dict[str, List[Todo]] = dict()
_SCRAPE_CACHE_LOCK: Lock = Lock() _SCRAPE_CACHE_LOCK: Lock = Lock()
__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}' __py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
@ -53,8 +55,9 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration:
def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime): def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime):
global _SCRAPE_CACHE, _SCRAPE_CACHE_LOCK global _EVENT_SCRAPE_CACHE, _TODO_SCRAPE_CACHE, _SCRAPE_CACHE_LOCK
events = [] events = []
todos = []
opener: urllib.request.OpenerDirector = config.get_url_opener() opener: urllib.request.OpenerDirector = config.get_url_opener()
try: try:
@ -88,8 +91,19 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da
for occurence in occurences: for occurence in occurences:
if start <= occurence + duration and occurence < end: if start <= occurence + duration and occurence < end:
events.append(Event(name, element, occurence, occurence + duration)) events.append(Event(name, element, occurence, occurence + duration))
elif element.name == "VTODO":
dtstart = element.get('dtstamp').dt
duration = timedelta(0)
if 'dtstart' in element:
dtstart = element.get('dtstart').dt
if 'duration' in element:
duration = element.get('duration').dt
todos.append(Todo(name, element, dtstart, dtstart + duration))
with _SCRAPE_CACHE_LOCK: with _SCRAPE_CACHE_LOCK:
_SCRAPE_CACHE[name] = events _EVENT_SCRAPE_CACHE[name] = events
_TODO_SCRAPE_CACHE[name] = todos
def scrape_calendar(name: str, config: CalendarConfig): def scrape_calendar(name: str, config: CalendarConfig):
@ -115,7 +129,13 @@ def start_scrape_calendar(name: str, config: CalendarConfig):
cron.start() cron.start()
def get_calendar(name: str): def get_calendar_events(name: str):
global _SCRAPE_CACHE global _EVENT_SCRAPE_CACHE
with _SCRAPE_CACHE_LOCK: with _SCRAPE_CACHE_LOCK:
return _SCRAPE_CACHE.get(name, []) return _EVENT_SCRAPE_CACHE.get(name, [])
def get_calendar_todos(name: str):
global _TODO_SCRAPE_CACHE
with _SCRAPE_CACHE_LOCK:
return _TODO_SCRAPE_CACHE.get(name, [])

View file

@ -1,4 +1,4 @@
from typing import Any, Dict, List, Set from typing import Any, Dict, List
import icalendar import icalendar
import jinja2 import jinja2

View file

@ -172,7 +172,7 @@ class MetricQuery:
elif filterstate != 0: elif filterstate != 0:
raise ValueError('Unexpected EOF') raise ValueError('Unexpected EOF')
def __call__(self, metric: Metric): def __call__(self, metric: Metric) -> bool:
""" """
Applies the filter deducted from the query string to the given metric. Applies the filter deducted from the query string to the given metric.
@ -188,3 +188,7 @@ class MetricQuery:
return False return False
# Return True if all filters matched # Return True if all filters matched
return True return True
@property
def name(self) -> str:
return self._metric_name

View file

@ -0,0 +1,72 @@
from typing import Any, Dict, List
import icalendar
import jinja2
from datetime import datetime, timedelta
from icalendar_timeseries_server.config import get_config, get_jenv
from icalendar_timeseries_server.query import Metric
_ATTRIBUTES: List[str] = [
'class',
'description',
'geo',
'location',
'organizer',
'percent-complete',
'priority',
'status',
'summary',
'url',
'due',
'attach'
]
class Todo(Metric):
def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime):
self.calendar: str = cname
self.start = start
# self.attributes: Dict[str, str] = dict()
attributes: Dict[str, str] = dict()
tmp: Dict[str, Any] = {
'calendar': cname,
'start': start,
'end': end
}
for attr in _ATTRIBUTES:
tmp[attr] = todo.get(attr, '')
substitution_keys = set(_ATTRIBUTES)
substitution_keys.update(tmp.keys())
substitution_keys.update(get_config().key_replace.keys())
substitution_keys.update(get_config().value_replace.keys())
for attr in substitution_keys:
newkey: str = get_config().key_replace.get(attr, attr)
value: str = tmp.get(attr, '')
newval_template: str = get_config().value_replace.get(attr, str(value))
jtemplate: jinja2.Template = get_jenv().from_string(newval_template)
newvalue: str = jtemplate.render(**tmp)
attributes[newkey] = newvalue
self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}'
due = todo.get('due', None)
if due:
self.due = due.dt
else:
self.due = datetime.now(get_config().tz) + timedelta(days=36500)
self.priority = todo.get('priority', '0')
super().__init__('todo', attributes)
def serialize(self) -> Dict[str, Any]:
todo: Dict[str, Any] = {
'metric': {
'__name__': 'todo',
'calendar': self.calendar
},
'value': [
self.start.timestamp(),
1
]
}
todo['metric'].update(self._labels)
return todo