Rename project to icalendar-timeseries-server
This commit is contained in:
s3lph
commit
5722fd7c98
29 changed files with 1450 additions and 0 deletions
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
**/.idea/
|
||||
*.iml
|
||||
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
**/*.egg-info/
|
||||
*.coverage
|
||||
**/.mypy_cache/
|
||||
91
.gitlab-ci.yml
Normal file
91
.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
---
|
||||
image: s3lph/icalendar-timeseries-server-ci:20190820-01
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- release
|
||||
|
||||
|
||||
|
||||
before_script:
|
||||
- export ITS_VERSION=$(python -c 'import icalendar_timeseries_server; print(icalendar_timeseries_server.__version__)')
|
||||
|
||||
|
||||
|
||||
test:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- sudo -u its python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server
|
||||
- sudo -u its python3 -m coverage combine
|
||||
- sudo -u its python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
codestyle:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- sudo -u its pycodestyle icalendar_timeseries_server
|
||||
|
||||
|
||||
|
||||
build_wheel:
|
||||
stage: build
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- python3 setup.py egg_info bdist_wheel
|
||||
- cd dist
|
||||
- sha256sum *.whl > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "dist/*.whl"
|
||||
- dist/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
build_debian:
|
||||
stage: build
|
||||
script:
|
||||
# The Python package name provided by the python3-magic Debian package is "python-magic" rather than "file-magic".
|
||||
- sed -re 's/file-magic/python-magic/' -i setup.py
|
||||
- echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
- |
|
||||
for version in "$(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4)"; do
|
||||
echo "icalendar-timeseries-server (${version}-1); urgency=medium\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
done
|
||||
- gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
- python3.7 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1
|
||||
- cd package/debian
|
||||
- mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/
|
||||
- rsync -a icalendar-timeseries-server/usr/lib/python3.7/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/
|
||||
- rm -rf icalendar-timeseries-server/usr/lib/python3.7/
|
||||
- find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
|
||||
- find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
|
||||
- mv icalendar-timeseries-server/usr/bin/icalendar-timeseries-server icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server
|
||||
- rm -rf icalendar-timeseries-server/usr/bin
|
||||
- sed -re 's$#!/usr/local/bin/python3.7$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server
|
||||
- find icalendar-timeseries-server -type f -exec chmod 0644 {} \;
|
||||
- find icalendar-timeseries-server -type d -exec chmod 755 {} \;
|
||||
- find icalendar-timeseries-server -type f -name .gitkeep -delete
|
||||
- chmod +x icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server icalendar-timeseries-server/DEBIAN/postinst icalendar-timeseries-server/DEBIAN/prerm icalendar-timeseries-server/DEBIAN/postrm
|
||||
- dpkg-deb --build icalendar-timeseries-server
|
||||
- mv icalendar-timeseries-server.deb "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb"
|
||||
- sudo -u nobody lintian "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb"
|
||||
- sha256sum *.deb > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "package/debian/*.deb"
|
||||
- package/debian/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- python package/release.py
|
||||
only:
|
||||
- tags
|
||||
17
CHANGELOG.md
Normal file
17
CHANGELOG.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# iCalendar Timeseries Server Changelog
|
||||
|
||||
<!-- BEGIN RELEASE v0.1 -->
|
||||
## Version 0.1
|
||||
|
||||
First pre-1.0 release of iCalendar Timeseries Server.
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.1 -->
|
||||
- Configure multiple scrape sources
|
||||
- Authorization: HTTP Basic Auth or TLS Client Certificates
|
||||
- Offer API endpoints /api/v1/query and /api/v1/query_range
|
||||
- Support simple PromQL label filters
|
||||
<!-- END CHANGES 0.1 -->
|
||||
|
||||
<!-- END RELEASE v0.1 -->
|
||||
8
Dockerfile
Normal file
8
Dockerfile
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
FROM python:3.7-stretch
|
||||
|
||||
RUN useradd -d /home/its -m its \
|
||||
&& apt-get update -qy \
|
||||
&& apt-get install -y --no-install-recommends sudo build-essential lintian rsync \
|
||||
&& python3.7 -m pip install coverage wheel pycodestyle mypy
|
||||
|
||||
WORKDIR /home/its
|
||||
176
README.md
Normal file
176
README.md
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
# iCalendar Timeseries Server
|
||||
|
||||
This project is a small service that scrapes iCalendar files served
|
||||
over HTTP, parses their contents and returns the data in a timeseries
|
||||
format compatible to the `/api/v1/query` API endpoint of a Prometheus
|
||||
server. This allows e.g. a Grafana administrator to add a Prometheus
|
||||
data source pointing at this server, returning the events in the
|
||||
calendars in the `event` metric.
|
||||
|
||||
## Example
|
||||
|
||||
Consider the following iCalendar file:
|
||||
|
||||
```
|
||||
BEGIN:VCALENDAR
|
||||
VERSION:2.0
|
||||
PRODID:-//ACME//NONSGML Rocket Powered Anvil//EN
|
||||
BEGIN:VEVENT
|
||||
UID:20190603T032500CEST-foo
|
||||
SUMMARY:Foo
|
||||
DESCRIPTION:An example event
|
||||
DTSTART;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T032500
|
||||
DTEND;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T040000
|
||||
END:VEVENT
|
||||
BEGIN:VEVENT
|
||||
UID:20190603T032500CEST-bar
|
||||
SUMMARY:Bar
|
||||
DESCRIPTION:Another example event
|
||||
DTSTART;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T032500
|
||||
DTEND;TZID=Europe/Zurich;VALUE=DATE-TIME:20190603T040000
|
||||
END:VEVENT
|
||||
END:VCALENDAR
|
||||
```
|
||||
|
||||
The server would transform this into the following API response:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"resultType": "vector",
|
||||
"result": [
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "event",
|
||||
"calendar": "0",
|
||||
"uid": "20190603T032500CEST-foo",
|
||||
"summary": "Foo",
|
||||
"description": "An example event"
|
||||
},
|
||||
"value": [
|
||||
1560043497,
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric": {
|
||||
"__name__": "event",
|
||||
"calendar": "1",
|
||||
"uid": "20190603T032500CEST-bar",
|
||||
"summary": "Bar",
|
||||
"description": "Another example event"
|
||||
},
|
||||
"value": [
|
||||
1560043497,
|
||||
1
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `bottle`: Serve the results
|
||||
- `dateutil`: Work with recurring events
|
||||
- `icalendar`: Parse iCalendar
|
||||
- `isodate`: Parse ISO-8601 time periods
|
||||
- `jinja2`: Template value replacements
|
||||
- `pytz`: Work with timezones
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration is done through a JSON config file:
|
||||
|
||||
### Example
|
||||
|
||||
```json
|
||||
{
|
||||
"addr": "127.0.0.1",
|
||||
"port": 8090,
|
||||
"start_delta": "-PT3H",
|
||||
"end_delta": "P30D",
|
||||
"cache": "PT15M",
|
||||
"tz": "Europe/Zurich",
|
||||
"calendars": {
|
||||
"private": {
|
||||
"url": "https://example.cloud/dav/me/private.ics",
|
||||
"auth": {
|
||||
"type": "basic",
|
||||
"username": "me",
|
||||
"password": "mysupersecurepassword"
|
||||
}
|
||||
},
|
||||
"public": {
|
||||
"url": "https://example.cloud/dav/me/public.ics"
|
||||
},
|
||||
"confidential": {
|
||||
"url": "https://example.cloud/dav/me/confidential.ics",
|
||||
"ca": "/etc/ssl/ca.pem",
|
||||
"auth": {
|
||||
"type": "tls",
|
||||
"keyfile": "/etc/ssl/client.pem",
|
||||
"passphrase": "mysupersecurepassword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key_replace": {
|
||||
"summary": "01_summary",
|
||||
"description": "02_description"
|
||||
},
|
||||
"value_replace": {
|
||||
"summary": "{{ summary|truncate(100) }}",
|
||||
"description": "{{ description|truncate(100) }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Explanation
|
||||
|
||||
|
||||
| JMESPath | Type | Description |
|
||||
|----------|------|-------------|
|
||||
| `addr` | string | The address to listen on. |
|
||||
| `port` | int | The port to listen on. |
|
||||
| `start_delta` | string | A signed ISO 8601 duration string, describing the event range start offset relative to the current time. |
|
||||
| `end_delta` | string | An unsigned ISO 8601 duration string, describing the event range end offset relative to the current time. |
|
||||
| `cache` | string | An unsigned ISO 8601 duration string, describing the cache timeout duration. |
|
||||
| `tz` | string | The local timezone. |
|
||||
| `calendars` | dict | The calendars to scrape. |
|
||||
| `keys(calendars)` | string | Name of the calendar. |
|
||||
| `calendars.*.url` | string | The HTTP or HTTPS URL to scrape. |
|
||||
| `calendars.*.ca` | string | Path to the CA certificate file to validate the server's TLS certificate against, in PEM format (optional). |
|
||||
| `calendars.*.auth` | dict | Authorization config for the calendar. |
|
||||
| `calendars.*.auth[].type` | string | Authorization type, one of `none` (no authorization), `basic` (HTTP Basic Authentication), `tls` (TLS client certificate). |
|
||||
| `calendars.*.auth[?type=='basic'].username` | string | The Basic Auth username to authenticate with. |
|
||||
| `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. |
|
||||
| `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. |
|
||||
| `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). |
|
||||
| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. |
|
||||
| `keys(key_replace)` | string | The labels to rename. |
|
||||
| `key_replace.*` | string | The names to rename the labels to. |
|
||||
| `value_replace` | dict | Label values to postprocess. |
|
||||
| `keys(value_replace)` | string | Original label name to postprocess, also may introduce new labels. |
|
||||
| `value_replace.*` | string | The new value for the label. Supports Jinja2 templates. All original label values can be accessed by using their original name as a Jinja2 variable name. |
|
||||
|
||||
## Queries
|
||||
|
||||
The most basic query is simply the metric name:
|
||||
|
||||
```
|
||||
event
|
||||
```
|
||||
|
||||
In addition, PromQL label filters can be used.
|
||||
|
||||
```
|
||||
event{calendar="public",foo=~".*"}
|
||||
```
|
||||
|
||||
## Why Prometheus API
|
||||
|
||||
- It's JSON. A JSON generator is builtin in Python, so no further dependency.
|
||||
- The Prometheus Data Source is builtin into Grafana.
|
||||
- The API is simple.
|
||||
29
icalendar-timeseries-server.json
Normal file
29
icalendar-timeseries-server.json
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"addr": "127.0.0.1",
|
||||
"port": 8090,
|
||||
"start_delta": "-PT3H",
|
||||
"end_delta": "P60D",
|
||||
"cache": "PT3M",
|
||||
"tz": "Europe/Zurich",
|
||||
"calendars": {
|
||||
"tlstest": {
|
||||
"url": "https://localhost/private.ics",
|
||||
"ca": "/home/sebastian/tlstest/ca/ca/ca.crt",
|
||||
"auth": {
|
||||
"type": "tls",
|
||||
"keyfile": "/home/sebastian/tlstest/client/combined.pem"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key_replace": {
|
||||
"summary": "a_summary",
|
||||
"description": "b_description",
|
||||
"calendar": "c_calendar"
|
||||
},
|
||||
"value_replace": {
|
||||
"summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}",
|
||||
"description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}",
|
||||
"calendar": "{{ 0 if calendar == 'private' else 1 }}",
|
||||
"useless_metric": "{{ start.timestamp() + end.timestamp() }}"
|
||||
}
|
||||
}
|
||||
2
icalendar_timeseries_server/__init__.py
Normal file
2
icalendar_timeseries_server/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
__version__ = '0.1'
|
||||
4
icalendar_timeseries_server/__main__.py
Normal file
4
icalendar_timeseries_server/__main__.py
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
|
||||
from icalendar_timeseries_server.main import main
|
||||
|
||||
main()
|
||||
71
icalendar_timeseries_server/api.py
Normal file
71
icalendar_timeseries_server/api.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from typing import List
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from urllib.error import HTTPError
|
||||
import traceback
|
||||
|
||||
import bottle
|
||||
from isodate import Duration
|
||||
|
||||
from icalendar_timeseries_server.config import get_config
|
||||
from icalendar_timeseries_server.event import Event
|
||||
from icalendar_timeseries_server.cal import scrape_calendar
|
||||
from icalendar_timeseries_server.query import MetricQuery
|
||||
|
||||
|
||||
@bottle.route('/api/v1/query')
|
||||
@bottle.route('/api/v1/query_range')
|
||||
def prometheus_api():
|
||||
tz = get_config().tz
|
||||
now: datetime = datetime.now(tz)
|
||||
start_delta: Duration = get_config().start_delta
|
||||
end_delta: Duration = get_config().end_delta
|
||||
start: datetime = now + start_delta
|
||||
end: datetime = now + end_delta
|
||||
events: List[Event] = []
|
||||
|
||||
try:
|
||||
q = MetricQuery(bottle.request.query['query'])
|
||||
except ValueError as e:
|
||||
response = {
|
||||
'status': 'error',
|
||||
'errorType': 'bad_data',
|
||||
'error': str(e)
|
||||
}
|
||||
bottle.response.status = 400
|
||||
traceback.print_exc()
|
||||
bottle.response.add_header('Content-Type', 'application/json')
|
||||
return json.dumps(response)
|
||||
|
||||
try:
|
||||
for name, caldef in get_config().calendars.items():
|
||||
events.extend(scrape_calendar(name, caldef, start, end))
|
||||
events = list(filter(q, events))
|
||||
events.sort(key=lambda e: e.start)
|
||||
response = {
|
||||
'status': 'success',
|
||||
'data': {
|
||||
'resultType': 'vector',
|
||||
'result': [e.serialize() for e in events]
|
||||
}
|
||||
}
|
||||
except HTTPError as e:
|
||||
response = {
|
||||
'status': 'error',
|
||||
'errorType': 'internal',
|
||||
'error': str(e)
|
||||
}
|
||||
bottle.response.status = 500
|
||||
traceback.print_exc()
|
||||
except BaseException:
|
||||
response = {
|
||||
'status': 'error',
|
||||
'errorType': 'internal',
|
||||
'error': 'An internal error occurred.'
|
||||
}
|
||||
bottle.response.status = 500
|
||||
traceback.print_exc()
|
||||
|
||||
bottle.response.add_header('Content-Type', 'application/json')
|
||||
return json.dumps(response)
|
||||
92
icalendar_timeseries_server/cal.py
Normal file
92
icalendar_timeseries_server/cal.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
from typing import Dict, List, Iterable, Tuple
|
||||
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, date, timedelta
|
||||
|
||||
from dateutil import rrule
|
||||
from icalendar import cal
|
||||
|
||||
from icalendar_timeseries_server import __version__
|
||||
from icalendar_timeseries_server.config import get_config, CalendarConfig
|
||||
from icalendar_timeseries_server.event import Event
|
||||
|
||||
|
||||
_SCRAPE_CACHE: Dict[str, Tuple[datetime, List[Event]]] = dict()
|
||||
|
||||
__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
|
||||
USER_AGENT: str = f'icalendar-timeseries-server/{__version__} (Python/{__py_version})'
|
||||
|
||||
|
||||
def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration: timedelta) -> List[datetime]:
|
||||
occurences: List[datetime] = []
|
||||
|
||||
evstart = event.get('dtstart').dt
|
||||
# First occurence lies in the future; no need to process further
|
||||
if evstart >= end:
|
||||
return occurences
|
||||
|
||||
# Extract recurrence rules from ical
|
||||
ical_lines = event.to_ical().decode('utf-8').split('\r\n')
|
||||
recurrence = '\n'.join(
|
||||
[x for x in ical_lines
|
||||
if x.startswith('RRULE') or x.startswith('RDATE') or x.startswith('EXRULE') or x.startswith('EXDATE')])
|
||||
# Create a generator that yields a timestamp for each recurrence
|
||||
generator = rrule.rrulestr(recurrence, dtstart=evstart)
|
||||
|
||||
# Generate an event entry for each occurence of the event
|
||||
for dt in generator:
|
||||
# Skip past occurences and break once the the event lies too far in the future
|
||||
if dt + duration < start:
|
||||
continue
|
||||
if dt > end:
|
||||
break
|
||||
# Create an event entry
|
||||
occurences.append(dt)
|
||||
return occurences
|
||||
|
||||
|
||||
def _parse_calendar(name: str, calendar: cal.Calendar, start: datetime, end: datetime) -> List[Event]:
|
||||
events = []
|
||||
for element in calendar.walk():
|
||||
if element.name == "VEVENT":
|
||||
dtstart = element.get('dtstart').dt
|
||||
if isinstance(dtstart, date):
|
||||
dtstart = datetime(dtstart.year, dtstart.month, dtstart.day, tzinfo=start.tzinfo)
|
||||
# Process either end timestamp or duration, if present
|
||||
if 'dtend' in element:
|
||||
evend = element.get('dtend').dt
|
||||
if isinstance(evend, date):
|
||||
evend = datetime(evend.year, evend.month, evend.day, tzinfo=start.tzinfo)
|
||||
duration = evend - dtstart
|
||||
elif 'duration' in element:
|
||||
duration = element.get('duration').dt
|
||||
else:
|
||||
duration = timedelta(0)
|
||||
if element.get('rrule') is not None or element.get('rdate') is not None:
|
||||
occurences: Iterable[datetime] = _parse_recurring(element, start, end, duration)
|
||||
else:
|
||||
occurences = [dtstart]
|
||||
for occurence in occurences:
|
||||
if start <= occurence < end:
|
||||
events.append(Event(name, element, occurence, occurence + duration))
|
||||
return events
|
||||
|
||||
|
||||
def scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime) -> List[Event]:
|
||||
global _SCRAPE_CACHE
|
||||
now: datetime = datetime.now(tz=get_config().tz)
|
||||
if get_config().cache.total_seconds() > 0 and name in _SCRAPE_CACHE:
|
||||
cache_timeout, cached = _SCRAPE_CACHE[name]
|
||||
if now < cache_timeout:
|
||||
print('serving cached')
|
||||
return cached
|
||||
print('doing request')
|
||||
|
||||
opener: urllib.request.OpenerDirector = config.get_url_opener()
|
||||
with opener.open(config.url) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
calendar = cal.Calendar.from_ical(data)
|
||||
parsed: List[Event] = _parse_calendar(name, calendar, start, end)
|
||||
_SCRAPE_CACHE[name] = now + get_config().cache, parsed
|
||||
return parsed
|
||||
228
icalendar_timeseries_server/config.py
Normal file
228
icalendar_timeseries_server/config.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
from typing import Any, Dict, List, Optional, Type, Union
|
||||
|
||||
import base64
|
||||
import json
|
||||
from datetime import timedelta
|
||||
import ssl
|
||||
import urllib.request
|
||||
import sys
|
||||
|
||||
import pytz
|
||||
import jinja2
|
||||
from isodate import Duration, parse_duration
|
||||
|
||||
from icalendar_timeseries_server import __version__
|
||||
|
||||
|
||||
CONFIG: Optional['Config'] = None
|
||||
|
||||
JENV: Optional[jinja2.Environment] = None
|
||||
|
||||
|
||||
__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
|
||||
USER_AGENT: str = f'icalendar-timeseries-server/{__version__} (Python/{__py_version})'
|
||||
|
||||
|
||||
class CalendarConfig:
|
||||
|
||||
def __init__(self, config: Dict[str, Any], config_path: str) -> None:
|
||||
self._url: str = _keycheck('url', config, str, config_path)
|
||||
self._ca: Optional[str] = _keycheck('ca', config, str, config_path, optional=True)
|
||||
auth: Dict[str, Any] = _keycheck('auth', config, dict, config_path, default_value={'type': 'none'})
|
||||
self._authtype: str = _keycheck('type', auth, str, f'{config_path}.auth',
|
||||
default_value='none',
|
||||
valid_values=['none', 'basic', 'tls'])
|
||||
self._request_headers: Dict[str, str] = {
|
||||
'User-Agent': USER_AGENT
|
||||
}
|
||||
|
||||
if self._authtype == 'none':
|
||||
# No auth, nothing to do
|
||||
pass
|
||||
|
||||
elif self._authtype == 'basic':
|
||||
# Build the Authorization header required for HTTP Basic Auth
|
||||
username: str = _keycheck('username', auth, str, f'{config_path}.auth')
|
||||
password: str = _keycheck('password', auth, str, f'{config_path}.auth')
|
||||
token: str = base64.b64encode(f'{username}:{password}'.encode()).decode()
|
||||
self._request_headers['Authorization'] = f'Basic {token}'
|
||||
|
||||
elif self._authtype == 'tls':
|
||||
# Store TLS config for later use
|
||||
self._tls_keyfile: str = _keycheck('keyfile', auth, str, f'{config_path}.auth')
|
||||
self._tls_passphrase: str = _keycheck('passphrase', auth, str, f'{config_path}.auth', optional=True)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self._url
|
||||
|
||||
def get_url_opener(self) -> urllib.request.OpenerDirector:
|
||||
|
||||
if self._authtype == 'tls':
|
||||
# Create an OpenSSL context and load the CA and client certificates
|
||||
context: ssl.SSLContext = ssl.create_default_context(cafile=self._ca)
|
||||
context.load_cert_chain(certfile=self._tls_keyfile, password=self._tls_passphrase)
|
||||
opener: urllib.request.OpenerDirector = \
|
||||
urllib.request.build_opener(urllib.request.HTTPSHandler(context=context))
|
||||
|
||||
elif self._ca is not None:
|
||||
# No TLS Client Certificate, but load the CA certificate
|
||||
context: ssl.SSLContext = ssl.create_default_context(cafile=self._ca)
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=context))
|
||||
|
||||
else:
|
||||
# No TLS config, return default opener
|
||||
opener = urllib.request.build_opener()
|
||||
|
||||
# Set request headers, at least User-Agent, optionally Authorization
|
||||
opener.addheaders = [(k, v) for k, v in self._request_headers.items()]
|
||||
return opener
|
||||
|
||||
|
||||
class Config:
|
||||
|
||||
def __init__(self, config: Dict[str, Any] = None) -> None:
|
||||
if config is None:
|
||||
config = dict()
|
||||
self._addr: str = _keycheck('addr', config, str, '', default_value='127.0.0.1')
|
||||
self._port: int = _keycheck('port', config, int, '', default_value=8090)
|
||||
self._tz: pytz.tzinfo = _parse_timezone('tz', config, '', default_value='UTC')
|
||||
self._start_delta: Duration = _parse_timedelta('start_delta', config, '', default_value='PT')
|
||||
self._end_delta: Duration = _parse_timedelta('end_delta', config, '', default_value='P30D')
|
||||
self._cache: Duration = _parse_timedelta('cache', config, '', default_value='PT', force_positive=True)
|
||||
self._calendars: Dict[str, CalendarConfig] = self._parse_calendars_config('calendars', config, '')
|
||||
self._key_replace = _parse_key_replace('key_replace', config, '')
|
||||
self._value_replace = _parse_value_replace('value_replace', config, '')
|
||||
|
||||
@staticmethod
|
||||
def _parse_calendars_config(key: str,
|
||||
config: Dict[str, Any],
|
||||
path: str) -> Dict[str, CalendarConfig]:
|
||||
cdef: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={})
|
||||
calendars: Dict[str, CalendarConfig] = dict()
|
||||
for name, c in cdef.items():
|
||||
calendar: CalendarConfig = CalendarConfig(c, f'{path}.{name}')
|
||||
calendars[name] = calendar
|
||||
return calendars
|
||||
|
||||
@property
|
||||
def addr(self) -> str:
|
||||
return self._addr
|
||||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return self._port
|
||||
|
||||
@property
|
||||
def tz(self) -> pytz.tzinfo:
|
||||
return self._tz
|
||||
|
||||
@property
|
||||
def start_delta(self) -> Duration:
|
||||
return self._start_delta
|
||||
|
||||
@property
|
||||
def end_delta(self) -> Duration:
|
||||
return self._end_delta
|
||||
|
||||
@property
|
||||
def cache(self) -> Duration:
|
||||
return self._cache
|
||||
|
||||
@property
|
||||
def calendars(self) -> Dict[str, CalendarConfig]:
|
||||
return self._calendars
|
||||
|
||||
@property
|
||||
def key_replace(self) -> Dict[str, str]:
|
||||
return self._key_replace
|
||||
|
||||
@property
|
||||
def value_replace(self) -> Dict[str, str]:
|
||||
return self._value_replace
|
||||
|
||||
|
||||
def _keycheck(key: str,
|
||||
config: Dict[str, Any],
|
||||
typ: Type,
|
||||
path: str,
|
||||
default_value: Any = None,
|
||||
optional: bool = False,
|
||||
valid_values: Optional[List[Any]] = None) -> Any:
|
||||
if key not in config:
|
||||
if default_value is not None or optional:
|
||||
return default_value
|
||||
raise KeyError(f'Expected member "{key}" not found at path {path}')
|
||||
value: Any = config[key]
|
||||
if not isinstance(value, typ):
|
||||
raise TypeError(f'Expected {typ}, not {type(value).__name__} for path {path}.{key}')
|
||||
if valid_values is not None:
|
||||
if value not in valid_values:
|
||||
raise ValueError(f'Expected one of {", ".join(valid_values)} ({typ}), not {value} for path {path}.{key}')
|
||||
return value
|
||||
|
||||
|
||||
def _parse_timedelta(key: str,
|
||||
config: Dict[str, Any],
|
||||
path: str,
|
||||
default_value: Any = None,
|
||||
force_positive: bool = False) -> Duration:
|
||||
tdstring: str = _keycheck(key, config, str, path, default_value=default_value)
|
||||
duration: Union[Duration, timedelta] = parse_duration(tdstring)
|
||||
if isinstance(duration, timedelta):
|
||||
duration = Duration(days=duration.days,
|
||||
seconds=duration.seconds,
|
||||
microseconds=duration.microseconds)
|
||||
if force_positive and duration.total_seconds() < 0:
|
||||
raise ValueError(f'Duration must be positive for path {path}.{key}')
|
||||
return duration
|
||||
|
||||
|
||||
def _parse_timezone(key: str,
|
||||
config: Dict[str, Any],
|
||||
path: str,
|
||||
default_value: Any = None) -> Any:
|
||||
zonename: str = _keycheck(key, config, str, path, default_value=default_value)
|
||||
return pytz.timezone(zonename)
|
||||
|
||||
|
||||
def _parse_key_replace(key: str,
|
||||
config: Dict[str, Any],
|
||||
path: str) -> Dict[str, str]:
|
||||
key_replace: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={})
|
||||
for label in key_replace.keys():
|
||||
_keycheck(label, key_replace, str, f'{path}.{key}')
|
||||
return key_replace
|
||||
|
||||
|
||||
def _parse_value_replace(key: str,
|
||||
config: Dict[str, Any],
|
||||
path: str) -> Dict[str, str]:
|
||||
value_replace: Dict[str, Any] = _keycheck(key, config, dict, path, default_value={})
|
||||
for label in value_replace.keys():
|
||||
_keycheck(label, value_replace, str, f'{path}.{key}')
|
||||
return value_replace
|
||||
|
||||
|
||||
def get_config() -> Config:
|
||||
global CONFIG
|
||||
return CONFIG
|
||||
|
||||
|
||||
def get_jenv() -> jinja2.Environment:
|
||||
global JENV
|
||||
return JENV
|
||||
|
||||
|
||||
def load_config(filename: str):
|
||||
global CONFIG, JENV
|
||||
with open(filename, 'r') as f:
|
||||
json_config = json.loads(f.read())
|
||||
CONFIG = Config(json_config)
|
||||
JENV = jinja2.Environment()
|
||||
|
||||
|
||||
def load_default_config():
|
||||
global CONFIG, JENV
|
||||
CONFIG = Config({})
|
||||
JENV = jinja2.Environment()
|
||||
66
icalendar_timeseries_server/event.py
Normal file
66
icalendar_timeseries_server/event.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
from typing import Any, Dict, List, Set
|
||||
|
||||
import icalendar
|
||||
import jinja2
|
||||
from datetime import datetime
|
||||
|
||||
from icalendar_timeseries_server.config import get_config, get_jenv
|
||||
from icalendar_timeseries_server.query import Metric
|
||||
|
||||
_ATTRIBUTES: List[str] = [
|
||||
'attachs',
|
||||
'categories',
|
||||
'class',
|
||||
'comment',
|
||||
'description',
|
||||
'geo',
|
||||
'location',
|
||||
'percent-complete',
|
||||
'priority',
|
||||
'resources',
|
||||
'status',
|
||||
'summary'
|
||||
]
|
||||
|
||||
|
||||
class Event(Metric):
|
||||
|
||||
def __init__(self, cname: str, event: icalendar.cal.Event, start: datetime, end: datetime):
|
||||
self.start: datetime = start
|
||||
self.calendar: str = cname
|
||||
# self.attributes: Dict[str, str] = dict()
|
||||
attributes: Dict[str, str] = dict()
|
||||
tmp: Dict[str, Any] = {
|
||||
'calendar': cname,
|
||||
'start': start,
|
||||
'end': end
|
||||
}
|
||||
for attr in _ATTRIBUTES:
|
||||
tmp[attr] = event.get(attr, '')
|
||||
substitution_keys = set(_ATTRIBUTES)
|
||||
substitution_keys.update(['start', 'end'])
|
||||
substitution_keys.update(get_config().key_replace.keys())
|
||||
substitution_keys.update(get_config().value_replace.keys())
|
||||
for attr in substitution_keys:
|
||||
newkey: str = get_config().key_replace.get(attr, attr)
|
||||
value: str = tmp.get(attr, '')
|
||||
newval_template: str = get_config().value_replace.get(attr, str(value))
|
||||
jtemplate: jinja2.Template = get_jenv().from_string(newval_template)
|
||||
newvalue: str = jtemplate.render(**tmp)
|
||||
attributes[newkey] = newvalue
|
||||
self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}'
|
||||
super().__init__('event', attributes)
|
||||
|
||||
def serialize(self) -> Dict[str, Any]:
|
||||
event: Dict[str, Any] = {
|
||||
'metric': {
|
||||
'__name__': 'event',
|
||||
'calendar': self.calendar
|
||||
},
|
||||
'value': [
|
||||
self.start.timestamp(),
|
||||
1
|
||||
]
|
||||
}
|
||||
event['metric'].update(self._labels)
|
||||
return event
|
||||
24
icalendar_timeseries_server/main.py
Normal file
24
icalendar_timeseries_server/main.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import sys
|
||||
|
||||
import bottle
|
||||
|
||||
from icalendar_timeseries_server.config import load_config, load_default_config, get_config
|
||||
|
||||
# Contains decorated bottle handler function for /api/v1/query
|
||||
# noinspection PyUnresolvedReferences
|
||||
from icalendar_timeseries_server.api import prometheus_api
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) == 1:
|
||||
load_default_config()
|
||||
elif len(sys.argv) == 2:
|
||||
load_config(sys.argv[1])
|
||||
else:
|
||||
print(f'Can only read one config file, got "{" ".join(sys.argv[1:])}"')
|
||||
exit(1)
|
||||
bottle.run(host=get_config().addr, port=get_config().port)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
189
icalendar_timeseries_server/query.py
Normal file
189
icalendar_timeseries_server/query.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
from typing import Dict
|
||||
|
||||
import re
|
||||
|
||||
LABEL_MATCH_OPERATORS = [
|
||||
'=',
|
||||
'!=',
|
||||
'=~',
|
||||
'!~'
|
||||
]
|
||||
|
||||
|
||||
class Metric:
|
||||
|
||||
def __init__(self, name: str, labels: Dict[str, str]):
|
||||
self._name: str = name
|
||||
self._labels: Dict[str, str] = dict()
|
||||
for k, v in labels.items():
|
||||
self._labels[k] = v
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def keys(self):
|
||||
return self._labels.keys()
|
||||
|
||||
def __getitem__(self, item: str) -> str:
|
||||
return self._labels.get(item, None)
|
||||
|
||||
def __getattr__(self, item: str) -> str:
|
||||
return self._labels.get(item, None)
|
||||
|
||||
|
||||
class LabelFilter:
|
||||
|
||||
def __init__(self, op: str, label: str, value: str):
|
||||
if op not in LABEL_MATCH_OPERATORS:
|
||||
raise ValueError()
|
||||
self._op = op
|
||||
self._label = label
|
||||
self._value = value
|
||||
|
||||
def __call__(self, metric: Metric) -> bool:
|
||||
if self._label not in metric.keys():
|
||||
return False
|
||||
if self._op == '=':
|
||||
return metric[self._label] == self._value
|
||||
elif self._op == '!=':
|
||||
return metric[self._label] != self._value
|
||||
elif self._op == '=~':
|
||||
return re.match(self._value, metric[self._label]) is not None
|
||||
elif self._op == '!~':
|
||||
return re.match(self._value, metric[self._label]) is None
|
||||
|
||||
|
||||
class MetricQuery:
|
||||
|
||||
def __init__(self, q: str) -> None:
|
||||
self._filters = []
|
||||
# The query '1+1' is used by Grafana to test the connection.
|
||||
if q == "1+1":
|
||||
return
|
||||
self.__parse(q)
|
||||
|
||||
def __parse(self, q: str):
|
||||
print(q)
|
||||
# globalstate:
|
||||
# 0 = parsing metric name
|
||||
# 1 = parsing filters
|
||||
# 2 = fully parsed query
|
||||
globalstate = 0
|
||||
self._metric_name = ''
|
||||
# buffer for the name of the filter that is being parsed
|
||||
cfiltername = ''
|
||||
# buffer for the filter operator that is being parsed
|
||||
coperator = ''
|
||||
# buffer for the filter value that is being parsed
|
||||
cfiltervalue = ''
|
||||
# filterstate:
|
||||
# 0 = parsing label name
|
||||
# 1 = parsing operator
|
||||
# 2 = parsing filter value
|
||||
filterstate = 0
|
||||
# Whether currently inside a quoted string
|
||||
quoted = False
|
||||
# Whether the last character started an escape sequence
|
||||
escaping = False
|
||||
# query string index
|
||||
i = 0
|
||||
while i < len(q):
|
||||
c = q[i]
|
||||
i += 1
|
||||
if globalstate == 0:
|
||||
# Parsing the metric name: append the character to the name buffer,
|
||||
# or move on to the filter parsing step when "{" is encountered.
|
||||
if c == '{':
|
||||
globalstate = 1
|
||||
else:
|
||||
self._metric_name += c
|
||||
elif globalstate == 1:
|
||||
# Parsing the query's filters, until an unescaped "}" is encountered.
|
||||
if filterstate == 0:
|
||||
# Parsing the filter's label name.
|
||||
# When a character is not accepted, re-loop over it in the next mode.
|
||||
if c in '_: 0123456789 ' or ('a' <= c <= 'z') or ('A' <= c <= 'Z'):
|
||||
cfiltername += c
|
||||
else:
|
||||
filterstate = 1
|
||||
i -= 1
|
||||
elif filterstate == 1:
|
||||
# Parsing the filter's operator, one of =, !=, =~, !~.
|
||||
# When a character is not accepted, re-loop over it in the next mode.
|
||||
if c in '!~= ':
|
||||
coperator += c
|
||||
else:
|
||||
filterstate = 2
|
||||
i -= 1
|
||||
elif filterstate == 2:
|
||||
# Parsing the filter's value.
|
||||
# When a "," is encountered, start with the next filter. When a "}" is encountered, parsing is done.
|
||||
if not quoted:
|
||||
if c.strip() == '':
|
||||
continue
|
||||
if c == '"':
|
||||
quoted = True
|
||||
elif c == ',' or c == '}':
|
||||
# Done parsing the filter. Validate label name and operator.
|
||||
if coperator not in LABEL_MATCH_OPERATORS:
|
||||
raise ValueError(f'Invalid operator: {coperator.strip()}')
|
||||
if not re.match('^[a-zA-Z_:][a-zA-Z0-9_:]*$', cfiltername.strip()):
|
||||
raise ValueError(f'Invalid label name: {cfiltername.strip()}')
|
||||
self._filters.append(LabelFilter(coperator.strip(), cfiltername.strip(), cfiltervalue))
|
||||
# Reset filter buffers
|
||||
cfiltername = ''
|
||||
coperator = ''
|
||||
cfiltervalue = ''
|
||||
filterstate = 0
|
||||
if c == '}':
|
||||
globalstate = 2
|
||||
else:
|
||||
raise ValueError()
|
||||
else:
|
||||
# Quoting and escaping
|
||||
if c == '"' and not escaping:
|
||||
quoted = False
|
||||
else:
|
||||
if escaping:
|
||||
escaping = False
|
||||
cfiltervalue += c
|
||||
else:
|
||||
if c == '\\':
|
||||
escaping = True
|
||||
else:
|
||||
cfiltervalue += c
|
||||
else:
|
||||
if c.strip() == '':
|
||||
# Trim remaining whitespace
|
||||
continue
|
||||
else:
|
||||
raise ValueError('Garbage after metric query')
|
||||
# Match metric name against the regex from the Prometheus documentation
|
||||
if not re.match('^[a-zA-Z_:][a-zA-Z0-9_:]*$', self._metric_name.strip()):
|
||||
raise ValueError(f'Invalid metric name: {self._metric_name.strip()}')
|
||||
if escaping:
|
||||
raise ValueError('Expected escape sequence, got EOF')
|
||||
elif quoted:
|
||||
raise ValueError('Expected \'\"\', got EOF')
|
||||
elif globalstate == 1:
|
||||
raise ValueError('Expected \'}\', got EOF')
|
||||
elif filterstate != 0:
|
||||
raise ValueError('Unexpected EOF')
|
||||
|
||||
def __call__(self, metric: Metric):
|
||||
"""
|
||||
Applies the filter deducted from the query string to the given metric.
|
||||
|
||||
:param metric: The metric to apply the filter to.
|
||||
:return: True if the filter matches, False otherwise.
|
||||
"""
|
||||
# Never match a metric with a different name
|
||||
if metric.name != self._metric_name:
|
||||
return False
|
||||
# Iterate the single filters, return False if even one does not match
|
||||
for f in self._filters:
|
||||
if not f(metric):
|
||||
return False
|
||||
# Return True if all filters matched
|
||||
return True
|
||||
0
icalendar_timeseries_server/test/__init__.py
Normal file
0
icalendar_timeseries_server/test/__init__.py
Normal file
128
icalendar_timeseries_server/test/test_config.py
Normal file
128
icalendar_timeseries_server/test/test_config.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
|
||||
import unittest
|
||||
|
||||
import json
|
||||
import pytz
|
||||
from datetime import timedelta
|
||||
|
||||
from isodate.duration import Duration
|
||||
|
||||
from icalendar_timeseries_server.config import _keycheck, _parse_timedelta, _parse_timezone, Config
|
||||
|
||||
|
||||
_CONFIG_VALID = """
|
||||
{
|
||||
"addr": "127.0.0.1",
|
||||
"port": 8090,
|
||||
"start_delta": "-PT3H",
|
||||
"end_delta": "P30D",
|
||||
"cache": "PT15M",
|
||||
"tz": "Europe/Zurich",
|
||||
"calendars": {
|
||||
"private": {
|
||||
"url": "https://example.cloud/dav/me/private.ics",
|
||||
"auth": {
|
||||
"type": "basic",
|
||||
"username": "me",
|
||||
"password": "mysupersecurepassword"
|
||||
}
|
||||
},
|
||||
"public": {
|
||||
"url": "https://example.cloud/dav/me/public.ics"
|
||||
},
|
||||
"confidential": {
|
||||
"url": "https://example.cloud/dav/me/confidential.ics",
|
||||
"ca": "/etc/ssl/ca.pem",
|
||||
"auth": {
|
||||
"type": "tls",
|
||||
"keyfile": "/etc/ssl/client.pem",
|
||||
"passphrase": "mysupersecurepassword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"key_replace": {
|
||||
"summary": "01_summary",
|
||||
"description": "02_description"
|
||||
},
|
||||
"value_replace": {
|
||||
"summary": "{{ summary|truncate(100) }}",
|
||||
"description": "{{ description|truncate(100) }}"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class ConfigTest(unittest.TestCase):
|
||||
|
||||
def test_keycheck_valid(self) -> None:
|
||||
config = {
|
||||
'foo': 'bar',
|
||||
'bar': 42,
|
||||
'baz': [],
|
||||
'qux': {}
|
||||
}
|
||||
strvalue = _keycheck('foo', config, str, '', valid_values=['foo', 'bar', 'baz'])
|
||||
self.assertEqual(strvalue, 'bar')
|
||||
strdefvalue = _keycheck('fooo', config, str, '', default_value='baar')
|
||||
self.assertEqual(strdefvalue, 'baar')
|
||||
stroptvalue = _keycheck('foooo', config, str, '', optional=True)
|
||||
self.assertIsNone(stroptvalue)
|
||||
intvalue = _keycheck('bar', config, int, '')
|
||||
self.assertEqual(intvalue, 42)
|
||||
listvalue = _keycheck('baz', config, list, '')
|
||||
self.assertEqual(listvalue, [])
|
||||
objvalue = _keycheck('qux', config, dict, '')
|
||||
self.assertEqual(objvalue, {})
|
||||
|
||||
def test_keycheck_missing(self) -> None:
|
||||
config = {
|
||||
'foo': 'bar',
|
||||
}
|
||||
with self.assertRaises(KeyError):
|
||||
_keycheck('baz', config, str, '')
|
||||
|
||||
def test_keycheck_type(self) -> None:
|
||||
config = {
|
||||
'foo': 'bar',
|
||||
}
|
||||
with self.assertRaises(TypeError):
|
||||
_keycheck('foo', config, int, '')
|
||||
|
||||
def test_keycheck_value(self) -> None:
|
||||
config = {
|
||||
'foo': '1337',
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
_keycheck('foo', config, str, '', valid_values=['foo', 'bar', 'baz'])
|
||||
|
||||
def test_parse_timedelta(self) -> None:
|
||||
config = {
|
||||
'pos': 'PT42S',
|
||||
'neg': '-PT42S'
|
||||
}
|
||||
self.assertEqual(_parse_timedelta('zero', config, '', default_value='PT'), timedelta(seconds=0))
|
||||
self.assertEqual(_parse_timedelta('pos', config, ''), timedelta(seconds=42))
|
||||
self.assertEqual(_parse_timedelta('neg', config, ''), timedelta(seconds=-42))
|
||||
self.assertEqual(_parse_timedelta('pos', config, '', force_positive=True), timedelta(seconds=42))
|
||||
with self.assertRaises(ValueError):
|
||||
_parse_timedelta('neg', config, '', force_positive=True)
|
||||
|
||||
def test_parse_timezone(self) -> None:
|
||||
config = {
|
||||
'tz': 'Europe/Zurich',
|
||||
'notz': 'North/Winterfell'
|
||||
}
|
||||
self.assertEqual(_parse_timezone('tz', config, ''), pytz.timezone('Europe/Zurich'))
|
||||
self.assertEqual(_parse_timezone('def', config, '', default_value='Europe/Berlin'),
|
||||
pytz.timezone('Europe/Berlin'))
|
||||
with self.assertRaises(pytz.exceptions.UnknownTimeZoneError):
|
||||
_parse_timezone('notz', config, '')
|
||||
|
||||
def test_parse_full_config_valid(self):
|
||||
config = Config(json.loads(_CONFIG_VALID))
|
||||
self.assertEqual(config.addr, '127.0.0.1')
|
||||
self.assertEqual(config.port, 8090)
|
||||
self.assertEqual(config.start_delta, Duration(hours=-3))
|
||||
self.assertEqual(config.end_delta, Duration(days=30))
|
||||
self.assertEqual(config.cache, Duration(minutes=15))
|
||||
self.assertEqual(config.tz, pytz.timezone('Europe/Zurich'))
|
||||
|
|
@ -0,0 +1 @@
|
|||
/etc/icalendar-timeseries-server.json
|
||||
14
package/debian/icalendar-timeseries-server/DEBIAN/control
Normal file
14
package/debian/icalendar-timeseries-server/DEBIAN/control
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
Package: icalendar-timeseries-server
|
||||
Version: 0.1
|
||||
Maintainer: s3lph <account-gitlab-ideynizv@kernelpanic.lol>
|
||||
Section: web
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate, python3-tz
|
||||
Description: Scrape iCalendar endpoints and present their data in a
|
||||
timeseries format. A small service that scrapes iCalendar files
|
||||
served over HTTP, parses their contents and returns a timeseries
|
||||
format compatible to the /api/v1/query API endpoint of a Prometheus
|
||||
server. This allows e.g. a Grafana administrator to add a Prometheus
|
||||
data source pointing at this server, returning the events in the
|
||||
calendars in the event metric.
|
||||
20
package/debian/icalendar-timeseries-server/DEBIAN/postinst
Executable file
20
package/debian/icalendar-timeseries-server/DEBIAN/postinst
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "configure" ]]; then
|
||||
|
||||
if ! getent group its >/dev/null; then
|
||||
groupadd --system its
|
||||
fi
|
||||
|
||||
if ! getent passwd its >/dev/null; then
|
||||
useradd --system --create-home --gid its --home-dir /var/lib/its --shell /usr/sbin/nologin its
|
||||
fi
|
||||
|
||||
chown its:its /var/lib/its
|
||||
chmod 0750 /var/lib/its
|
||||
|
||||
systemctl daemon-reload || true
|
||||
|
||||
fi
|
||||
9
package/debian/icalendar-timeseries-server/DEBIAN/postrm
Executable file
9
package/debian/icalendar-timeseries-server/DEBIAN/postrm
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "remove" ]]; then
|
||||
|
||||
systemctl daemon-reload || true
|
||||
|
||||
fi
|
||||
9
package/debian/icalendar-timeseries-server/DEBIAN/prerm
Executable file
9
package/debian/icalendar-timeseries-server/DEBIAN/prerm
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$1" == "remove" ]]; then
|
||||
|
||||
userdel its
|
||||
|
||||
fi
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"addr": "127.0.0.1",
|
||||
"port": 8090,
|
||||
"start_delta": "-PT3H",
|
||||
"end_delta": "P60D",
|
||||
"cache": "PT3M",
|
||||
"tz": "Europe/Zurich",
|
||||
"calendars": {},
|
||||
"key_replace": {
|
||||
"summary": "01_summary",
|
||||
"description": "02_description",
|
||||
"calendar": "03_calendar"
|
||||
},
|
||||
"value_replace": {
|
||||
"summary": "{{ summary|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}",
|
||||
"description": "{{ description|truncate(100, end=' \\N{HORIZONTAL ELLIPSIS}') }}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[Unit]
|
||||
Description=icalendar-timeseries-server
|
||||
After=networking.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/python3 -m icalendar_timeseries_server /etc/icalendar-timeseries-server.json
|
||||
User=its
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
Copyright 2019 s3lph
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
||||
associated documentation files (the "Software"), to deal in the Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
||||
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or
|
||||
substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
||||
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
||||
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
157
package/release.py
Executable file
157
package/release.py
Executable file
|
|
@ -0,0 +1,157 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib.request
|
||||
import http.client
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def parse_changelog(tag: str) -> Optional[str]:
|
||||
release_changelog: str = ''
|
||||
with open('CHANGELOG.md', 'r') as f:
|
||||
in_target: bool = False
|
||||
done: bool = False
|
||||
for line in f.readlines():
|
||||
if in_target:
|
||||
if f'<!-- END RELEASE {tag} -->' in line:
|
||||
done = True
|
||||
break
|
||||
release_changelog += line
|
||||
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
|
||||
in_target = True
|
||||
continue
|
||||
if not done:
|
||||
return None
|
||||
return release_changelog
|
||||
|
||||
|
||||
def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]:
|
||||
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.read()
|
||||
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
|
||||
|
||||
jobidmap: Dict[str, str] = {}
|
||||
for job in joblist:
|
||||
name: str = job['name']
|
||||
job_id: str = job['id']
|
||||
jobidmap[name] = job_id
|
||||
return jobidmap
|
||||
|
||||
|
||||
def fetch_single_shafile(url: str) -> str:
|
||||
req = urllib.request.Request(url)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.readline()
|
||||
shafile: str = resp_data.decode()
|
||||
filename: str = shafile.strip().split(' ')[-1].strip()
|
||||
return filename
|
||||
|
||||
|
||||
def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
|
||||
wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS'
|
||||
wheel_filename: str = fetch_single_shafile(wheel_sha_url)
|
||||
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
|
||||
return wheel_url, wheel_sha_url
|
||||
|
||||
|
||||
def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
|
||||
debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS'
|
||||
debian_filename: str = fetch_single_shafile(debian_sha_url)
|
||||
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
|
||||
return debian_url, debian_sha_url
|
||||
|
||||
|
||||
def main():
|
||||
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
|
||||
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
|
||||
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
|
||||
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
|
||||
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
|
||||
if api_token is None:
|
||||
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if release_tag is None:
|
||||
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_name is None:
|
||||
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_id is None:
|
||||
print('CI_PROJECT_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if pipeline_id is None:
|
||||
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
changelog: Optional[str] = parse_changelog(release_tag)
|
||||
if changelog is None:
|
||||
print('Changelog could not be parsed.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
|
||||
|
||||
base_url: str = f'https://gitlab.com/{project_name}/-'
|
||||
|
||||
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids)
|
||||
debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids)
|
||||
|
||||
augmented_changelog = f'''{changelog.strip()}
|
||||
|
||||
### Download
|
||||
|
||||
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
|
||||
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
|
||||
|
||||
post_body: str = json.dumps({'description': augmented_changelog})
|
||||
|
||||
gitlab_release_api_url: str = \
|
||||
f'https://gitlab.com/api/v4/projects/{project_id}/repository/tags/{release_tag}/release'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'Content-Type': 'application/json; charset=utf-8'
|
||||
}
|
||||
|
||||
request = urllib.request.Request(
|
||||
gitlab_release_api_url,
|
||||
post_body.encode('utf-8'),
|
||||
headers=headers,
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
response: http.client.HTTPResponse = urllib.request.urlopen(request)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
response_bytes: bytes = response.read()
|
||||
response_str: str = response_bytes.decode()
|
||||
response_data: Dict[str, Any] = json.loads(response_str)
|
||||
|
||||
if response_data['tag_name'] != release_tag:
|
||||
print('Something went wrong...', file=sys.stderr)
|
||||
print(response_str, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(response_data['description'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1
requirements.txt
Normal file
1
requirements.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
.
|
||||
22
setup.cfg
Normal file
22
setup.cfg
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
|
||||
#
|
||||
# PyCodestyle
|
||||
#
|
||||
|
||||
[pycodestyle]
|
||||
max-line-length = 120
|
||||
statistics = True
|
||||
|
||||
#
|
||||
# Coverage
|
||||
#
|
||||
|
||||
[run]
|
||||
branch = True
|
||||
parallel = True
|
||||
source = icalendar_timeseries_server/
|
||||
|
||||
[report]
|
||||
show_missing = True
|
||||
include = icalendar_timeseries_server/*
|
||||
omit = */test/*.py
|
||||
40
setup.py
Executable file
40
setup.py
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
from icalendar_timeseries_server import __version__
|
||||
|
||||
setup(
|
||||
name='icalendar_timeseries_server',
|
||||
version=__version__,
|
||||
author='s3lph',
|
||||
author_email='',
|
||||
description='',
|
||||
license='MIT',
|
||||
keywords='ical,icalendar,timeseries,prometheus,grafana',
|
||||
url='https://gitlab.com/s3lph/icalendar-timeseries-server',
|
||||
packages=find_packages(exclude=['*.test']),
|
||||
long_description='',
|
||||
python_requires='>=3.6',
|
||||
install_requires=[
|
||||
'bottle',
|
||||
'python-dateutil',
|
||||
'icalendar',
|
||||
'isodate',
|
||||
'jinja2',
|
||||
'pytz'
|
||||
],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'icalendar-timeseries-server = icalendar_timeseries_server:main'
|
||||
]
|
||||
},
|
||||
classifiers=[
|
||||
'Development Status :: 3 - Alpha',
|
||||
'Environment :: Web Environment',
|
||||
'Framework :: Bottle',
|
||||
'Intended Audience :: System Administrators',
|
||||
'License :: OSI Approved :: MIT License',
|
||||
'Topic :: System :: Monitoring'
|
||||
],
|
||||
)
|
||||
Loading…
Reference in a new issue