Compare commits
14 commits
Author | SHA1 | Date | |
---|---|---|---|
5e52dffc27 | |||
5f8653797f | |||
fd85f5b1c2 | |||
![]() |
c94003cba7 | ||
![]() |
69315c80e8 | ||
![]() |
879bd5b1b7 | ||
![]() |
675e391ea5 | ||
![]() |
dc3f7101f0 | ||
![]() |
396e0e5c5f | ||
![]() |
90b80ae529 | ||
![]() |
28c2b1d425 | ||
![]() |
f99496c497 | ||
![]() |
fae85de8ca | ||
![]() |
395393345e |
17 changed files with 323 additions and 311 deletions
38
.forgejo/workflows/package.yml
Normal file
38
.forgejo/workflows/package.yml
Normal file
|
@ -0,0 +1,38 @@
|
|||
---
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
jobs:
|
||||
|
||||
build_wheel:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: Build Python wheel
|
||||
run: |
|
||||
apt update; apt install -y python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
python3 setup.py egg_info bdist_wheel
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-wheel-package-upload@v3
|
||||
with:
|
||||
username: ${{ secrets.API_USERNAME }}
|
||||
password: ${{ secrets.API_PASSWORD }}
|
||||
|
||||
build_debian:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-python-debian-package@v5
|
||||
with:
|
||||
python_module: icalendar_timeseries_server
|
||||
package_name: icalendar-timeseries-server
|
||||
package_root: package/debian/icalendar-timeseries-server
|
||||
package_output_path: package/debian
|
||||
- uses: https://git.kabelsalat.ch/s3lph/forgejo-action-debian-package-upload@v2
|
||||
with:
|
||||
username: ${{ secrets.API_USERNAME }}
|
||||
password: ${{ secrets.API_PASSWORD }}
|
||||
deb: "package/debian/*.deb"
|
27
.forgejo/workflows/test.yml
Normal file
27
.forgejo/workflows/test.yml
Normal file
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
|
||||
on: push
|
||||
|
||||
jobs:
|
||||
|
||||
test:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: test
|
||||
run: |
|
||||
apt update; apt install --yes python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server
|
||||
python3 -m coverage combine
|
||||
python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
codestyle:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: https://code.forgejo.org/actions/checkout@v4
|
||||
- name: codestyle
|
||||
run: |
|
||||
apt update; apt install --yes python3-pip
|
||||
pip3 install --break-system-packages -e .[test]
|
||||
pycodestyle icalendar_timeseries_server
|
108
.gitlab-ci.yml
108
.gitlab-ci.yml
|
@ -1,108 +0,0 @@
|
|||
---
|
||||
image: s3lph/icalendar-timeseries-server-ci:20190820-01
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- release
|
||||
|
||||
|
||||
|
||||
before_script:
|
||||
- export ITS_VERSION=$(python -c 'import icalendar_timeseries_server; print(icalendar_timeseries_server.__version__)')
|
||||
|
||||
|
||||
|
||||
test:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- sudo -u its python3 -m coverage run --rcfile=setup.cfg -m unittest discover icalendar_timeseries_server
|
||||
- sudo -u its python3 -m coverage combine
|
||||
- sudo -u its python3 -m coverage report --rcfile=setup.cfg
|
||||
|
||||
codestyle:
|
||||
stage: test
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- sudo -u its pycodestyle icalendar_timeseries_server
|
||||
|
||||
|
||||
|
||||
build_wheel:
|
||||
stage: build
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- python3 setup.py egg_info bdist_wheel
|
||||
- cd dist
|
||||
- sha256sum *.whl > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "dist/*.whl"
|
||||
- dist/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
build_debian:
|
||||
stage: build
|
||||
script:
|
||||
# The Python package name provided by the python3-magic Debian package is "python-magic" rather than "file-magic".
|
||||
- sed -re 's/file-magic/python-magic/' -i setup.py
|
||||
- echo -n > package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
- |
|
||||
cat > package/debian/icalendar-timeseries-server/DEBIAN/control <<EOF
|
||||
Package: icalendar-timeseries-server
|
||||
Version: ${ITS_VERSION}
|
||||
Maintainer: ${PACKAGE_AUTHOR}
|
||||
Section: web
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate
|
||||
Description: Scrape iCalendar endpoints and present their data in a
|
||||
timeseries format. A small service that scrapes iCalendar files
|
||||
served over HTTP, parses their contents and returns a timeseries
|
||||
format compatible to the /api/v1/query API endpoint of a Prometheus
|
||||
server. This allows e.g. a Grafana administrator to add a Prometheus
|
||||
data source pointing at this server, returning the events in the
|
||||
calendars in the event metric.
|
||||
EOF
|
||||
- |
|
||||
for version in $(cat CHANGELOG.md | grep '<!-- BEGIN CHANGES' | cut -d ' ' -f 4); do
|
||||
echo "icalendar-timeseries-server (${version}-1); urgency=medium\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
cat CHANGELOG.md | grep -A 1000 "<"'!'"-- BEGIN CHANGES ${version} -->" | grep -B 1000 "<"'!'"-- END CHANGES ${version} -->" | tail -n +2 | head -n -1 | sed -re 's/^-/ */g' >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
echo "\n -- ${PACKAGE_AUTHOR} $(date -R)\n" >> package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
done
|
||||
- gzip -9n package/debian/icalendar-timeseries-server/usr/share/doc/icalendar-timeseries-server/changelog
|
||||
- python3.7 setup.py egg_info install --root=package/debian/icalendar-timeseries-server/ --prefix=/usr --optimize=1
|
||||
- cd package/debian
|
||||
- mkdir -p icalendar-timeseries-server/usr/lib/python3/dist-packages/
|
||||
- rsync -a icalendar-timeseries-server/usr/lib/python3.7/site-packages/ icalendar-timeseries-server/usr/lib/python3/dist-packages/
|
||||
- rm -rf icalendar-timeseries-server/usr/lib/python3.7/
|
||||
- find icalendar-timeseries-server/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
|
||||
- find icalendar-timeseries-server/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
|
||||
- mv icalendar-timeseries-server/usr/bin/icalendar-timeseries-server icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server
|
||||
- rm -rf icalendar-timeseries-server/usr/bin
|
||||
- sed -re 's$#!/usr/local/bin/python3.7$#!/usr/bin/python3$' -i icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server
|
||||
- find icalendar-timeseries-server -type f -exec chmod 0644 {} \;
|
||||
- find icalendar-timeseries-server -type d -exec chmod 755 {} \;
|
||||
- find icalendar-timeseries-server -type f -name .gitkeep -delete
|
||||
- chmod +x icalendar-timeseries-server/usr/lib/icalendar-timeseries-server/icalendar-timeseries-server icalendar-timeseries-server/DEBIAN/postinst icalendar-timeseries-server/DEBIAN/prerm icalendar-timeseries-server/DEBIAN/postrm
|
||||
- dpkg-deb --build icalendar-timeseries-server
|
||||
- mv icalendar-timeseries-server.deb "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb"
|
||||
- sudo -u nobody lintian "icalendar-timeseries-server_${ITS_VERSION}-1_all.deb"
|
||||
- sha256sum *.deb > SHA256SUMS
|
||||
artifacts:
|
||||
paths:
|
||||
- "package/debian/*.deb"
|
||||
- package/debian/SHA256SUMS
|
||||
only:
|
||||
- tags
|
||||
|
||||
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- python package/release.py
|
||||
only:
|
||||
- tags
|
0
.woodpecker.yml
Normal file
0
.woodpecker.yml
Normal file
86
CHANGELOG.md
86
CHANGELOG.md
|
@ -1,6 +1,92 @@
|
|||
# iCalendar Timeseries Server Changelog
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.6.3 -->
|
||||
## Version 0.6.3
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.6.3 -->
|
||||
- Migration from Woodpecker to Forgejo Actions.
|
||||
<!-- END CHANGES 0.6.3 -->
|
||||
|
||||
<!-- END RELEASE v0.6.3 -->
|
||||
|
||||
<!-- BEGIN RELEASE v0.6.2 -->
|
||||
## Version 0.6.2
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.6.2 -->
|
||||
- Migration from Gitlab-CI to Woodpecker
|
||||
<!-- END CHANGES 0.6.2 -->
|
||||
|
||||
<!-- END RELEASE v0.6.2 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.6.1 -->
|
||||
## Version 0.6.1
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.6.1 -->
|
||||
- Same fix, but for todo as well as events.
|
||||
|
||||
<!-- END CHANGES 0.6.1 -->
|
||||
|
||||
<!-- END RELEASE v0.6.1 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.6 -->
|
||||
## Version 0.6
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.6 -->
|
||||
- Fix: A specific API field has to be a string rather than float, and recent Grafana versions validate this.
|
||||
|
||||
<!-- END CHANGES 0.6 -->
|
||||
|
||||
<!-- END RELEASE v0.6 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.5 -->
|
||||
## Version 0.5
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.5 -->
|
||||
- Retry calendar scraping with exponential backoff.
|
||||
<!-- END CHANGES 0.5 -->
|
||||
|
||||
<!-- END RELEASE v0.5 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.4.1 -->
|
||||
## Version 0.4.1
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.4.1 -->
|
||||
- Fix todo sorting by due date.
|
||||
- Update README regarding `todo` time series.
|
||||
<!-- END CHANGES 0.4.1 -->
|
||||
|
||||
<!-- END RELEASE v0.4.1 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.4.0 -->
|
||||
## Version 0.4.0
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- BEGIN CHANGES 0.4.0 -->
|
||||
- VTODO components are exported in a second time series, `todo` . Todo recurrence is not supported yet though.
|
||||
<!-- END CHANGES 0.4.0 -->
|
||||
|
||||
<!-- END RELEASE v0.4.0 -->
|
||||
|
||||
|
||||
<!-- BEGIN RELEASE v0.3.3 -->
|
||||
## Version 0.3.3
|
||||
|
||||
|
|
12
README.md
12
README.md
|
@ -4,8 +4,8 @@ This project is a small service that scrapes iCalendar files served
|
|||
over HTTP, parses their contents and returns the data in a timeseries
|
||||
format compatible to the `/api/v1/query` API endpoint of a Prometheus
|
||||
server. This allows e.g. a Grafana administrator to add a Prometheus
|
||||
data source pointing at this server, returning the events in the
|
||||
calendars in the `event` metric.
|
||||
data source pointing at this server, returning calendar events in the
|
||||
`event` metric and todos in the `todo` metric.
|
||||
|
||||
## Example
|
||||
|
||||
|
@ -147,7 +147,7 @@ Configuration is done through a JSON config file:
|
|||
| `calendars.*.auth[?type=='basic'].password` | string | The Basic Auth password to authenticate with. |
|
||||
| `calendars.*.auth[?type=='tls'].keyfile` | string | Path to the key file containing the TLS private key, client certificate and certificate chain, in PEM format. |
|
||||
| `calendars.*.auth[?type=='tls'].passphrase` | string | Passphrase for the private key (optional). |
|
||||
| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana. |
|
||||
| `key_replace` | dict | Labels to rename, might be necessary e.g. for column ordering in Grafana 6 and earlier. |
|
||||
| `keys(key_replace)` | string | The labels to rename. |
|
||||
| `key_replace.*` | string | The names to rename the labels to. |
|
||||
| `value_replace` | dict | Label values to postprocess. |
|
||||
|
@ -168,6 +168,12 @@ In addition, PromQL label filters can be used.
|
|||
event{calendar="public",foo=~".*"}
|
||||
```
|
||||
|
||||
Alongside with events, todos are exported in a second time series:
|
||||
|
||||
```
|
||||
todo{status!="COMPLETED"}
|
||||
```
|
||||
|
||||
## Why Prometheus API
|
||||
|
||||
- It's JSON. A JSON generator is builtin in Python, so no further dependency.
|
||||
|
|
|
@ -1,2 +1,2 @@
|
|||
|
||||
__version__ = '0.3.3'
|
||||
__version__ = '0.6.3'
|
||||
|
|
|
@ -7,7 +7,7 @@ import bottle
|
|||
|
||||
from icalendar_timeseries_server.config import get_config
|
||||
from icalendar_timeseries_server.event import Metric
|
||||
from icalendar_timeseries_server.cal import get_calendar
|
||||
from icalendar_timeseries_server.cal import get_calendar_events, get_calendar_todos
|
||||
from icalendar_timeseries_server.query import MetricQuery
|
||||
|
||||
|
||||
|
@ -31,9 +31,15 @@ def prometheus_api():
|
|||
|
||||
try:
|
||||
for name in get_config().calendars.keys():
|
||||
events.extend(get_calendar(name))
|
||||
events = list(filter(q, events))
|
||||
events.sort(key=lambda e: e.start)
|
||||
if q.name == 'event':
|
||||
events.extend(get_calendar_events(name))
|
||||
events = list(filter(q, events))
|
||||
events.sort(key=lambda e: e.start)
|
||||
elif q.name == 'todo':
|
||||
events.extend(get_calendar_todos(name))
|
||||
events = list(filter(q, events))
|
||||
# Sort by due date and priority
|
||||
events.sort(key=lambda e: (e.due is None, e.due, e.priority))
|
||||
response = {
|
||||
'status': 'success',
|
||||
'data': {
|
||||
|
|
|
@ -13,9 +13,11 @@ from isodate import Duration
|
|||
from icalendar_timeseries_server import __version__
|
||||
from icalendar_timeseries_server.config import get_config, CalendarConfig
|
||||
from icalendar_timeseries_server.event import Event
|
||||
from icalendar_timeseries_server.todo import Todo
|
||||
|
||||
|
||||
_SCRAPE_CACHE: Dict[str, List[Event]] = dict()
|
||||
_EVENT_SCRAPE_CACHE: Dict[str, List[Event]] = dict()
|
||||
_TODO_SCRAPE_CACHE: Dict[str, List[Todo]] = dict()
|
||||
_SCRAPE_CACHE_LOCK: Lock = Lock()
|
||||
|
||||
__py_version: str = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
|
||||
|
@ -53,16 +55,13 @@ def _parse_recurring(event: cal.Event, start: datetime, end: datetime, duration:
|
|||
|
||||
|
||||
def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: datetime):
|
||||
global _SCRAPE_CACHE, _SCRAPE_CACHE_LOCK
|
||||
global _EVENT_SCRAPE_CACHE, _TODO_SCRAPE_CACHE, _SCRAPE_CACHE_LOCK
|
||||
events = []
|
||||
todos = []
|
||||
|
||||
opener: urllib.request.OpenerDirector = config.get_url_opener()
|
||||
try:
|
||||
with opener.open(config.url) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
except BaseException:
|
||||
logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ({config.url})')
|
||||
return
|
||||
with opener.open(config.url) as response:
|
||||
data = response.read().decode('utf-8')
|
||||
calendar = cal.Calendar.from_ical(data)
|
||||
|
||||
for element in calendar.walk():
|
||||
|
@ -88,34 +87,58 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da
|
|||
for occurence in occurences:
|
||||
if start <= occurence + duration and occurence < end:
|
||||
events.append(Event(name, element, occurence, occurence + duration))
|
||||
elif element.name == "VTODO":
|
||||
dtstart = element.get('dtstamp').dt
|
||||
duration = timedelta(0)
|
||||
if 'dtstart' in element:
|
||||
dtstart = element.get('dtstart').dt
|
||||
if 'duration' in element:
|
||||
duration = element.get('duration').dt
|
||||
todos.append(Todo(name, element, dtstart, dtstart + duration))
|
||||
|
||||
with _SCRAPE_CACHE_LOCK:
|
||||
_SCRAPE_CACHE[name] = events
|
||||
_EVENT_SCRAPE_CACHE[name] = events
|
||||
_TODO_SCRAPE_CACHE[name] = todos
|
||||
|
||||
|
||||
def scrape_calendar(name: str, config: CalendarConfig):
|
||||
def scrape_calendar(name: str, config: CalendarConfig, retry: int):
|
||||
# Get current time in configured timezone
|
||||
tz = get_config().tz
|
||||
now: datetime = datetime.now(tz)
|
||||
# Reschedule calendar scraping
|
||||
cron = Timer(config.interval.totimedelta(start=now).total_seconds(),
|
||||
lambda: scrape_calendar(name, config))
|
||||
cron.start()
|
||||
# Only scrape at most once a minute
|
||||
interval = max(int(config.interval.totimedelta(start=now).total_seconds()), 60)
|
||||
# Compute interval for which to return events
|
||||
start_delta: Duration = get_config().start_delta
|
||||
end_delta: Duration = get_config().end_delta
|
||||
start: datetime = now + start_delta
|
||||
end: datetime = now + end_delta
|
||||
# Scrape and parse the calendar
|
||||
_scrape_calendar(name, config, start, end)
|
||||
try:
|
||||
_scrape_calendar(name, config, start, end)
|
||||
# Reschedule calendar scraping
|
||||
cron = Timer(interval, lambda: scrape_calendar(name, config, 0))
|
||||
except BaseException:
|
||||
# reschedule with exponential backoff, but no more than the regular scrape interval
|
||||
backoff_seconds = min(60 * 2**retry, interval)
|
||||
logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" '
|
||||
f'({config.url}), retrying in {backoff_seconds}s.')
|
||||
cron = Timer(backoff_seconds, lambda: scrape_calendar(name, config, retry+1))
|
||||
cron.start()
|
||||
|
||||
|
||||
def start_scrape_calendar(name: str, config: CalendarConfig):
|
||||
# Schedule first calendar scraping
|
||||
cron = Timer(0, lambda: scrape_calendar(name, config))
|
||||
cron = Timer(0, lambda: scrape_calendar(name, config, retry=0))
|
||||
cron.start()
|
||||
|
||||
|
||||
def get_calendar(name: str):
|
||||
global _SCRAPE_CACHE
|
||||
def get_calendar_events(name: str):
|
||||
global _EVENT_SCRAPE_CACHE
|
||||
with _SCRAPE_CACHE_LOCK:
|
||||
return _SCRAPE_CACHE.get(name, [])
|
||||
return _EVENT_SCRAPE_CACHE.get(name, [])
|
||||
|
||||
|
||||
def get_calendar_todos(name: str):
|
||||
global _TODO_SCRAPE_CACHE
|
||||
with _SCRAPE_CACHE_LOCK:
|
||||
return _TODO_SCRAPE_CACHE.get(name, [])
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Any, Dict, List, Set
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import icalendar
|
||||
import jinja2
|
||||
|
@ -59,7 +59,7 @@ class Event(Metric):
|
|||
},
|
||||
'value': [
|
||||
self.start.timestamp(),
|
||||
1
|
||||
"1"
|
||||
]
|
||||
}
|
||||
event['metric'].update(self._labels)
|
||||
|
|
|
@ -172,7 +172,7 @@ class MetricQuery:
|
|||
elif filterstate != 0:
|
||||
raise ValueError('Unexpected EOF')
|
||||
|
||||
def __call__(self, metric: Metric):
|
||||
def __call__(self, metric: Metric) -> bool:
|
||||
"""
|
||||
Applies the filter deducted from the query string to the given metric.
|
||||
|
||||
|
@ -188,3 +188,7 @@ class MetricQuery:
|
|||
return False
|
||||
# Return True if all filters matched
|
||||
return True
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._metric_name
|
||||
|
|
77
icalendar_timeseries_server/todo.py
Normal file
77
icalendar_timeseries_server/todo.py
Normal file
|
@ -0,0 +1,77 @@
|
|||
from typing import Any, Dict, List
|
||||
|
||||
import icalendar
|
||||
import jinja2
|
||||
from datetime import datetime, date, timedelta
|
||||
|
||||
from icalendar_timeseries_server.config import get_config, get_jenv
|
||||
from icalendar_timeseries_server.query import Metric
|
||||
|
||||
_ATTRIBUTES: List[str] = [
|
||||
'class',
|
||||
'description',
|
||||
'geo',
|
||||
'location',
|
||||
'organizer',
|
||||
'percent-complete',
|
||||
'priority',
|
||||
'status',
|
||||
'summary',
|
||||
'url',
|
||||
'attach'
|
||||
]
|
||||
|
||||
|
||||
class Todo(Metric):
|
||||
|
||||
def __init__(self, cname: str, todo: icalendar.cal.Todo, start: datetime, end: datetime):
|
||||
self.calendar: str = cname
|
||||
self.start = start
|
||||
due = todo.get('due', None)
|
||||
if due:
|
||||
if isinstance(due.dt, datetime):
|
||||
self.due = due.dt
|
||||
elif isinstance(due.dt, date):
|
||||
self.due = datetime.combine(due.dt, datetime.min.time())
|
||||
self.due = self.due.replace(tzinfo=get_config().tz)
|
||||
else:
|
||||
self.due = None
|
||||
# self.attributes: Dict[str, str] = dict()
|
||||
attributes: Dict[str, str] = dict()
|
||||
tmp: Dict[str, Any] = {
|
||||
'calendar': cname,
|
||||
'start': start,
|
||||
'end': end
|
||||
}
|
||||
if self.due:
|
||||
tmp['due'] = str(self.due)
|
||||
for attr in _ATTRIBUTES:
|
||||
tmp[attr] = todo.get(attr, '')
|
||||
substitution_keys = set(_ATTRIBUTES)
|
||||
substitution_keys.update(tmp.keys())
|
||||
substitution_keys.update(get_config().key_replace.keys())
|
||||
substitution_keys.update(get_config().value_replace.keys())
|
||||
for attr in substitution_keys:
|
||||
newkey: str = get_config().key_replace.get(attr, attr)
|
||||
value: str = tmp.get(attr, '')
|
||||
newval_template: str = get_config().value_replace.get(attr, str(value))
|
||||
jtemplate: jinja2.Template = get_jenv().from_string(newval_template)
|
||||
newvalue: str = jtemplate.render(**tmp)
|
||||
attributes[newkey] = newvalue
|
||||
self.uid: str = f'{cname}-{start.strftime("%Y%m%dT%H%M%S%Z")}'
|
||||
self.priority = todo.get('priority', '0')
|
||||
super().__init__('todo', attributes)
|
||||
|
||||
def serialize(self) -> Dict[str, Any]:
|
||||
todo: Dict[str, Any] = {
|
||||
'metric': {
|
||||
'__name__': 'todo',
|
||||
'calendar': self.calendar
|
||||
},
|
||||
'value': [
|
||||
self.start.timestamp(),
|
||||
"1"
|
||||
]
|
||||
}
|
||||
todo['metric'].update(self._labels)
|
||||
return todo
|
14
package/debian/icalendar-timeseries-server/DEBIAN/control
Normal file
14
package/debian/icalendar-timeseries-server/DEBIAN/control
Normal file
|
@ -0,0 +1,14 @@
|
|||
Package: icalendar-timeseries-server
|
||||
Version: __VERSION__
|
||||
Maintainer: s3lph <s3lph@kabelsalat.ch>
|
||||
Section: web
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
Depends: python3 (>= 3.7), python3-jinja2, python3-bottle, python3-dateutil, python3-icalendar, python3-isodate
|
||||
Description: Scrape iCalendar endpoints and present their data in a timeseries format.
|
||||
A small service that scrapes iCalendar files served over HTTP, parses
|
||||
their contents and returns a timeseries format compatible to the
|
||||
/api/v1/query API endpoint of a Prometheus server. This allows e.g. a
|
||||
Grafana administrator to add a Prometheus data source pointing at
|
||||
this server, returning the events in the calendars in the event
|
||||
metric.
|
|
@ -15,11 +15,7 @@ if [[ "$1" == "configure" ]]; then
|
|||
chown its:its /var/lib/its
|
||||
chmod 0750 /var/lib/its
|
||||
|
||||
systemctl daemon-reload || true
|
||||
|
||||
if [[ "$2" != "" ]]; then
|
||||
# Restart after upgrading the package
|
||||
systemctl restart icalendar-timeseries-server.service
|
||||
fi
|
||||
deb-systemd-helper enable icalendar-timeseries-server.service
|
||||
deb-systemd-invoke restart icalendar-timeseries-server.service
|
||||
|
||||
fi
|
||||
|
|
|
@ -4,6 +4,6 @@ set -e
|
|||
|
||||
if [[ "$1" == "remove" ]]; then
|
||||
|
||||
userdel its
|
||||
deb-systemd-invoke stop icalendar-timeseries-server.service
|
||||
|
||||
fi
|
||||
|
|
|
@ -1,165 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib.request
|
||||
import http.client
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
USER_AGENT = 'curl/7.70.0'
|
||||
|
||||
|
||||
def parse_changelog(tag: str) -> Optional[str]:
|
||||
release_changelog: str = ''
|
||||
with open('CHANGELOG.md', 'r') as f:
|
||||
in_target: bool = False
|
||||
done: bool = False
|
||||
for line in f.readlines():
|
||||
if in_target:
|
||||
if f'<!-- END RELEASE {tag} -->' in line:
|
||||
done = True
|
||||
break
|
||||
release_changelog += line
|
||||
elif f'<!-- BEGIN RELEASE {tag} -->' in line:
|
||||
in_target = True
|
||||
continue
|
||||
if not done:
|
||||
return None
|
||||
return release_changelog
|
||||
|
||||
|
||||
def fetch_job_ids(project_id: int, pipeline_id: int, api_token: str) -> Dict[str, str]:
|
||||
url: str = f'https://gitlab.com/api/v4/projects/{project_id}/pipelines/{pipeline_id}/jobs'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'User-Agent': USER_AGENT
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.read()
|
||||
joblist: List[Dict[str, Any]] = json.loads(resp_data.decode())
|
||||
|
||||
jobidmap: Dict[str, str] = {}
|
||||
for job in joblist:
|
||||
name: str = job['name']
|
||||
job_id: str = job['id']
|
||||
jobidmap[name] = job_id
|
||||
return jobidmap
|
||||
|
||||
|
||||
def fetch_single_shafile(url: str) -> str:
|
||||
headers: Dict[str, str] = {
|
||||
'User-Agent': USER_AGENT
|
||||
}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
resp: http.client.HTTPResponse = urllib.request.urlopen(req)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
resp_data: bytes = resp.readline()
|
||||
shafile: str = resp_data.decode()
|
||||
filename: str = shafile.strip().split(' ')[-1].strip()
|
||||
return filename
|
||||
|
||||
|
||||
def fetch_wheel_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_wheel"]}/artifacts/raw'
|
||||
wheel_sha_url: str = f'{mybase}/dist/SHA256SUMS'
|
||||
wheel_filename: str = fetch_single_shafile(wheel_sha_url)
|
||||
wheel_url: str = f'{mybase}/dist/{wheel_filename}'
|
||||
return wheel_url, wheel_sha_url
|
||||
|
||||
|
||||
def fetch_debian_url(base_url: str, job_ids: Dict[str, str]) -> Optional[Tuple[str, str]]:
|
||||
mybase: str = f'{base_url}/jobs/{job_ids["build_debian"]}/artifacts/raw'
|
||||
debian_sha_url: str = f'{mybase}/package/debian/SHA256SUMS'
|
||||
debian_filename: str = fetch_single_shafile(debian_sha_url)
|
||||
debian_url: str = f'{mybase}/package/debian/{debian_filename}'
|
||||
return debian_url, debian_sha_url
|
||||
|
||||
|
||||
def main():
|
||||
api_token: Optional[str] = os.getenv('GITLAB_API_TOKEN')
|
||||
release_tag: Optional[str] = os.getenv('CI_COMMIT_TAG')
|
||||
project_name: Optional[str] = os.getenv('CI_PROJECT_PATH')
|
||||
project_id: Optional[str] = os.getenv('CI_PROJECT_ID')
|
||||
pipeline_id: Optional[str] = os.getenv('CI_PIPELINE_ID')
|
||||
if api_token is None:
|
||||
print('GITLAB_API_TOKEN is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if release_tag is None:
|
||||
print('CI_COMMIT_TAG is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_name is None:
|
||||
print('CI_PROJECT_PATH is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if project_id is None:
|
||||
print('CI_PROJECT_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if pipeline_id is None:
|
||||
print('CI_PIPELINE_ID is not set.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
changelog: Optional[str] = parse_changelog(release_tag)
|
||||
if changelog is None:
|
||||
print('Changelog could not be parsed.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
job_ids: Dict[str, str] = fetch_job_ids(project_id, pipeline_id, api_token)
|
||||
|
||||
base_url: str = f'https://gitlab.com/{project_name}/-'
|
||||
|
||||
wheel_url, wheel_sha_url = fetch_wheel_url(base_url, job_ids)
|
||||
debian_url, debian_sha_url = fetch_debian_url(base_url, job_ids)
|
||||
|
||||
augmented_changelog = f'''{changelog.strip()}
|
||||
|
||||
### Download
|
||||
|
||||
- [Python Wheel]({wheel_url}) ([sha256]({wheel_sha_url}))
|
||||
- [Debian Package]({debian_url}) ([sha256]({debian_sha_url}))'''
|
||||
|
||||
post_body: str = json.dumps({'description': augmented_changelog})
|
||||
|
||||
gitlab_release_api_url: str = \
|
||||
f'https://gitlab.com/api/v4/projects/{project_id}/repository/tags/{release_tag}/release'
|
||||
headers: Dict[str, str] = {
|
||||
'Private-Token': api_token,
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'User-Agent': USER_AGENT
|
||||
}
|
||||
|
||||
request = urllib.request.Request(
|
||||
gitlab_release_api_url,
|
||||
post_body.encode('utf-8'),
|
||||
headers=headers,
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
response: http.client.HTTPResponse = urllib.request.urlopen(request)
|
||||
except HTTPError as e:
|
||||
print(e.read().decode())
|
||||
sys.exit(1)
|
||||
response_bytes: bytes = response.read()
|
||||
response_str: str = response_bytes.decode()
|
||||
response_data: Dict[str, Any] = json.loads(response_str)
|
||||
|
||||
if response_data['tag_name'] != release_tag:
|
||||
print('Something went wrong...', file=sys.stderr)
|
||||
print(response_str, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(response_data['description'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
10
setup.py
10
setup.py
|
@ -12,7 +12,7 @@ setup(
|
|||
description='',
|
||||
license='MIT',
|
||||
keywords='ical,icalendar,timeseries,prometheus,grafana',
|
||||
url='https://gitlab.com/s3lph/icalendar-timeseries-server',
|
||||
url='https://git.kabelsalat.ch/s3lph/icalendar-timeseries-server',
|
||||
packages=find_packages(exclude=['*.test']),
|
||||
long_description='',
|
||||
python_requires='>=3.6',
|
||||
|
@ -23,6 +23,14 @@ setup(
|
|||
'isodate',
|
||||
'jinja2'
|
||||
],
|
||||
extras_require={
|
||||
'test': [
|
||||
'coverage',
|
||||
'pycodestyle',
|
||||
'mypy',
|
||||
'twine'
|
||||
]
|
||||
},
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'icalendar-timeseries-server = icalendar_timeseries_server:main'
|
||||
|
|
Loading…
Reference in a new issue