commit 264e9a5dcf5400d1936740c9c23dccf4e5f92cd4 Author: s3lph Date: Sun Jul 23 20:19:37 2023 +0200 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..a50a7ce --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,51 @@ +--- + +pipeline: + + test: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - python3 -m unittest coverage run --rcfile=setup.cfg -m unittest discover pkgtls_exporter + - python3 -m unittest coverage combine + - python3 -m unittest coverage report --rcfile=setup.cfg + + codestyle: + image: python:3.11-bookworm + group: test + commands: + - pip3 install -e .[test] + - pycodestyle pkgtls_exporter + + build_wheel: + image: python:3.11-bookworm + group: package + commands: + - python3 setup.py egg_info bdist_wheel + - cd dist + - sha256sum *.whl > SHA256SUMS + + build_debian: + image: python:3.11-bookworm + group: package + commands: + - apt update; apt install lintian + - export EXPORTER_VERSION=$(python3 -c 'import tlsrpt_exporter; print(tlsrpt_exporter.__version__)') + - mkdir -p package/debian/prometheus-tlsrpt-exporter/usr/share/prometheus-tlsrpt-exporter + - cp -r templates/ package/debian/prometheus-tlsrpt-exporter/usr/share/prometheus-tlsrpt-exporter/templates/ + - python3 setup.py egg_info install --root=package/debian/prometheus-tlsrpt-exporter/ --prefix=/usr --optimize=1 + - cd package/debian + - mkdir -p prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages/ + - rsync -a prometheus-tlsrpt-exporter/usr/lib/python3.11/site-packages/ matemat/usr/lib/python3/dist-packages/ + - rm -rf prometheus-tlsrpt-exporter/usr/lib/python3.11/ + - find prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true + - find prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \; + - sed -re 's$#!/usr/local/bin/python3.11$#!/usr/bin/python3$' -i prometheus-tlsrpt-exporter/usr/bin/prometheus-tlsrpt-exporter + - find prometheus-tlsrpt-exporter -type f -exec chmod 0644 {} \; + - find prometheus-tlsrpt-exporter -type d -exec chmod 755 {} \; + - chmod +x prometheus-tlsrpt-exporter/usr/bin/prometheus-tlsrpt-exporter prometheus-tlsrpt-exporter/DEBIAN/postinst prometheus-tlsrpt-exporter/DEBIAN/prerm + - dpkg-deb --build prometheus-tlsrpt-exporter + - mv prometheus-tlsrpt-exporter.deb "prometheus-tlsrpt-exporter_${EXPORTER_VERSION}-1_all.deb" + - sudo -u nobody lintian "prometheus-tlsrpt-exporter_${EXPORTER_VERSION}-1_all.deb" + - sha256sum *.deb > SHA256SUMS diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..01049fe --- /dev/null +++ b/LICENSE @@ -0,0 +1,16 @@ +Copyright 2023 s3lph + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/package/debian/prometheus-tlsrpt-exporter/DEBIAN/conffiles b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/conffiles new file mode 100644 index 0000000..3068a58 --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/conffiles @@ -0,0 +1 @@ +/etc/default/prometheus-tlsrpt-exporter diff --git a/package/debian/prometheus-tlsrpt-exporter/DEBIAN/control b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/control new file mode 100644 index 0000000..db78b2a --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/control @@ -0,0 +1,10 @@ +Package: prometheus-tlsrpt-exporter +Version: __VERSION__ +Maintainer: s3lph +Section: web +Priority: optional +Architecture: all +Depends: python3 (>= 3.11), python3-bottle, python3-jinja2 +Description: Prometheus exporter for TLSRPT reports + Collects TLSRPT reports and exposes their statistics per domain as + Prometheus metrics as well as on a web interface. diff --git a/package/debian/prometheus-tlsrpt-exporter/DEBIAN/postinst b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/postinst new file mode 100755 index 0000000..28195f0 --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/postinst @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "configure" ]]; then + + if ! getent group prometheus-tlsrpt-exporter >/dev/null; then + groupadd --system prometheus-tlsrpt-exporter + fi + + if ! getent passwd prometheus-tlsrpt-exporter >/dev/null; then + useradd --system --create-home --gid prometheus-tlsrpt-exporter --home-dir /var/lib/prometheus-tlsrpt-exporter --shell /usr/sbin/nologin prometheus-tlsrpt-exporter + fi + + chown prometheus-tlsrpt-exporter:prometheus-tlsrpt-exporter /var/lib/prometheus-tlsrpt-exporter + chmod 0750 /var/lib/prometheus-tlsrpt-exporter + + deb-systemd-helper enable prometheus-tlsrpt-exporter.service + deb-systemd-invoke restart prometheus-tlsrpt-exporter.service + +fi diff --git a/package/debian/prometheus-tlsrpt-exporter/DEBIAN/prerm b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/prerm new file mode 100755 index 0000000..ed58a85 --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/DEBIAN/prerm @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [[ "$1" == "remove" ]]; then + + deb-systemd-invoke stop prometheus-tlsrpt-exporter + +fi diff --git a/package/debian/prometheus-tlsrpt-exporter/etc/default/prometheus-tlsrpt-exporter b/package/debian/prometheus-tlsrpt-exporter/etc/default/prometheus-tlsrpt-exporter new file mode 100644 index 0000000..a2f65ec --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/etc/default/prometheus-tlsrpt-exporter @@ -0,0 +1,2 @@ + +ARGS="--base-url http://localhost:9123 --data-dir /var/lib/prometheus-tlsrpt-exporter --template-dir /usr/shre/prometheus-tlsrpt-exporter/templates" diff --git a/package/debian/prometheus-tlsrpt-exporter/usr/share/doc/prometheus-tlsrpt-exporter/copyright b/package/debian/prometheus-tlsrpt-exporter/usr/share/doc/prometheus-tlsrpt-exporter/copyright new file mode 100644 index 0000000..01049fe --- /dev/null +++ b/package/debian/prometheus-tlsrpt-exporter/usr/share/doc/prometheus-tlsrpt-exporter/copyright @@ -0,0 +1,16 @@ +Copyright 2023 s3lph + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7c8abda --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ + +# +# PyCodestyle +# + +[pycodestyle] +max-line-length = 120 +statistics = True + +# +# Coverage +# + +[run] +branch = True +parallel = True +source = tlsrpt_exporter/ + +[report] +show_missing = True +include = tlsrpt_exporter/* +omit = */test/*.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..eea3355 --- /dev/null +++ b/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup, find_packages + +from tlsrpt_exporter import __version__ + +setup( + name='tlsrpt_exporter', + version=__version__, + url='https://git.kabelsalat.ch/s3lph/prometheus-tlsrpt-exporter', + license='MIT', + author='s3lph', + author_email='', + description='Prometheus exporter for TLSRPT reports', + long_description=''' +Collects TLSRPT reports and exposes their statistics per domain as +Prometheus metrics as well as on a web interface. +''', + packages=find_packages(exclude=['*.test']), + python_requires='>=3.11', + install_requires=[ + 'bottle', + 'jinja2', + ], + extras_require={ + 'test': [ + 'coverage', + 'pycodestyle' + ] + }, + test_loader='unittest:TestLoader', + entry_points={ + 'console_scripts': [ + 'prometheus-tlsrpt-exporter = tlsrpt_exporter.__main__:main' + ] + } +) diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..04b13dc --- /dev/null +++ b/templates/base.html @@ -0,0 +1,23 @@ + + + + TLSRPT Exporter + + + +
+

TLSRPT Exporter

+
+ +
+ {% block main %} + {% endblock %} +
+ + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..7d6bc13 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,43 @@ +{% extends 'base.html' %} + +{% block main %} +

TLS Reports

+{% for domain, rpts in reports.items() %} +
+ {%- set count_s = ((rpts | map(attribute='stats') | unzip)[0] | unzip)[0] | sum %} + {%- set count_f = ((rpts | map(attribute='stats') | unzip)[0] | unzip)[1] | sum %} +

{{ domain }}{% if count_s + count_f > 0%}: {{ (count_s / (count_s + count_f)) | int * 100 }}%{% endif %}

+ + + + + + + + + + + {%- for report in rpts %} + {%- set cs = report.stats | sum(attribute=0) %} + {%- set cf = report.stats | sum(attribute=1) %} + + + + + + + {% endfor %} + +
DateReport IDSuccess RateActions
{{ report.report_id }} + {% if cs + cf == 0 %} + 0 + {% else %} + {{ (cs / (cs + cf)) | int * 100 }}% ({{ cs + cf }}) + {% endif %} + + View + Download +
+
+{% endfor %} +{% endblock %} diff --git a/templates/report.html b/templates/report.html new file mode 100644 index 0000000..108abec --- /dev/null +++ b/templates/report.html @@ -0,0 +1,7 @@ +{% extends 'base.html' %} + +{% block main %} +

TLS Report: {{ report['report-id']}}

+
{{ json.dumps(report, indent=4) }}
+Download JSON +{% endblock %} diff --git a/templates/style.css b/templates/style.css new file mode 100644 index 0000000..eaf5fdd --- /dev/null +++ b/templates/style.css @@ -0,0 +1,36 @@ +html, body { + font-family: sans-serif; +} + +span.success { + color: green; +} +span.success::pre { + content: "✓"; +} + +span.failure { + color: red; +} +span.failure::pre { + content: "✗"; +} + +table { + border-collapse: separate; + border-spacing: 0 5px; +} + +tbody > tr { + background: #efefef; + border: 1px solid #dddddd; +} + +tbody > tr > td { + margin: 0; + padding: 10px; +} + +td.report-id { + font-family: monospace; +} diff --git a/tlsrpt_exporter/__init__.py b/tlsrpt_exporter/__init__.py new file mode 100644 index 0000000..373d726 --- /dev/null +++ b/tlsrpt_exporter/__init__.py @@ -0,0 +1,2 @@ + +__version__ = '0.1' diff --git a/tlsrpt_exporter/__main__.py b/tlsrpt_exporter/__main__.py new file mode 100644 index 0000000..5bf7ea9 --- /dev/null +++ b/tlsrpt_exporter/__main__.py @@ -0,0 +1,179 @@ + +import argparse +import os +import dataclasses +import json +from datetime import datetime + +import bottle +import jinja2 + + +def unzip(xs): + return list(zip(*xs)) + + +class Reporter(bottle.Bottle): + + def __init__(self, metrics, ns): + super().__init__() + self.datadir = ns.data_dir + self.metrics = metrics + self.route('/', 'POST', self.report) + + def handle(self, data): + print(data) + if 'report-id' not in data: + return + prefix = f'{int(datetime.utcnow().timestamp())}+' + if len(data.get('policies', [])) == 0: + return + domain = None + for p in data.get('policies', []): + policy_domain = p.get('policy', {}).get('policy-domain') + if domain is None: + domain = policy_domain + count_s = p.get('summary', {}).get('total-successful-session-count', 0) + count_f = p.get('summary', {}).get('total-failure-session-count', 0) + self.metrics.add_report(policy_domain, count_s, count_f) + prefix += f'{count_s},{count_f};' + if domain is None: + return + with open(os.path.join(self.datadir, 'reports', prefix[:-1] + '+' + domain + '+' + data['report-id']), 'w') as f: + json.dump(data, f) + + def report(self): + bottle.response.add_header('Accept', 'application/tlsrpt+gzip') + bottle.response.add_header('Accept', 'application/tlsrpt+json') + if 'application/tlsrpt+gzip' in bottle.request.content_type: + self.handle(self.decompress(bottle.request.body)) + elif 'application/tlsrpt+json' in bottle.request.content_type: + self.handle(json.load(bottle.request.body)) + else: + bottle.response.status = 406 + + +class Metrics(bottle.Bottle): + + def __init__(self, ns): + super().__init__() + self.datadir = ns.data_dir + self.count = {} + self.count_s = {} + self.count_f = {} + self.load_reports() + self.route('/', 'GET', self.metrics) + + def add_report(self, domain, count_s, count_f): + self.count_s[domain] = self.count_s.setdefault(domain, 0) + count_s + self.count_f[domain] = self.count_f.setdefault(domain, 0) + count_f + self.count[domain] = self.count.setdefault(domain, 0) + 1 + + def load_reports(self): + for report in os.listdir(os.path.join(self.datadir, 'reports')): + if report.startswith('.'): + continue + _, stats, domain, _ = report.split('+', 3) + for p in stats.split(';'): + s, f = p.split(',') + self.add_report(domain, int(s), int(f)) + + def metrics(self): + bottle.response.headers['Content-Type'] = 'text/plain; version=0.0.4' + response = '# TYPE tlsrpt_successful counter\n' + response += '# HELP tlsrpt_successful Number of successful sessions\n' + for domain, count in self.count_s.items(): + response += f'tlsrpt_successful{{domain="{domain}"}} {count}\n' + response += '# TYPE tlsrpt_failed counter\n' + response += '# HELP tlsrpt_failed Number of failed sessions\n' + for domain, count in self.count_f.items(): + response += f'tlsrpt_failed{{domain="{domain}"}} {count}\n' + response += '# TYPE tlsrpt_count counter\n' + response += '# HELP tlsrpt_count Number of reports\n' + for domain, count in self.count.items(): + response += f'tlsrpt_count{{domain="{domain}"}} {count}\n' + return response + + +@dataclasses.dataclass +class ReportInfo: + ts: datetime + stats: list[tuple[int, int]] + report_id: str + filename: str + + +class UI(bottle.Bottle): + + def __init__(self, ns): + super().__init__() + self.datadir = ns.data_dir + self.baseurl = ns.base_url or f'//{ns.host}:{ns.port}' + self.route('/', 'GET', self.list_reports) + self.route('/report/', 'GET', self.report_details) + self.route('/report//json', 'GET', self.report_download) + self.route('/style.css', 'GET', self.style) + self.env = jinja2.Environment( + loader=jinja2.FileSystemLoader([ns.template_dir]) + ) + self.env.filters['unzip'] = unzip + + def style(self): + bottle.response.headers['Content-Type'] = 'text/css' + return self.env.get_template('style.css').render() + + def list_reports(self): + reports = {} + for report in os.listdir(os.path.join(self.datadir, 'reports')): + if report.startswith('.'): + continue + dt, stats, domain, report_id = report.split('+', 3) + dt = datetime.fromtimestamp(int(float(dt))) + stats = [(int(p.split(',', 1)[0]), int(p.split(',', 1)[1])) for p in stats.split(';')] + reports.setdefault(domain, []).append(ReportInfo(dt, stats, report_id, report)) + for v in reports.values(): + v.sort(key=lambda x: x.ts) + return self.env.get_template('index.html').render(baseurl=self.baseurl, reports=reports) + + def get_report(self, report_id: str): + report = None + for r in os.listdir(os.path.join(self.datadir, 'reports')): + tokens = r.split('+', 3) + if len(tokens) != 4: + continue + if tokens[3] != report_id: + continue + with open(os.path.join(self.datadir, 'reports', r), 'r') as f: + report = f.read() + return report + + def report_details(self, report_id: str): + report = self.get_report(report_id) + return self.env.get_template('report.html').render(baseurl=self.baseurl, report=json.loads(report), json=json) + + def report_download(self, report_id: str): + report = self.get_report(report_id) + bottle.response.headers['Content-Type'] = 'application/tlsrpt+json' + bottle.response.headers['Content-Disposition'] = f'attachment; filename={report_id}.json' + return report + + +def main(): + ap = argparse.ArgumentParser('prometheus-tlsrpt-exporter') + ap.add_argument('--host', '-H', type=str, default='::', help='Address to bind to') + ap.add_argument('--port', '-p', type=int, default=9123, help='Port to bind to') + ap.add_argument('--base-url', '-B', type=str, default=None, help='Base URL') + ap.add_argument('--data-dir', '-d', type=str, default='/tmp/prometheus-tlsrpt-exporter', help='Directory to save reports to') + ap.add_argument('--template-dir', '-t', type=str, default='templates', help='Directory to load templates from') + ns = ap.parse_args() + os.makedirs(os.path.join(ns.data_dir, 'reports'), exist_ok=True) + + metrics = Metrics(ns) + bottle.mount('/report', Reporter(metrics, ns)) + bottle.mount('/metrics', metrics) + bottle.mount('/ui', UI(ns)) + bottle.run(host=ns.host, port=ns.port) + + +if __name__ == '__main__': + main()