Initial commit

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.woodpecker.yml

@@ -0,0 +1,51 @@
+---
+
+pipeline:
+
+  test:
+    image: python:3.11-bookworm
+    group: test
+    commands:
+      - pip3 install -e .[test]
+      - python3 -m unittest coverage run --rcfile=setup.cfg -m unittest discover pkgtls_exporter
+      - python3 -m unittest coverage combine
+      - python3 -m unittest coverage report --rcfile=setup.cfg
+
+  codestyle:
+    image: python:3.11-bookworm
+    group: test
+    commands:
+      - pip3 install -e .[test]
+      - pycodestyle pkgtls_exporter
+
+  build_wheel:
+    image: python:3.11-bookworm
+    group: package
+    commands:
+      - python3 setup.py egg_info bdist_wheel
+      - cd dist
+      - sha256sum *.whl > SHA256SUMS
+
+  build_debian:
+    image: python:3.11-bookworm
+    group: package
+    commands:
+      - apt update; apt install lintian
+      - export EXPORTER_VERSION=$(python3 -c 'import tlsrpt_exporter; print(tlsrpt_exporter.__version__)')
+      - mkdir -p package/debian/prometheus-tlsrpt-exporter/usr/share/prometheus-tlsrpt-exporter
+      - cp -r templates/ package/debian/prometheus-tlsrpt-exporter/usr/share/prometheus-tlsrpt-exporter/templates/
+      - python3 setup.py egg_info install --root=package/debian/prometheus-tlsrpt-exporter/ --prefix=/usr --optimize=1
+      - cd package/debian
+      - mkdir -p prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages/
+      - rsync -a prometheus-tlsrpt-exporter/usr/lib/python3.11/site-packages/ matemat/usr/lib/python3/dist-packages/
+      - rm -rf prometheus-tlsrpt-exporter/usr/lib/python3.11/
+      - find prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages -name __pycache__ -exec rm -r {} \; 2>/dev/null || true
+      - find prometheus-tlsrpt-exporter/usr/lib/python3/dist-packages -name '*.pyc' -exec rm {} \;
+      - sed -re 's$#!/usr/local/bin/python3.11$#!/usr/bin/python3$' -i prometheus-tlsrpt-exporter/usr/bin/prometheus-tlsrpt-exporter
+      - find prometheus-tlsrpt-exporter -type f -exec chmod 0644 {} \;
+      - find prometheus-tlsrpt-exporter -type d -exec chmod 755 {} \;
+      - chmod +x prometheus-tlsrpt-exporter/usr/bin/prometheus-tlsrpt-exporter prometheus-tlsrpt-exporter/DEBIAN/postinst prometheus-tlsrpt-exporter/DEBIAN/prerm
+      - dpkg-deb --build prometheus-tlsrpt-exporter
+      - mv prometheus-tlsrpt-exporter.deb "prometheus-tlsrpt-exporter_${EXPORTER_VERSION}-1_all.deb"
+      - sudo -u nobody lintian "prometheus-tlsrpt-exporter_${EXPORTER_VERSION}-1_all.deb"
+      - sha256sum *.deb > SHA256SUMS

LICENSE

@@ -0,0 +1,16 @@
+Copyright 2023 s3lph
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/debian/prometheus-tlsrpt-exporter/DEBIAN/conffiles

@@ -0,0 +1 @@
+/etc/default/prometheus-tlsrpt-exporter

package/debian/prometheus-tlsrpt-exporter/DEBIAN/control

@@ -0,0 +1,10 @@
+Package: prometheus-tlsrpt-exporter
+Version: __VERSION__
+Maintainer: s3lph <s3lph@kabelsalat.ch>
+Section: web
+Priority: optional
+Architecture: all
+Depends: python3 (>= 3.11), python3-bottle, python3-jinja2
+Description: Prometheus exporter for TLSRPT reports
+ Collects TLSRPT reports and exposes their statistics per domain as
+ Prometheus metrics as well as on a web interface.

package/debian/prometheus-tlsrpt-exporter/DEBIAN/postinst

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "configure" ]]; then
+
+    if ! getent group prometheus-tlsrpt-exporter >/dev/null; then
+        groupadd --system prometheus-tlsrpt-exporter
+    fi
+
+    if ! getent passwd prometheus-tlsrpt-exporter >/dev/null; then
+        useradd --system --create-home --gid prometheus-tlsrpt-exporter --home-dir /var/lib/prometheus-tlsrpt-exporter --shell /usr/sbin/nologin prometheus-tlsrpt-exporter
+    fi
+
+    chown prometheus-tlsrpt-exporter:prometheus-tlsrpt-exporter /var/lib/prometheus-tlsrpt-exporter
+    chmod 0750 /var/lib/prometheus-tlsrpt-exporter
+
+    deb-systemd-helper enable prometheus-tlsrpt-exporter.service
+    deb-systemd-invoke restart prometheus-tlsrpt-exporter.service
+
+fi

package/debian/prometheus-tlsrpt-exporter/DEBIAN/prerm

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "remove" ]]; then
+
+    deb-systemd-invoke stop prometheus-tlsrpt-exporter
+
+fi

package/debian/prometheus-tlsrpt-exporter/etc/default/prometheus-tlsrpt-exporter

@@ -0,0 +1,2 @@
+
+ARGS="--base-url http://localhost:9123 --data-dir /var/lib/prometheus-tlsrpt-exporter --template-dir /usr/shre/prometheus-tlsrpt-exporter/templates"

package/debian/prometheus-tlsrpt-exporter/usr/share/doc/prometheus-tlsrpt-exporter/copyright

@@ -0,0 +1,16 @@
+Copyright 2023 s3lph
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

requirements.txt

@@ -0,0 +1 @@
+.

setup.cfg

@@ -0,0 +1,22 @@
+
+#
+# PyCodestyle
+#
+
+[pycodestyle]
+max-line-length = 120
+statistics = True
+
+#
+# Coverage
+#
+
+[run]
+branch = True
+parallel = True
+source = tlsrpt_exporter/
+
+[report]
+show_missing = True
+include = tlsrpt_exporter/*
+omit = */test/*.py

setup.py

@@ -0,0 +1,35 @@
+from setuptools import setup, find_packages
+
+from tlsrpt_exporter import __version__
+
+setup(
+    name='tlsrpt_exporter',
+    version=__version__,
+    url='https://git.kabelsalat.ch/s3lph/prometheus-tlsrpt-exporter',
+    license='MIT',
+    author='s3lph',
+    author_email='',
+    description='Prometheus exporter for TLSRPT reports',
+    long_description='''
+Collects TLSRPT reports and exposes their statistics per domain as
+Prometheus metrics as well as on a web interface.
+''',
+    packages=find_packages(exclude=['*.test']),
+    python_requires='>=3.11',
+    install_requires=[
+        'bottle',
+        'jinja2',
+    ],
+    extras_require={
+        'test': [
+            'coverage',
+            'pycodestyle'
+        ]
+    },
+    test_loader='unittest:TestLoader',
+    entry_points={
+        'console_scripts': [
+            'prometheus-tlsrpt-exporter = tlsrpt_exporter.__main__:main'
+        ]
+    }
+)

templates/base.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html>
+    <head>
+	<title>TLSRPT Exporter</title>
+	<link rel="stylesheet" type="text/css" href="{{ baseurl }}/ui/style.css" />
+    </head>
+    <body>
+	<header>
+	    <h1>TLSRPT Exporter</h1>
+	</header>
+	<nav>
+	    <ul>
+		<li>Web UI: <a href="{{ baseurl }}/ui">{{ baseurl }}/ui</a></li>
+		<li>Reporting URL: <a href="{{ baseurl }}/report">{{ baseurl }}/report</a></li>
+		<li>Metrics URL: <a href="{{ baseurl }}/metrics">{{ baseurl }}/metrics</a></li>
+	    </ul>
+	</nav>
+	<main>
+	    {% block main %}
+	    {% endblock %}
+	</main>
+    </body>
+</html>

templates/index.html

@@ -0,0 +1,43 @@
+{% extends 'base.html' %}
+
+{% block main %}
+<h2>TLS Reports</h2>
+{% for domain, rpts in reports.items() %}
+<section id="{{ domain }}">
+    {%- set count_s = ((rpts | map(attribute='stats') | unzip)[0] | unzip)[0] | sum %}
+    {%- set count_f = ((rpts | map(attribute='stats') | unzip)[0] | unzip)[1] | sum %}
+    <h3>{{ domain }}{% if count_s + count_f > 0%}: {{ (count_s / (count_s + count_f)) | int * 100 }}%{% endif %}</h3>
+    <table>
+	<thead>
+	    <tr>
+		<th>Date</th>
+		<th>Report ID</th>
+		<th>Success Rate</th>
+		<th>Actions</th>
+	    </tr>
+	</thead>
+	<tbody>
+	    {%- for report in rpts %}
+	    {%- set cs = report.stats | sum(attribute=0) %}
+	    {%- set cf = report.stats | sum(attribute=1) %}
+	    <tr class="report-{{ report.class }}">
+		<td class="report-date"><time datetime="{{ report.ts.isoformat('T', timespec='seconds') }}">{{ report.ts.isoformat(' ', timespec='seconds') }}</time></td>
+		<td class="report-id">{{ report.report_id }}</td>
+		<td class="report-stats">
+		    {% if cs + cf == 0 %}
+		    0
+		    {% else %}
+		    {{ (cs / (cs + cf)) | int * 100 }}% ({{ cs + cf }})
+		    {% endif %}
+		</td>
+		<td class="report-actions">
+		    <a href="{{ baseurl }}/ui/report/{{ report.report_id | urlencode }}">View</a>
+		    <a href="{{ baseurl }}/ui/report/{{ report.report_id | urlencode }}/json">Download</a>
+		</td>
+	    </tr>
+	    {% endfor %}
+	</tbody>
+    </table>
+</section>
+{% endfor %}
+{% endblock %}

templates/report.html

@@ -0,0 +1,7 @@
+{% extends 'base.html' %}
+
+{% block main %}
+<h2>TLS Report: {{ report['report-id']}}</h2>
+<pre>{{ json.dumps(report, indent=4) }}</pre>
+<a href="{{ baseurl }}/ui/report/{{ report['report-id'] | urlencode }}/json">Download JSON</a>
+{% endblock %}

templates/style.css

@@ -0,0 +1,36 @@
+html, body {
+    font-family: sans-serif;
+}
+
+span.success {
+    color: green;
+}
+span.success::pre {
+    content: "✓";
+}
+
+span.failure {
+    color: red;
+}
+span.failure::pre {
+    content: "✗";
+}
+
+table {
+    border-collapse: separate;
+    border-spacing: 0 5px;
+}
+
+tbody > tr {
+    background: #efefef;
+    border: 1px solid #dddddd;
+}
+
+tbody > tr > td {
+    margin: 0;
+    padding: 10px;
+}
+
+td.report-id {
+    font-family: monospace;
+}

tlsrpt_exporter/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1'

tlsrpt_exporter/__main__.py

@@ -0,0 +1,179 @@
+
+import argparse
+import os
+import dataclasses
+import json
+from datetime import datetime
+
+import bottle
+import jinja2
+
+
+def unzip(xs):
+    return list(zip(*xs))
+
+
+class Reporter(bottle.Bottle):
+
+    def __init__(self, metrics, ns):
+        super().__init__()
+        self.datadir = ns.data_dir
+        self.metrics = metrics
+        self.route('/', 'POST', self.report)
+
+    def handle(self, data):
+        print(data)
+        if 'report-id' not in data:
+            return
+        prefix = f'{int(datetime.utcnow().timestamp())}+'
+        if len(data.get('policies', [])) == 0:
+            return
+        domain = None
+        for p in data.get('policies', []):
+            policy_domain = p.get('policy', {}).get('policy-domain')
+            if domain is None:
+                domain = policy_domain
+            count_s = p.get('summary', {}).get('total-successful-session-count', 0)
+            count_f = p.get('summary', {}).get('total-failure-session-count', 0)
+            self.metrics.add_report(policy_domain, count_s, count_f)
+            prefix += f'{count_s},{count_f};'
+        if domain is None:
+            return
+        with open(os.path.join(self.datadir, 'reports', prefix[:-1] + '+' + domain + '+' + data['report-id']), 'w') as f:
+            json.dump(data, f)
+
+    def report(self):
+        bottle.response.add_header('Accept', 'application/tlsrpt+gzip')
+        bottle.response.add_header('Accept', 'application/tlsrpt+json')
+        if 'application/tlsrpt+gzip' in bottle.request.content_type:
+            self.handle(self.decompress(bottle.request.body))
+        elif 'application/tlsrpt+json' in  bottle.request.content_type:
+            self.handle(json.load(bottle.request.body))
+        else:
+            bottle.response.status = 406
+
+
+class Metrics(bottle.Bottle):
+
+    def __init__(self, ns):
+        super().__init__()
+        self.datadir = ns.data_dir
+        self.count = {}
+        self.count_s = {}
+        self.count_f = {}
+        self.load_reports()
+        self.route('/', 'GET', self.metrics)
+
+    def add_report(self, domain, count_s, count_f):
+        self.count_s[domain] = self.count_s.setdefault(domain, 0) + count_s
+        self.count_f[domain] = self.count_f.setdefault(domain, 0) + count_f
+        self.count[domain] = self.count.setdefault(domain, 0) + 1
+
+    def load_reports(self): 
+        for report in os.listdir(os.path.join(self.datadir, 'reports')):
+            if report.startswith('.'):
+                continue
+            _, stats, domain, _ = report.split('+', 3)
+            for p in stats.split(';'):
+                s, f = p.split(',')
+                self.add_report(domain, int(s), int(f))
+
+    def metrics(self):
+        bottle.response.headers['Content-Type'] = 'text/plain; version=0.0.4'
+        response = '# TYPE tlsrpt_successful counter\n'
+        response += '# HELP tlsrpt_successful Number of successful sessions\n'
+        for domain, count in self.count_s.items():
+            response += f'tlsrpt_successful{{domain="{domain}"}} {count}\n'
+        response += '# TYPE tlsrpt_failed counter\n'
+        response += '# HELP tlsrpt_failed Number of failed sessions\n'
+        for domain, count in self.count_f.items():
+            response += f'tlsrpt_failed{{domain="{domain}"}} {count}\n'
+        response += '# TYPE tlsrpt_count counter\n'
+        response += '# HELP tlsrpt_count Number of reports\n'
+        for domain, count in self.count.items():
+            response += f'tlsrpt_count{{domain="{domain}"}} {count}\n'
+        return response
+
+
+@dataclasses.dataclass
+class ReportInfo:
+    ts: datetime
+    stats: list[tuple[int, int]]
+    report_id: str
+    filename: str
+
+
+class UI(bottle.Bottle):
+
+    def __init__(self, ns):
+        super().__init__()
+        self.datadir = ns.data_dir
+        self.baseurl = ns.base_url or f'//{ns.host}:{ns.port}'
+        self.route('/', 'GET', self.list_reports)
+        self.route('/report/<report_id>', 'GET', self.report_details)
+        self.route('/report/<report_id>/json', 'GET', self.report_download)
+        self.route('/style.css', 'GET', self.style)
+        self.env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader([ns.template_dir])
+        )
+        self.env.filters['unzip'] = unzip
+
+    def style(self):
+        bottle.response.headers['Content-Type'] = 'text/css'
+        return self.env.get_template('style.css').render()
+
+    def list_reports(self):
+        reports = {}
+        for report in os.listdir(os.path.join(self.datadir, 'reports')):
+            if report.startswith('.'):
+                continue
+            dt, stats, domain, report_id = report.split('+', 3)
+            dt = datetime.fromtimestamp(int(float(dt)))
+            stats = [(int(p.split(',', 1)[0]), int(p.split(',', 1)[1])) for p in stats.split(';')]
+            reports.setdefault(domain, []).append(ReportInfo(dt, stats, report_id, report))
+        for v in reports.values():
+            v.sort(key=lambda x: x.ts)
+        return self.env.get_template('index.html').render(baseurl=self.baseurl, reports=reports)
+
+    def get_report(self, report_id: str):
+        report = None
+        for r in os.listdir(os.path.join(self.datadir, 'reports')):
+            tokens = r.split('+', 3)
+            if len(tokens) != 4:
+                continue
+            if tokens[3] != report_id:
+                continue
+            with open(os.path.join(self.datadir, 'reports', r), 'r') as f:
+                report = f.read()
+        return report
+
+    def report_details(self, report_id: str):
+        report = self.get_report(report_id)
+        return self.env.get_template('report.html').render(baseurl=self.baseurl, report=json.loads(report), json=json)
+
+    def report_download(self, report_id: str):
+        report = self.get_report(report_id)
+        bottle.response.headers['Content-Type'] = 'application/tlsrpt+json'
+        bottle.response.headers['Content-Disposition'] = f'attachment; filename={report_id}.json'
+        return report
+
+
+def main():
+    ap = argparse.ArgumentParser('prometheus-tlsrpt-exporter')
+    ap.add_argument('--host', '-H', type=str, default='::', help='Address to bind to')
+    ap.add_argument('--port', '-p', type=int, default=9123, help='Port to bind to')
+    ap.add_argument('--base-url', '-B', type=str, default=None, help='Base URL')
+    ap.add_argument('--data-dir', '-d', type=str, default='/tmp/prometheus-tlsrpt-exporter', help='Directory to save reports to')
+    ap.add_argument('--template-dir', '-t', type=str, default='templates', help='Directory to load templates from')
+    ns = ap.parse_args()
+    os.makedirs(os.path.join(ns.data_dir, 'reports'), exist_ok=True)
+
+    metrics = Metrics(ns)
+    bottle.mount('/report', Reporter(metrics, ns))
+    bottle.mount('/metrics', metrics)
+    bottle.mount('/ui', UI(ns))
+    bottle.run(host=ns.host, port=ns.port)
+
+
+if __name__ == '__main__':
+    main()