diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9180290..b96e205 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,7 +10,7 @@ stages:
 
 
 before_script:
-- pip3 install coverage pycodestyle mypy aiosmtpd
+- pip3 install coverage pycodestyle mypy aiosmtpd deepdiff
 - export MULTISCHLEUDER_VERSION=$(python -c 'import multischleuder; print(multischleuder.__version__)')
 
 
@@ -66,7 +66,7 @@ schleuder:
     # Run a second time - should be idempotent and not trigger any new mails
     - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
     - sleep 5  # wait for mail delivery
-    - test/report.sh
+    - test/report.py
     - kill -9 ${API_DAEMON_PID} || true
     - /usr/sbin/postmulti -i - -p stop
     - sleep 5  # wait for daemons to terminate
diff --git a/test/multischleuder.yml b/test/multischleuder.yml
index 19a5b15..95d5ea4 100644
--- a/test/multischleuder.yml
+++ b/test/multischleuder.yml
@@ -37,7 +37,7 @@ smtp:
 
 conflict:
   interval: 3600  # 1 hour - you don't want this in production
-  statefile: conflict.json
+  statefile: /tmp/conflict.json
   template: |
     Hi {subscriber},
     
diff --git a/test/report.py b/test/report.py
new file mode 100755
index 0000000..266b144
--- /dev/null
+++ b/test/report.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import json
+import mailbox
+import os
+import subprocess
+
+import deepdiff
+import pgpy
+
+
+def subscribermap(subs):
+    return {
+        s: pgpy.PGPKey.from_file(f'/tmp/{s}.asc')[0].fingerprint.replace(' ', '')
+        for s in subs
+    }
+
+
+def keymap(subs):
+    return {
+        pgpy.PGPKey.from_file(f'/tmp/{s}.asc')[0].fingerprint.replace(' ', ''): s
+        for s in subs
+    }
+
+# Test subscribers
+
+subsp = subprocess.Popen(['/usr/bin/schleuder-cli', 'subscriptions', 'list', 'test-global@schleuder.example.org'],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+subsout, _ = subsp.communicate()
+keysp = subprocess.Popen(['/usr/bin/schleuder-cli', 'keys', 'list', 'test-global@schleuder.example.org'],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+keysout, _ = keysp.communicate()
+fprp = subprocess.Popen(['/usr/bin/schleuder-cli', 'lists', 'show', 'test-global@schleuder.example.org', 'fingerprint'],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+fprout, _ = fprp.communicate()
+
+expected_subscribers = subscribermap([
+    'aaron.example@example.org',
+    'admin@example.org',
+    'alex.example@example.org',
+    'amy.example@example.org',
+    'andy.example@example.org',
+    'anna.example@example.org',
+    'anotherspammer@example.org'
+])
+
+actual_subscribers = {}
+for s in subsout.decode().splitlines():
+    if '\t' not in s:
+        continue
+    uid, fpr, *_ = s.split('\t', 2)
+    actual_subscribers[uid] = fpr
+
+subsdiff = deepdiff.DeepDiff(expected_subscribers, actual_subscribers)
+if len(subsdiff) > 0:
+    print(subsdiff)
+    exit(1)
+
+# Test keys
+
+expected_keys = keymap([
+    'aaron.example@example.org',
+    'admin@example.org',
+    'alex.example@example.org',
+    'amy.example@example.org',
+    'andy.example@example.org',
+    'anna.example@example.org',
+    'anotherspammer@example.org'
+])
+expected_keys[fprout.strip().decode()] = 'test-global@schleuder.example.org'
+
+actual_keys = {}
+for s in keysout.decode().splitlines():
+    if ' ' not in s:
+        continue
+    fpr, uid = s.split(' ', 1)
+    actual_keys[fpr] = uid
+
+keysdiff = deepdiff.DeepDiff(expected_keys, actual_keys)
+if len(keysdiff) > 0:
+    print(keysdiff)
+    exit(1)
+
+# Test mbox
+
+mbox = mailbox.mbox('/var/spool/mail/root')
+if len(mbox) != 1:
+    print(f'Expected 1 message in mbox, got {len(mbox)}')
+    exit(1)
+_, msg = mbox.popitem()
+mbox.close()
+digest = msg['X-MultiSchleuder-Digest'].strip()
+
+if msg['From'] != 'test-global-owner@schleuder.example.org':
+    print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg["From"]}')
+    exit(1)
+if msg['To'] != 'andy.example@example.org':
+    print(f'Expected "To: andy.example@example.org", got {msg["To"]}')
+    exit(1)
+if msg['Auto-Submitted'] != 'auto-generated':
+    print(f'Expected "Auto-Submitted: auto-generated", got {msg["Auto-Submitted"]}')
+    exit(1)
+if msg['Precedence'] != 'list':
+    print(f'Expected "Precedence: list", got {msg["Precedence"]}')
+    exit(1)
+
+
+# Test conflict statefile
+
+with open('/tmp/conflict.json', 'r') as f:
+    conflict = json.load(f)
+if len(conflict) != 1:
+    print('Expected 1 entry in conflict statefile, got:')
+    print(json.dumps(conflict))
+    exit(1)
+if digest not in conflict:
+    print(f'Expected key "{digest}" in conflict statefile, got:')
+    print(json.dumps(conflict))
+    exit(1)
+
+print('All checks passed!')
diff --git a/test/report.sh b/test/report.sh
deleted file mode 100755
index 71dd9e7..0000000
--- a/test/report.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-echo Expected:
-echo
-echo aaron.example@example.org
-echo admin@example.org
-echo alex.example@example.org
-echo amy.example@example.org
-echo andy.example@example.org
-echo anna.example@example.org
-echo anotherspammer@example.org
-echo -- ---
-echo Actual:
-echo
-schleuder-cli subscriptions list test-global@schleuder.example.org
-
-schleuder-cli keys list test-global@schleuder.example.org
-
-cat /var/spool/mail/root