Refactor to take mailing out of the key conflict resolution components. Add admin reporting

This commit is contained in:
s3lph 2022-04-17 06:00:46 +02:00
parent 5c38407be8
commit ade119ca69
13 changed files with 503 additions and 306 deletions

View file

@ -66,6 +66,7 @@ schleuder:
# Run a second time - should be idempotent and not trigger any new mails # Run a second time - should be idempotent and not trigger any new mails
- python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose - python3 -m coverage run --rcfile=setup.cfg -m multischleuder --config test/multischleuder.yml --verbose
- sleep 5 # wait for mail delivery - sleep 5 # wait for mail delivery
- cat /var/spool/mail/root
- test/report.py - test/report.py
- kill -9 ${API_DAEMON_PID} || true - kill -9 ${API_DAEMON_PID} || true
- /usr/sbin/postmulti -i - -p stop - /usr/sbin/postmulti -i - -p stop

View file

@ -64,6 +64,19 @@ class SchleuderApi:
lists = self.__request('lists.json') lists = self.__request('lists.json')
return [SchleuderList.from_api(**s) for s in lists] return [SchleuderList.from_api(**s) for s in lists]
def get_list_admins(self, schleuder: SchleuderList) -> List[SchleuderSubscriber]:
response = self.__request('subscriptions.json', schleuder.id)
admins: List[SchleuderSubscriber] = []
for r in response:
if not r['admin']:
continue
key: Optional[SchleuderKey] = None
if r['fingerprint']:
key = self.get_key(r['fingerprint'], schleuder)
admin = SchleuderSubscriber.from_api(key, **r)
admins.append(admin)
return admins
# Subscriber Management # Subscriber Management
def get_subscribers(self, schleuder: SchleuderList) -> List[SchleuderSubscriber]: def get_subscribers(self, schleuder: SchleuderList) -> List[SchleuderSubscriber]:

View file

@ -1,5 +1,5 @@
from typing import Dict, List, Optional from typing import Dict, List, Optional, Tuple
import email.mime.base import email.mime.base
import email.mime.application import email.mime.application
@ -9,196 +9,121 @@ import email.utils
import hashlib import hashlib
import json import json
import logging import logging
import os
import struct import struct
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime from datetime import datetime
import pgpy # type: ignore import pgpy # type: ignore
from multischleuder.reporting import ConflictMessage
from multischleuder.types import SchleuderKey, SchleuderSubscriber from multischleuder.types import SchleuderKey, SchleuderSubscriber
from multischleuder.smtp import SmtpClient
class ConflictMessage:
def __init__(self,
schleuder: str,
chosen: SchleuderSubscriber,
affected: List[SchleuderSubscriber],
mail_from: str,
template: str):
self._schleuder: str = schleuder
self._chosen: SchleuderSubscriber = chosen
self._affected: List[SchleuderSubscriber] = affected
self._template: str = template
self._from: str = mail_from
# Generate a SHA1 digest that only changes when the subscription list changes
self._digest = self._make_digest()
def _make_digest(self) -> str:
# Sort so the hash stays the same if the set of subscriptions is the same.
# There is no guarantee that the subs are in any specific order.
subs: List[SchleuderSubscriber] = list(self._affected)
subs.sort(key=lambda x: x.schleuder)
h = hashlib.new('sha1')
# Include the chosen email an source sub-list
h.update(struct.pack('!sd',
self._chosen.email.encode(),
self._chosen.schleuder))
# Include all subscriptions, including the FULL key
for s in subs:
key = b'N/A'
if s.key is not None:
key = s.key.blob.encode()
h.update(struct.pack('!ds', s.schleuder, key))
return h.hexdigest()
@property
def digest(self) -> str:
return self._digest
@property
def mime(self) -> email.mime.base.MIMEBase:
# Render the message body
fpr = 'N/A' if self._chosen.key is None else self._chosen.key.fingerprint
_chosen = f'{fpr} {self._chosen.email}'
_affected = ''
for affected in self._affected:
fpr = 'N/A' if affected.key is None else affected.key.fingerprint
_affected += f'{fpr} {affected.schleuder}\n'
msg: str = self._template.format(
subscriber=self._chosen.email,
schleuder=self._schleuder,
chosen=_chosen,
affected=_affected
)
# Encrypt to all keys, if possible. Fall back to unencrypted otherwise - PGPy does not
# support every possible key algorithm yet, esp. it can't encrypt to ed25519 keys.
try:
mime: email.mime.base.MIMEBase = self._encrypt_message(msg)
except Exception:
mime = email.mime.text.MIMEText(msg, _subtype='plain', _charset='utf-8')
# Set all the email headers
mime['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
mime['From'] = self._from
mime['Reply-To'] = self._from
mime['To'] = self._chosen.email
mime['Date'] = email.utils.formatdate()
mime['Auto-Submitted'] = 'auto-generated'
mime['Precedence'] = 'list'
mime['List-Id'] = f'<{self._schleuder.replace("@", ".")}>'
mime['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
mime['X-MultiSchleuder-Digest'] = self._digest
return mime
def _encrypt_message(self, msg: str) -> email.mime.base.MIMEBase:
pgp = pgpy.PGPMessage.new(msg)
# Encrypt the message to all keys
cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
sessionkey = cipher.gen_key()
try:
for affected in self._affected:
if affected.key is not None:
key, _ = pgpy.PGPKey.from_blob(affected.key.blob)
key._require_usage_flags = False
pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
finally:
del sessionkey
# Build the MIME message
# First the small "version" part ...
mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
mp1['Content-Description'] = 'PGP/MIME version identification'
mp1['Content-Disposition'] = 'attachment'
# ... then the actual encrypted payload ...
mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
mp2['Content-Description'] = 'OpenPGP encrypted message'
mp2['Content-Disposition'] = 'inline; filename="message.asc"'
# ... and finally the root multipart container
mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
mp0.attach(mp1)
mp0.attach(mp2)
return mp0
class KeyConflictResolution: class KeyConflictResolution:
def __init__(self, smtp: SmtpClient, interval: int, statefile: str, template: str): def __init__(self, interval: int, statefile: str, template: str):
self._smtp = smtp
self._interval: int = interval self._interval: int = interval
self._state_file: str = statefile self._state_file: str = statefile
self._template: str = template self._template: str = template
self._messages: List[ConflictMessage] = [] self._dry_run: bool = False
self._logger: logging.Logger = logging.getLogger() self._logger: logging.Logger = logging.getLogger()
def dry_run(self):
self._dry_run = True
def resolve(self, def resolve(self,
target: str, target: str,
mail_from: str, mail_from: str,
subscriptions: List[SchleuderSubscriber]) -> List[SchleuderSubscriber]: subscriptions: List[SchleuderSubscriber]) -> Tuple[List[SchleuderSubscriber], List[ConflictMessage]]:
subs: Dict[str, List[SchleuderSubscriber]] = OrderedDict() subs: Dict[str, List[SchleuderSubscriber]] = OrderedDict()
for s in subscriptions: for s in subscriptions:
subs.setdefault(s.email, []).append(s) subs.setdefault(s.email, []).append(s)
# Perform conflict resolution for each set of subscribers with the same email # Perform conflict resolution for each set of subscribers with the same email
resolved = [self._resolve(target, mail_from, s) for s in subs.values()] resolved: List[SchleuderSubscriber] = []
return [r for r in resolved if r is not None] conflicts: List[ConflictMessage] = []
for c in subs.values():
r, m = self._resolve(target, mail_from, c)
if r is not None:
resolved.append(r)
if m is not None:
conflicts.append(m)
return resolved, conflicts
def _resolve(self, def _resolve(self,
target: str, target: str,
mail_from: str, mail_from: str,
subscriptions: List[SchleuderSubscriber]) -> Optional[SchleuderSubscriber]: subscriptions: List[SchleuderSubscriber]) \
-> Tuple[Optional[SchleuderSubscriber], Optional[ConflictMessage]]:
notnull = [s for s in subscriptions if s.key is not None] notnull = [s for s in subscriptions if s.key is not None]
if len(notnull) == 0: if len(notnull) == 0:
return None return None, None
if len({s.key.blob for s in subscriptions if s.key is not None}) == 1: if len({s.key.blob for s in subscriptions if s.key is not None}) == 1:
# No conflict if all keys are the same # No conflict if all keys are the same
return notnull[0] return notnull[0], None
# Conflict Resolution: Choose the OLDEST subscription with a key, but notify using ALL keys # Conflict Resolution: Choose the OLDEST subscription with a key, but notify using ALL keys
earliest: SchleuderSubscriber = min(notnull, key=lambda x: x.created_at) earliest: SchleuderSubscriber = min(notnull, key=lambda x: x.created_at)
assert earliest.key is not None # Make mypy happy; it can't know that earliest.key can't be None assert earliest.key is not None # Make mypy happy; it can't know that earliest.key can't be None
self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.key.fingerprint}:') self._logger.debug(f'Key Conflict for {earliest.email} in lists, chose {earliest.key.fingerprint}:')
for s in subscriptions: for s in subscriptions:
fpr = 'N/A' if s.key is None else s.key.fingerprint fpr = 'no key' if s.key is None else s.key.fingerprint
self._logger.debug(f' - {s.schleuder}: {fpr}') self._logger.debug(f' - {s.schleuder}: {fpr}')
# At this point, the messages are only queued locally, they will be sent afterwards # Generate a SHA1 digest that only changes when the subscription list changes
msg = ConflictMessage( digest = self._make_digest(earliest, subscriptions)
target, msg: Optional[ConflictMessage] = None
earliest, # Create a conflict message only if it hasn't been sent recently
subscriptions, if self._should_send(digest):
mail_from, msg = ConflictMessage(
self._template target,
) earliest,
self._messages.append(msg) subscriptions,
digest,
mail_from,
self._template
)
# Return the result of conflict resolution # Return the result of conflict resolution
return earliest return earliest, msg
def send_messages(self, dry_run: bool = False): def _should_send(self, digest: str) -> bool:
now = int(datetime.utcnow().timestamp()) now = int(datetime.utcnow().timestamp())
exists = os.path.exists(self._state_file)
with open(self._state_file, 'a+') as f: with open(self._state_file, 'a+') as f:
f.seek(0) f.seek(0)
try: if not exists:
state: Dict[str, int] = json.load(f) state: Dict[str, int] = {}
except BaseException: else:
self._logger.exception('Cannot read statefile. WARNING: This could lead to spamming') try:
# TODO: This could lead to a situation where multischleuder becomes a spammer state = json.load(f)
state = {} except BaseException:
self._logger.exception('Cannot read statefile. Not sending any messages!')
return False
# Remove all state entries older than conflict_interval # Remove all state entries older than conflict_interval
state = {k: v for k, v in state.items() if now-v < self._interval} state = {k: v for k, v in state.items() if now-v < self._interval}
# Remove all messages not already sent recently # Should send if it has not been sent before or has been removed in the line above
msgs = [m for m in self._messages if m.digest not in state] send = digest not in state
# Add all remaining messages to state dict # Add all remaining messages to state dict
for m in msgs: if send:
state[m.digest] = now state[digest] = now
# Write the new state to file # Write the new state to file
if not dry_run: if not self._dry_run:
f.seek(0) f.seek(0)
f.truncate() f.truncate()
json.dump(state, f) json.dump(state, f)
# Finally send the mails return send
if len(msgs) > 0 and not dry_run:
with self._smtp as smtp: def _make_digest(self, chosen: SchleuderSubscriber, candidates: List[SchleuderSubscriber]) -> str:
for m in msgs: # Sort so the hash stays the same if the set of subscriptions is the same.
msg = m.mime # There is no guarantee that the subs are in any specific order.
self._logger.debug(f'MIME Message:\n{str(m)}') subs: List[SchleuderSubscriber] = sorted(candidates, key=lambda x: x.schleuder)
self._logger.info(f'Sending key conflict message to {msg["To"]}') h = hashlib.new('sha1')
smtp.send_message(msg) # Include the chosen email an source sub-list
# Clear conflict messages h.update(struct.pack('!sd',
self._messages = [] chosen.email.encode(),
chosen.schleuder))
# Include all subscriptions, including the FULL key
for s in subs:
key = b'no key'
if s.key is not None:
key = s.key.blob.encode()
h.update(struct.pack('!ds', s.schleuder, key))
return h.hexdigest()

View file

@ -16,6 +16,7 @@ from multischleuder.smtp import SmtpClient
def parse_list_config(api: SchleuderApi, def parse_list_config(api: SchleuderApi,
kcr: KeyConflictResolution, kcr: KeyConflictResolution,
smtp: SmtpClient,
config: Dict[str, Any]) -> 'MultiList': config: Dict[str, Any]) -> 'MultiList':
target = config['target'] target = config['target']
default_from = target.replace('@', '-owner@') default_from = target.replace('@', '-owner@')
@ -29,7 +30,10 @@ def parse_list_config(api: SchleuderApi,
unmanaged=unmanaged, unmanaged=unmanaged,
mail_from=mail_from, mail_from=mail_from,
api=api, api=api,
kcr=kcr kcr=kcr,
smtp=smtp,
send_admin_reports=config.get('send_admin_reports', True),
send_conflict_messages=config.get('send_conflict_messages', True),
) )
@ -38,18 +42,21 @@ def parse_config(ns: argparse.Namespace) -> List['MultiList']:
c = yaml.safe_load(f) c = yaml.safe_load(f)
api = SchleuderApi(**c['api']) api = SchleuderApi(**c['api'])
if ns.dry_run:
api.dry_run()
smtp_config = c.get('smtp', {}) smtp_config = c.get('smtp', {})
smtp = SmtpClient.parse(smtp_config) smtp = SmtpClient.parse(smtp_config)
kcr_config = c.get('conflict', {}) kcr_config = c.get('conflict', {})
kcr = KeyConflictResolution(smtp, **kcr_config) kcr = KeyConflictResolution(**kcr_config)
if ns.dry_run:
api.dry_run()
kcr.dry_run()
smtp.dry_run()
lists = [] lists = []
for clist in c.get('lists', []): for clist in c.get('lists', []):
ml = parse_list_config(api, kcr, clist) ml = parse_list_config(api, kcr, smtp, clist)
lists.append(ml) lists.append(ml)
return lists return lists

View file

@ -5,6 +5,8 @@ import logging
from multischleuder.api import SchleuderApi from multischleuder.api import SchleuderApi
from multischleuder.conflict import KeyConflictResolution from multischleuder.conflict import KeyConflictResolution
from multischleuder.reporting import AdminReport, Message
from multischleuder.smtp import SmtpClient
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
@ -17,7 +19,10 @@ class MultiList:
banned: List[str], banned: List[str],
mail_from: str, mail_from: str,
api: SchleuderApi, api: SchleuderApi,
kcr: KeyConflictResolution): kcr: KeyConflictResolution,
smtp: SmtpClient,
send_admin_reports: bool,
send_conflict_messages: bool):
self._sources: List[str] = sources self._sources: List[str] = sources
self._target: str = target self._target: str = target
self._unmanaged: List[str] = unmanaged self._unmanaged: List[str] = unmanaged
@ -25,11 +30,16 @@ class MultiList:
self._mail_from: str = mail_from self._mail_from: str = mail_from
self._api: SchleuderApi = api self._api: SchleuderApi = api
self._kcr: KeyConflictResolution = kcr self._kcr: KeyConflictResolution = kcr
self._smtp: SmtpClient = smtp
self._send_admin_reports: bool = send_admin_reports
self._send_conflict_messages: bool = send_conflict_messages
self._messages: List[Message] = []
self._logger: logging.Logger = logging.getLogger() self._logger: logging.Logger = logging.getLogger()
def process(self, dry_run: bool = False): def process(self, dry_run: bool = False):
self._logger.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}') self._logger.info(f'Processing: {self._target} {"DRY RUN" if dry_run else ""}')
target_list, sources = self._lists_by_name() target_list, sources = self._lists_by_name()
target_admins = self._api.get_list_admins(target_list)
# Get current subs, except for unmanaged adresses # Get current subs, except for unmanaged adresses
current_subs: Set[SchleuderSubscriber] = { current_subs: Set[SchleuderSubscriber] = {
s s
@ -46,7 +56,9 @@ class MultiList:
continue continue
all_subs.append(s) all_subs.append(s)
# ... which is taken care of by the key conflict resolution routine # ... which is taken care of by the key conflict resolution routine
resolved = self._kcr.resolve(self._target, self._mail_from, all_subs) resolved, conflicts = self._kcr.resolve(self._target, self._mail_from, all_subs)
if self._send_conflict_messages:
self._messages.extend(conflicts)
intended_subs: Set[SchleuderSubscriber] = set(resolved) intended_subs: Set[SchleuderSubscriber] = set(resolved)
intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None} intended_keys: Set[SchleuderKey] = {s.key for s in intended_subs if s.key is not None}
# Determine the change set # Determine the change set
@ -71,12 +83,25 @@ class MultiList:
for key in to_remove: for key in to_remove:
self._api.delete_key(key, target_list) self._api.delete_key(key, target_list)
self._logger.info(f'Removed key: {key}') self._logger.info(f'Removed key: {key}')
# Finally, send any queued key conflict messages.
self._kcr.send_messages()
if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0: if len(to_add) + len(to_subscribe) + len(to_unsubscribe) + len(to_remove) == 0:
self._logger.info(f'No changes for {self._target}') self._logger.info(f'No changes for {self._target}')
else: else:
self._logger.info(f'Finished processing: {self._target}') if self._send_admin_reports:
for admin in target_admins:
report = AdminReport(self._target, admin.email, self._mail_from,
admin.key.blob if admin.key is not None else None,
to_subscribe, to_unsubscribe, to_update, to_add, to_remove)
self._messages.append(report)
print(str(report))
# Finally, send any queued messages.
if len(self._messages) > 0:
self._logger.info(f'Sending f{len(self._messages)} messages')
self._smtp.send_messages(self._messages)
self._messages = []
self._logger.info(f'Finished processing: {self._target}')
def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]: def _lists_by_name(self) -> Tuple[SchleuderList, List[SchleuderList]]:
lists = self._api.get_lists() lists = self._api.get_lists()

181
multischleuder/reporting.py Normal file
View file

@ -0,0 +1,181 @@
from typing import Dict, List, Optional, Set
import abc
import email.mime.base
import email.mime.application
import email.mime.multipart
import email.mime.text
import email.utils
import hashlib
import json
import logging
import struct
from collections import OrderedDict
from datetime import datetime
import pgpy # type: ignore
from multischleuder.types import SchleuderKey, SchleuderSubscriber
class Message(abc.ABC):
def __init__(self,
schleuder: str,
mail_from: str,
mail_to: str,
content: str,
encrypt_to: List[str]):
self._schleuder: str = schleuder
self._from: str = mail_from
self._to: str = mail_to
self._keys: List[str] = encrypt_to
self._mime: email.mime.base.MIMEBase = self._make_mime(content)
@property
def mime(self) -> email.mime.base.MIMEBase:
return self._mime
def _make_mime(self, content: str) -> email.mime.base.MIMEBase:
# Encrypt to all keys, if possible. Fall back to unencrypted otherwise - PGPy does not
# support every possible key algorithm yet, esp. it can't encrypt to ed25519 keys.
try:
self._mime = self._encrypt_message(content)
except Exception:
self._mime = email.mime.text.MIMEText(content, _subtype='plain', _charset='utf-8')
# Set all the email headers
self._mime['From'] = self._from
self._mime['Reply-To'] = self._from
self._mime['To'] = self._to
self._mime['Date'] = email.utils.formatdate()
self._mime['Auto-Submitted'] = 'auto-generated'
self._mime['Precedence'] = 'list'
self._mime['List-Id'] = f'<{self._schleuder.replace("@", ".")}>'
self._mime['List-Help'] = '<https://gitlab.com/s3lph/multischleuder>'
return self._mime
def _encrypt_message(self, content: str) -> email.mime.base.MIMEBase:
if len(self._keys) == 0:
return email.mime.text.MIMEText(content, _subtype='plain', _charset='utf-8')
pgp = pgpy.PGPMessage.new(content)
# Encrypt the message to all keys
cipher = pgpy.constants.SymmetricKeyAlgorithm.AES256
sessionkey = cipher.gen_key()
try:
for k in self._keys:
key, _ = pgpy.PGPKey.from_blob(k)
pgp = key.encrypt(pgp, cipher=cipher, sessionkey=sessionkey)
finally:
del sessionkey
# Build the MIME message
# First the small "version" part ...
mp1 = email.mime.application.MIMEApplication('Version: 1', _subtype='pgp-encrypted')
mp1['Content-Description'] = 'PGP/MIME version identification'
mp1['Content-Disposition'] = 'attachment'
# ... then the actual encrypted payload ...
mp2 = email.mime.application.MIMEApplication(str(pgp), _subtype='octet-stream', name='encrypted.asc')
mp2['Content-Description'] = 'OpenPGP encrypted message'
mp2['Content-Disposition'] = 'inline; filename="message.asc"'
# ... and finally the root multipart container
mp0 = email.mime.multipart.MIMEMultipart(_subtype='encrypted', protocol='application/pgp-encrypted')
mp0.attach(mp1)
mp0.attach(mp2)
return mp0
class ConflictMessage(Message):
def __init__(self,
schleuder: str,
chosen: SchleuderSubscriber,
affected: List[SchleuderSubscriber],
digest: str,
mail_from: str,
template: str):
# Render the message body
fpr = 'no key' if chosen.key is None else chosen.key.fingerprint
_chosen = f'{fpr} {chosen.email}'
_affected = ''
for a in affected:
fpr = 'no key' if a.key is None else a.key.fingerprint
_affected += f'{fpr} {a.schleuder}\n'
content = template.format(
subscriber=chosen.email,
schleuder=schleuder,
chosen=_chosen,
affected=_affected
)
self._chosen: SchleuderSubscriber = chosen
self._affected: List[SchleuderSubscriber] = affected
self._template: str = template
super().__init__(
schleuder=schleuder,
mail_from=mail_from,
mail_to=chosen.email,
content=content,
encrypt_to=[s.key.blob for s in affected if s.key is not None]
)
self.mime['Subject'] = f'MultiSchleuder {self._schleuder} - Key Conflict'
self.mime['X-MultiSchleuder-Digest'] = digest
class AdminReport(Message):
def __init__(self,
schleuder: str,
mail_to: str,
mail_from: str,
encrypt_to: Optional[str],
subscribed: Set[SchleuderSubscriber],
unsubscribed: Set[SchleuderSubscriber],
updated: Set[SchleuderSubscriber],
added: Set[SchleuderKey],
removed: Set[SchleuderKey]):
if len(subscribed) == 0 and len(unsubscribed) == 0 and \
len(removed) == 0 and len(added) == 0 and len(updated) == 0:
raise ValueError('No changes, not creating admin report')
content = f'''
== Admin Report for MultiSchleuder {schleuder} ==
'''
if len(subscribed) > 0:
content += '''
>>> Subscribed:
'''
for s in subscribed:
fpr = 'no key' if s.key is None else s.key.fingerprint
content += f'{s.email} ({fpr})\n'
if len(unsubscribed) > 0:
content += '''
>>> Unsubscribed:
'''
for s in unsubscribed:
fpr = 'no key' if s.key is None else s.key.fingerprint
content += f'{s.email} ({fpr})\n'
if len(updated) > 0:
content += '''
>>> Subscriber keys changed:
'''
for s in updated:
fpr = 'no key' if s.key is None else s.key.fingerprint
content += f'{s.email} ({fpr})\n'
if len(added) > 0:
content += '''
>>> Keys added:
'''
for k in added:
content += f'{k.fingerprint} ({k.email})\n'
if len(removed) > 0:
content += '''
>>> Keys deleted:
'''
for k in removed:
content += f'{k.fingerprint} ({k.email})\n'
super().__init__(
schleuder=schleuder,
mail_from=mail_from,
mail_to=mail_to,
content=content,
encrypt_to=[encrypt_to] if encrypt_to is not None else []
)
self.mime['Subject'] = f'MultiSchleuder Admin Report: {self._schleuder}'

View file

@ -1,11 +1,13 @@
from typing import Any, Dict, Optional from typing import Any, Dict, List, Optional
import email import email
import enum import enum
import logging import logging
import smtplib import smtplib
from multischleuder.reporting import Message
class TlsMode(enum.Enum): class TlsMode(enum.Enum):
PLAIN = 'smtp', 25 PLAIN = 'smtp', 25
@ -39,6 +41,7 @@ class SmtpClient:
self._username: Optional[str] = username self._username: Optional[str] = username
self._password: Optional[str] = password self._password: Optional[str] = password
self._smtp: Optional[smtplib.SMTP] = None self._smtp: Optional[smtplib.SMTP] = None
self._dry_run: bool = False
self._logger = logging.getLogger() self._logger = logging.getLogger()
@staticmethod @staticmethod
@ -52,10 +55,22 @@ class SmtpClient:
password=config.get('password') password=config.get('password')
) )
def dry_run(self):
self._dry_run = True
def send_messages(self, messages: List[Message]):
if len(messages) > 0 and not self._dry_run:
with self as smtp:
for m in messages:
msg = m.mime
self._logger.debug(f'MIME Message:\n{str(msg)}')
self.send_message(msg)
def send_message(self, msg: email.message.Message): def send_message(self, msg: email.message.Message):
if self._smtp is None: if self._smtp is None:
raise RuntimeError('SMTP connection is not established') raise RuntimeError('SMTP connection is not established')
self._smtp.send_message(msg) if not self._dry_run:
self._smtp.send_message(msg)
self._logger.debug(f'Sent email message.') self._logger.debug(f'Sent email message.')
def __enter__(self): def __enter__(self):

View file

@ -86,7 +86,7 @@ _SUBSCRIBER_RESPONSE_NOKEY = '''
"list_id": 42, "list_id": 42,
"email": "andy.example@example.org", "email": "andy.example@example.org",
"fingerprint": "", "fingerprint": "",
"admin": false, "admin": true,
"delivery_enabled": true, "delivery_enabled": true,
"created_at": "2022-04-15T01:11:12.123Z", "created_at": "2022-04-15T01:11:12.123Z",
"updated_at": "2022-04-15T01:11:12.123Z" "updated_at": "2022-04-15T01:11:12.123Z"
@ -144,6 +144,19 @@ class TestSchleuderApi(unittest.TestCase):
self.assertEqual('Basic c2NobGV1ZGVyOjg2Y2YyNjc2ZDA2NWRjOTAyNTQ4ZTU2M2FiMjJiNTc4NjhlZDJlYjY=', self.assertEqual('Basic c2NobGV1ZGVyOjg2Y2YyNjc2ZDA2NWRjOTAyNTQ4ZTU2M2FiMjJiNTc4NjhlZDJlYjY=',
mock.call_args[0][0].headers.get('Authorization')) mock.call_args[0][0].headers.get('Authorization'))
@patch('urllib.request.urlopen')
def test_get_list_admins(self, mock):
api = self._mock_api(mock)
admins = api.get_list_admins(SchleuderList(42, '', ''))
self.assertEqual(0, len(admins))
api = self._mock_api(mock, nokey=True)
admins = api.get_list_admins(SchleuderList(42, '', ''))
self.assertEqual(1, len(admins))
self.assertEqual(24, admins[0].id)
self.assertEqual('andy.example@example.org', admins[0].email)
self.assertEqual(42, admins[0].schleuder)
self.assertIsNone(admins[0].key)
@patch('urllib.request.urlopen') @patch('urllib.request.urlopen')
def test_get_subscribers(self, mock): def test_get_subscribers(self, mock):
api = self._mock_api(mock) api = self._mock_api(mock)

View file

@ -79,51 +79,36 @@ class TestKeyConflictResolution(unittest.TestCase):
key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id) key2 = SchleuderKey(_PRIVKEY_2.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_2.pubkey), sch2.id)
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
msg1 = ConflictMessage( digest1 = kcr._make_digest(sub2, [sub1, sub2])
schleuder='', digest2 = kcr._make_digest(sub2, [sub2, sub1])
chosen=sub2, digest3 = kcr._make_digest(sub1, [sub1, sub2])
affected=[sub1, sub2], digest4 = kcr._make_digest(sub1, [sub2, sub1])
mail_from='',
template='')
msg2 = ConflictMessage(
schleuder='',
chosen=sub2,
affected=[sub2, sub1],
mail_from='',
template='')
msg3 = ConflictMessage(
schleuder='',
chosen=sub1,
affected=[sub2, sub1],
mail_from='',
template='')
msg4 = ConflictMessage(
schleuder='foo',
chosen=sub1,
affected=[sub2, sub1],
mail_from='bar',
template='baz')
self.assertEqual(msg1.digest, msg2.digest) self.assertEqual(digest1, digest2)
self.assertNotEqual(msg1.digest, msg3.digest) self.assertNotEqual(digest1, digest3)
self.assertEqual(msg3.digest, msg4.digest) self.assertEqual(digest3, digest4)
def test_empty(self): def test_empty(self):
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
self.assertEqual(0, len(kcr.resolve('', '', []))) contents = io.StringIO(_CONFLICT_STATE_NONE)
self.assertEqual(0, len(kcr._messages)) with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mock_statefile().__enter__.return_value = contents
resolved, messages = kcr.resolve('', '', [])
self.assertEqual(0, len(resolved))
self.assertEqual(0, len(messages))
def test_one(self): def test_one(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id) key1 = SchleuderKey(_PRIVKEY_1.fingerprint.replace(' ', ''), 'foo@example.org', str(_PRIVKEY_1.pubkey), sch1.id)
date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1) sub1 = SchleuderSubscriber(3, 'foo@example.org', key1, sch1.id, date1)
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve('', '', [sub1]) resolved, messages = kcr.resolve('', '', [sub1])
self.assertEqual(1, len(resolved)) self.assertEqual(1, len(resolved))
self.assertEqual(sub1, resolved[0]) self.assertEqual(sub1, resolved[0])
self.assertEqual(0, len(kcr._messages)) self.assertEqual(0, len(messages))
def test_same_keys_conflict(self): def test_same_keys_conflict(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -137,16 +122,19 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve( contents = io.StringIO(_CONFLICT_STATE_NONE)
target='test@schleuder.example.org', with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mail_from='test-owner@schleuder.example.org', mock_statefile().__enter__.return_value = contents
subscriptions=[sub1, sub2]) resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(resolved)) self.assertEqual(1, len(resolved))
self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint) self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
# Same keys, no conflict message # Same keys, no conflict message
self.assertEqual(0, len(kcr._messages)) self.assertEqual(0, len(messages))
def test_different_keys_conflict(self): def test_different_keys_conflict(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -160,17 +148,20 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve( contents = io.StringIO(_CONFLICT_STATE_NONE)
target='test@schleuder.example.org', with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mail_from='test-owner@schleuder.example.org', mock_statefile().__enter__.return_value = contents
subscriptions=[sub1, sub2]) resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(resolved)) self.assertEqual(1, len(resolved))
self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint) self.assertEqual('135AFA0FB3FF584828911208B7913308392972A4', resolved[0].key.fingerprint)
# Different keys should trigger a conflict message # Different keys should trigger a conflict message
self.assertEqual(1, len(kcr._messages)) self.assertEqual(1, len(messages))
msg = kcr._messages[0].mime msg = messages[0].mime
pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True)) pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))
# Verify that the message is encrypted with both keys # Verify that the message is encrypted with both keys
dec1 = _PRIVKEY_1.decrypt(pgp) dec1 = _PRIVKEY_1.decrypt(pgp)
@ -193,28 +184,35 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', None, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', None, sch2.id, date2)
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve( contents = io.StringIO(_CONFLICT_STATE_NONE)
target='test@schleuder.example.org', with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mail_from='test-owner@schleuder.example.org', mock_statefile().__enter__.return_value = contents
subscriptions=[sub1, sub2]) resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(resolved)) self.assertEqual(1, len(resolved))
self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint) self.assertEqual('2FBBC0DF97FDBF1E4B704EEDE39EF4FAC420BEB6', resolved[0].key.fingerprint)
# Null key should not trigger a confict # Null key should not trigger a confict
self.assertEqual(0, len(kcr._messages)) self.assertEqual(0, len(messages))
def test_conflict_only_nullkeys(self): def test_conflict_only_nullkeys(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc()) date1 = datetime(2022, 4, 15, 5, 23, 42, 0, tzinfo=tzutc())
sub1 = SchleuderSubscriber(3, 'foo@example.org', None, sch1.id, date1) sub1 = SchleuderSubscriber(3, 'foo@example.org', None, sch1.id, date1)
kcr = KeyConflictResolution(None, 3600, '/tmp/state.json', _TEMPLATE) kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve( contents = io.StringIO(_CONFLICT_STATE_NONE)
target='test@schleuder.example.org', with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mail_from='test-owner@schleuder.example.org', mock_statefile().__enter__.return_value = contents
subscriptions=[sub1]) resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1])
self.assertEqual(0, len(resolved)) self.assertEqual(0, len(resolved))
self.assertEqual(0, len(messages))
def test_send_messages_nostate(self): def test_send_messages_nostate(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -229,31 +227,31 @@ class TestKeyConflictResolution(unittest.TestCase):
sub3 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub3 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
sub4 = SchleuderSubscriber(8, 'bar@example.org', key2, sch2.id, date1) sub4 = SchleuderSubscriber(8, 'bar@example.org', key2, sch2.id, date1)
mock_smtp = MagicMock() kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
mock_smtp.__enter__.return_value = mock_smtp
kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2, sub3, sub4])
self.assertEqual(2, len(kcr._messages))
msgs = kcr._messages
now = datetime.utcnow().timestamp()
contents = io.StringIO(_CONFLICT_STATE_NONE) contents = io.StringIO(_CONFLICT_STATE_NONE)
with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile: with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
mock_statefile().__enter__.return_value = contents mock_statefile().__enter__.return_value = contents
kcr.send_messages() resolved, msgs = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2, sub3, sub4])
self.assertEqual(2, len(msgs))
now = datetime.utcnow().timestamp()
mock_statefile.assert_called_with('/tmp/state.json', 'a+') mock_statefile.assert_called_with('/tmp/state.json', 'a+')
mock_smtp.__enter__.assert_called_once()
self.assertEqual(2, mock_smtp.send_message.call_count)
contents.seek(0) contents.seek(0)
state = json.loads(contents.read()) state = json.loads(contents.read())
self.assertEqual(2, len(state)) self.assertEqual(2, len(state))
self.assertIn(msgs[0].digest, state) self.assertIn(msgs[0].mime['X-MultiSchleuder-Digest'], state)
self.assertIn(msgs[1].digest, state) self.assertIn(msgs[1].mime['X-MultiSchleuder-Digest'], state)
self.assertLess(now - state[msgs[0].digest], 60) self.assertLess(now - state[msgs[0].mime['X-MultiSchleuder-Digest']], 60)
self.assertLess(now - state[msgs[1].digest], 60) self.assertLess(now - state[msgs[0].mime['X-MultiSchleuder-Digest']], 60)
# Todo: move this over to test_multilist
# mock_smtp = MagicMock()
# mock_smtp.__enter__.return_value = mock_smtp
# mock_smtp.__enter__.assert_called_once()
# self.assertEqual(2, mock_smtp.send_message.call_count)
def test_send_messages_stalestate(self): def test_send_messages_stalestate(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -266,29 +264,24 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
mock_smtp = MagicMock() kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
mock_smtp.__enter__.return_value = mock_smtp
kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(kcr._messages))
msg = kcr._messages[0]
now = datetime.utcnow().timestamp()
contents = io.StringIO(_CONFLICT_STATE_STALE) contents = io.StringIO(_CONFLICT_STATE_STALE)
with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile: with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile:
mock_statefile().__enter__.return_value = contents mock_statefile().__enter__.return_value = contents
kcr.send_messages() resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(messages))
msg = messages[0]
now = datetime.utcnow().timestamp()
mock_statefile.assert_called_with('/tmp/state.json', 'a+') mock_statefile.assert_called_with('/tmp/state.json', 'a+')
mock_smtp.__enter__.assert_called_once()
self.assertEqual(1, mock_smtp.send_message.call_count)
contents.seek(0) contents.seek(0)
state = json.loads(contents.read()) state = json.loads(contents.read())
self.assertEqual(1, len(state)) self.assertEqual(1, len(state))
self.assertIn(msg.digest, state) self.assertIn(msg.mime['X-MultiSchleuder-Digest'], state)
self.assertLess(now - state[msg.digest], 60) self.assertLess(now - state[msg.mime['X-MultiSchleuder-Digest']], 60)
def test_send_messages_recentstate(self): def test_send_messages_recentstate(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -301,32 +294,25 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
mock_smtp = MagicMock() kcr = KeyConflictResolution(86400, '/tmp/state.json', _TEMPLATE)
mock_smtp.__enter__.return_value = mock_smtp
kcr = KeyConflictResolution(mock_smtp, 86400, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(kcr._messages))
msg = kcr._messages[0]
now = datetime.utcnow().timestamp()
contents = io.StringIO(_CONFLICT_STATE_RECENT) contents = io.StringIO(_CONFLICT_STATE_RECENT)
with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_RECENT)) as mock_statefile: with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_RECENT)) as mock_statefile:
mock_statefile().__enter__.return_value = contents mock_statefile().__enter__.return_value = contents
kcr.send_messages() resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(0, len(messages))
now = datetime.utcnow().timestamp()
mock_statefile.assert_called_with('/tmp/state.json', 'a+') mock_statefile.assert_called_with('/tmp/state.json', 'a+')
# No message should be sent
mock_smtp.__enter__.assert_not_called()
mock_smtp.send_message.assert_not_called()
# Statefile should not have been updated # Statefile should not have been updated
contents.seek(0) contents.seek(0)
state = json.loads(contents.read()) state = json.loads(contents.read())
self.assertEqual(1, len(state)) self.assertEqual(1, len(state))
self.assertIn(msg.digest, state) for then in state.values():
self.assertLess(now - state[msg.digest], 86460) self.assertLess(now - then, 86460)
self.assertGreater(now - state[msg.digest], 60) self.assertGreater(now - then, 60)
def test_send_messages_dryrun(self): def test_send_messages_dryrun(self):
sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92') sch1 = SchleuderList(42, 'test-north@schleuder.example.org', '474777DA74528A7021184C8A0017324A6CFFBF92')
@ -339,25 +325,20 @@ class TestKeyConflictResolution(unittest.TestCase):
date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc()) date2 = datetime(2022, 4, 13, 5, 23, 42, 0, tzinfo=tzutc())
sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2) sub2 = SchleuderSubscriber(7, 'foo@example.org', key2, sch2.id, date2)
mock_smtp = MagicMock() kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
mock_smtp.__enter__.return_value = mock_smtp kcr.dry_run()
kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE)
resolved = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(kcr._messages))
msg = kcr._messages[0]
now = datetime.utcnow().timestamp()
contents = io.StringIO(_CONFLICT_STATE_STALE) contents = io.StringIO(_CONFLICT_STATE_STALE)
with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile: with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_STALE)) as mock_statefile:
mock_statefile().__enter__.return_value = contents mock_statefile().__enter__.return_value = contents
kcr.send_messages(dry_run=True) resolved, messages = kcr.resolve(
target='test@schleuder.example.org',
mail_from='test-owner@schleuder.example.org',
subscriptions=[sub1, sub2])
self.assertEqual(1, len(messages))
now = datetime.utcnow().timestamp()
mock_statefile.assert_called_with('/tmp/state.json', 'a+') mock_statefile.assert_called_with('/tmp/state.json', 'a+')
mock_statefile().write.assert_not_called() mock_statefile().write.assert_not_called()
mock_smtp.__enter__.assert_not_called()
mock_smtp.send_message.assert_not_called()
contents.seek(0) contents.seek(0)
self.assertEqual(_CONFLICT_STATE_STALE, contents.read()) self.assertEqual(_CONFLICT_STATE_STALE, contents.read())
@ -376,15 +357,16 @@ class TestKeyConflictResolution(unittest.TestCase):
date3 = datetime(2022, 4, 11, 5, 23, 42, 0, tzinfo=tzutc()) date3 = datetime(2022, 4, 11, 5, 23, 42, 0, tzinfo=tzutc())
sub3 = SchleuderSubscriber(7, 'foo@example.org', None, sch3.id, date3) sub3 = SchleuderSubscriber(7, 'foo@example.org', None, sch3.id, date3)
mock_smtp = MagicMock() kcr = KeyConflictResolution(3600, '/tmp/state.json', _TEMPLATE)
mock_smtp.__enter__.return_value = mock_smtp contents = io.StringIO(_CONFLICT_STATE_NONE)
kcr = KeyConflictResolution(mock_smtp, 3600, '/tmp/state.json', _TEMPLATE) with patch('builtins.open', mock_open(read_data=_CONFLICT_STATE_NONE)) as mock_statefile:
resolved = kcr.resolve( mock_statefile().__enter__.return_value = contents
target='test@schleuder.example.org', resolved, messages = kcr.resolve(
mail_from='test-owner@schleuder.example.org', target='test@schleuder.example.org',
subscriptions=[sub1, sub2, sub3]) mail_from='test-owner@schleuder.example.org',
self.assertEqual(1, len(kcr._messages)) subscriptions=[sub1, sub2, sub3])
msg = kcr._messages[0].mime self.assertEqual(1, len(messages))
msg = messages[0].mime
pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True)) pgp = pgpy.PGPMessage.from_blob(msg.get_payload()[1].get_payload(decode=True))
# Verify that the message is encrypted with both keys # Verify that the message is encrypted with both keys
self.assertEqual(2, len(pgp.encrypters)) self.assertEqual(2, len(pgp.encrypters))

View file

@ -1,5 +1,5 @@
from typing import Dict, List from typing import Dict, List, Tuple
import unittest import unittest
from datetime import datetime from datetime import datetime
@ -8,18 +8,19 @@ from unittest.mock import MagicMock
from dateutil.tz import tzutc from dateutil.tz import tzutc
from multischleuder.processor import MultiList from multischleuder.processor import MultiList
from multischleuder.reporting import ConflictMessage
from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber from multischleuder.types import SchleuderKey, SchleuderList, SchleuderSubscriber
def _resolve(target: str, def _resolve(target: str,
mail_from: str, mail_from: str,
subscriptions: List[SchleuderSubscriber]) -> List[SchleuderSubscriber]: subscriptions: List[SchleuderSubscriber]) -> Tuple[List[SchleuderSubscriber], List[ConflictMessage]]:
# Mock conflict resolution that does not send or prepare messages # Mock conflict resolution that does not send or prepare messages
subs: Dict[str, List[SchleuderSubscriber]] = {} subs: Dict[str, List[SchleuderSubscriber]] = {}
for s in subscriptions: for s in subscriptions:
if s.key is not None: if s.key is not None:
subs.setdefault(s.email, []).append(s) subs.setdefault(s.email, []).append(s)
return [min(s, key=lambda x: x.created_at) for s in subs.values()] return [min(s, key=lambda x: x.created_at)for s in subs.values()], []
def _list_lists(): def _list_lists():
@ -152,13 +153,17 @@ class TestMultiList(unittest.TestCase):
'test-west@schleuder.example.org' 'test-west@schleuder.example.org'
] ]
api = self._api_mock() api = self._api_mock()
smtp = MagicMock()
ml = MultiList(sources=sources, ml = MultiList(sources=sources,
target='test-global@schleuder.example.org', target='test-global@schleuder.example.org',
unmanaged=['admin@example.org'], unmanaged=['admin@example.org'],
banned=['aspammer@example.org', 'anotherspammer@example.org'], banned=['aspammer@example.org', 'anotherspammer@example.org'],
mail_from='test-global-owner@schleuder.example.org', mail_from='test-global-owner@schleuder.example.org',
api=api, api=api,
kcr=self._kcr_mock()) kcr=self._kcr_mock(),
smtp=smtp,
send_admin_reports=True,
send_conflict_messages=True)
ml.process() ml.process()
# Key uploads # Key uploads
@ -211,3 +216,5 @@ class TestMultiList(unittest.TestCase):
self.assertEqual(2, c[1].id) self.assertEqual(2, c[1].id)
self.assertEqual('aspammer@example.org', c[0].email) self.assertEqual('aspammer@example.org', c[0].email)
self.assertEqual('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', c[0].fingerprint) self.assertEqual('8258FAF8B161B3DD8F784874F73E2DDF045AE2D6', c[0].fingerprint)
# Todo: check message queue

View file

@ -19,10 +19,12 @@ lists:
- test-south@schleuder.example.org - test-south@schleuder.example.org
- test-west@schleuder.example.org - test-west@schleuder.example.org
from: test-global-owner@schleuder.example.org from: test-global-owner@schleuder.example.org
send_admin_reports: no
- target: test2-global@schleuder.example.org - target: test2-global@schleuder.example.org
unmanaged: unmanaged:
- admin@example.org - admin@example.org
- admin2@example.org
banned: banned:
- aspammer@example.org - aspammer@example.org
- anotherspammer@example.org - anotherspammer@example.org

View file

@ -84,3 +84,6 @@ sleep 5 # to get different subscription dates
schleuder-cli subscriptions new test-west@schleuder.example.org andy.example@example.org /tmp/andy.example@example.org.1.asc schleuder-cli subscriptions new test-west@schleuder.example.org andy.example@example.org /tmp/andy.example@example.org.1.asc
# should not be subscribed # should not be subscribed
subscribe test-west@schleuder.example.org amy.example@example.org # should be subscribed - conflict but same key subscribe test-west@schleuder.example.org amy.example@example.org # should be subscribed - conflict but same key
schleuder-cli subscriptions new test2-global@schleuder.example.org arno.example@example.org
# should be unsubscribed - no key

View file

@ -84,24 +84,47 @@ if len(keysdiff) > 0:
# Test mbox # Test mbox
mbox = mailbox.mbox('/var/spool/mail/root') mbox = mailbox.mbox('/var/spool/mail/root')
if len(mbox) != 1: if len(mbox) != 2:
print(f'Expected 1 message in mbox, got {len(mbox)}') print(f'Expected 2 messages in mbox, got {len(mbox)}')
exit(1) exit(1)
_, msg = mbox.popitem() _, msg1 = mbox.popitem()
_, msg2 = mbox.popitem()
mbox.close() mbox.close()
digest = msg['X-MultiSchleuder-Digest'].strip() if 'X-MultiSchleuder-Digest' not in msg1:
msg1, msg2 = msg2, msg1
if msg['From'] != 'test-global-owner@schleuder.example.org':
print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg["From"]}') if 'X-MultiSchleuder-Digest' not in msg1:
print(f'Key conflict message should have a X-MultiSchleuder-Digest header, missing')
exit(1) exit(1)
if msg['To'] != 'andy.example@example.org': digest = msg1['X-MultiSchleuder-Digest'].strip()
print(f'Expected "To: andy.example@example.org", got {msg["To"]}') if msg1['From'] != 'test-global-owner@schleuder.example.org':
print(f'Expected "From: test-global-owner@schleuder.example.org", got {msg1["From"]}')
exit(1) exit(1)
if msg['Auto-Submitted'] != 'auto-generated': if msg1['To'] != 'andy.example@example.org':
print(f'Expected "Auto-Submitted: auto-generated", got {msg["Auto-Submitted"]}') print(f'Expected "To: andy.example@example.org", got {msg1["To"]}')
exit(1) exit(1)
if msg['Precedence'] != 'list': if msg1['Auto-Submitted'] != 'auto-generated':
print(f'Expected "Precedence: list", got {msg["Precedence"]}') print(f'Expected "Auto-Submitted: auto-generated", got {msg1["Auto-Submitted"]}')
exit(1)
if msg1['Precedence'] != 'list':
print(f'Expected "Precedence: list", got {msg1["Precedence"]}')
exit(1)
if 'X-MultiSchleuder-Digest' in msg2:
print(f'Admin report message should not have a X-MultiSchleuder-Digest header, got {msg2["X-MultiSchleuder-Digest"]}')
exit(1)
if msg2['From'] != 'test2-global-owner@schleuder.example.org':
print(f'Expected "From: test2-global-owner@schleuder.example.org", got {msg2["From"]}')
exit(1)
if msg2['To'] != 'admin2@example.org':
print(f'Expected "To: admin2@example.org", got {msg2["To"]}')
exit(1)
if msg2['Auto-Submitted'] != 'auto-generated':
print(f'Expected "Auto-Submitted: auto-generated", got {msg2["Auto-Submitted"]}')
exit(1)
if msg2['Precedence'] != 'list':
print(f'Expected "Precedence: list", got {msg2["Precedence"]}')
exit(1) exit(1)