s3lph/matemat
s3lph/matemat
1
0
Fork 2
Code Issues Pull requests 2 Projects Releases Packages 2 Wiki Activity Actions

Update docker images

Browse source
This commit is contained in:
s3lph 2020-02-03 23:39:35 +01:00
parent 50b088b0ec
commit 18c720fdab
7 changed files with 26 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.gitlab-ci.yml
View file

@ -1,5 +1,5 @@
---
image: s3lph/matemat-ci:20181107-02
image: s3lph/matemat-ci:20200203-01
stages:
- test
@ -17,9 +17,9 @@ test:
stage: test
script:
- pip3 install -e .
- sudo -u matemat python3.6 -m coverage run --rcfile=setup.cfg -m unittest discover matemat
- sudo -u matemat python3.6 -m coverage combine
- sudo -u matemat python3.6 -m coverage report --rcfile=setup.cfg
- sudo -u matemat python3.7 -m coverage run --rcfile=setup.cfg -m unittest discover matemat
- sudo -u matemat python3.7 -m coverage combine
- sudo -u matemat python3.7 -m coverage report --rcfile=setup.cfg
codestyle:
stage: test

73
matemat/webserver/test/abstract_httpd_test.py
View file

@ -1,73 +0,0 @@
import threading
from http.client import HTTPResponse
from typing import Any, Callable, Dict, Tuple, Union
import unittest.mock
import urllib.request
from tempfile import TemporaryDirectory
from abc import ABC
from datetime import datetime
import logging
import jinja2
import bottle
class MockServer:
"""
A mock implementation of http.server.HTTPServer. Only used for matemat-specific storage.
"""
def __init__(self, webroot: str = '/var/matemat/webroot') -> None:
# Session timeout and variables for all sessions
self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict()
# Webroot for statically served content
self.webroot: str = webroot
# Variables to pass to pagelets
self.pagelet_variables: Dict[str, str] = dict()
# Static response headers
self.static_headers: Dict[str, str] = {
'X-Static-Testing-Header': 'helloworld!'
}
# Jinja environment with a single, static template
self.jinja_env = jinja2.Environment(
loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'})
)
# Set up logger
self.logger: logging.Logger = logging.getLogger('matemat unit test')
self.logger.setLevel(logging.DEBUG)
# Disable logging
self.logger.addHandler(logging.NullHandler())
# Initalize a log handler to stderr and set the log format
# sh: logging.StreamHandler = logging.StreamHandler()
# sh.setFormatter(logging.Formatter('%(asctime)s %(name)s [%(levelname)s]: %(message)s'))
# self.logger.addHandler(sh)
bottle.run(host='::1', port='8888')
class AbstractHttpdTest(ABC, unittest.TestCase):
"""
An abstract test case that can be inherited by test case classes that want to test part of the webserver's core
functionality.
Usage (subclass test method):
self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n')
handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
packet = self.client_sock.get_response()
"""
def setUp(self) -> None:
self.tempdir: TemporaryDirectory = TemporaryDirectory(prefix='matemat.', dir='/tmp/')
self.server = threading.Thread(target=bottle.run, kwargs={'host': '::1', 'port': 8888, 'debug': True})
self.server.start()
def tearDown(self):
self.server._stop()
self.tempdir.cleanup()
@staticmethod
def request(query) -> HTTPResponse:
return urllib.request.urlopen(f'http://[::1]:8888{query}')

20
matemat/webserver/test/test_config.py
View file

@ -8,7 +8,7 @@ from io import StringIO
import logging
import sys
from matemat.webserver.config import parse_config_file
from matemat.webserver.config import parse_config_file, get_config
_EMPTY_CONFIG = ''
@ -112,7 +112,8 @@ class TestConfig(TestCase):
# Mock the open() function to return an empty config file example
with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)):
# The filename is only a placeholder, file content is determined by mocking open
config = parse_config_file('test')
parse_config_file('test')
config = get_config()
# Make sure all mandatory values are present
self.assertIn('listen', config)
self.assertIn('port', config)
@ -141,7 +142,8 @@ class TestConfig(TestCase):
# Mock the open() function to return a full config file example
with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)):
# The filename is only a placeholder, file content is determined by mocking open
config = parse_config_file('test')
parse_config_file('test')
config = get_config()
# Make sure all mandatory values are present
self.assertIn('listen', config)
self.assertIn('port', config)
@ -186,7 +188,8 @@ class TestConfig(TestCase):
# second call
with patch('builtins.open', return_value=IterOpenMock([_FULL_CONFIG, _PARTIAL_CONFIG])):
# These filenames are only placeholders, file content is determined by mocking open
config = parse_config_file(['full', 'partial'])
parse_config_file(['full', 'partial'])
config = get_config()
# Make sure all mandatory values are present
self.assertIn('listen', config)
self.assertIn('port', config)
@ -216,7 +219,8 @@ class TestConfig(TestCase):
# Mock the open() function to return a config file example with disabled logging
with patch('builtins.open', return_value=StringIO(_LOG_NONE_CONFIG)):
# The filename is only a placeholder, file content is determined by mocking open
config = parse_config_file('test')
parse_config_file('test')
config = get_config()
# Make sure the returned log handler is a null handler
self.assertIsInstance(config['log_handler'], logging.NullHandler)
@ -227,7 +231,8 @@ class TestConfig(TestCase):
# Mock the open() function to return a config file example with stdout logging
with patch('builtins.open', return_value=StringIO(_LOG_STDOUT_CONFIG)):
# The filename is only a placeholder, file content is determined by mocking open
config = parse_config_file('test')
parse_config_file('test')
config = get_config()
# Make sure the returned log handler is a stdout handler
self.assertIsInstance(config['log_handler'], logging.StreamHandler)
self.assertEqual(sys.stdout, config['log_handler'].stream)
@ -239,7 +244,8 @@ class TestConfig(TestCase):
# Mock the open() function to return a config file example with stdout logging
with patch('builtins.open', return_value=StringIO(_LOG_STDERR_CONFIG)):
# The filename is only a placeholder, file content is determined by mocking open
config = parse_config_file('test')
parse_config_file('test')
config = get_config()
# Make sure the returned log handler is a stdout handler
self.assertIsInstance(config['log_handler'], logging.StreamHandler)
self.assertEqual(sys.stderr, config['log_handler'].stream)

65
matemat/webserver/test/test_pagelet_cron.py
View file

@ -1,65 +0,0 @@
from typing import Dict
import unittest
import logging
from threading import Lock, Thread, Timer
from time import sleep
import jinja2
from matemat.webserver.cron import cron
lock: Lock = Lock()
cron1called: int = 0
cron2called: int = 0
@cron(seconds=4)
def cron1(config: Dict[str, str],
jinja_env: jinja2.Environment,
logger: logging.Logger) -> None:
global cron1called
with lock:
cron1called += 1
@cron(seconds=3)
def cron2(config: Dict[str, str],
jinja_env: jinja2.Environment,
logger: logging.Logger) -> None:
global cron2called
with lock:
cron2called += 1
class TestPageletCron(unittest.TestCase):
def setUp(self):
self.srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {},
logging.NOTSET, logging.NullHandler())
self.srv_port = int(self.srv._httpd.socket.getsockname()[1])
self.timer = Timer(10.0, self.srv._httpd.shutdown)
self.timer.start()
def tearDown(self):
self.timer.cancel()
if self.srv is not None:
self.srv._httpd.socket.close()
def test_cron(self):
"""
Test that the cron functions are called properly.
"""
thread = Thread(target=self.srv.start)
thread.start()
sleep(12)
self.srv._httpd.shutdown()
with lock:
self.assertEqual(2, cron1called)
self.assertEqual(3, cron2called)
# Make sure the cron threads were stopped
sleep(5)
with lock:
self.assertEqual(2, cron1called)
self.assertEqual(3, cron2called)

148
matemat/webserver/test/test_session.py
View file

@ -1,148 +0,0 @@
from datetime import datetime, timedelta
from time import sleep
from bottle import route, run
from matemat.webserver import session
from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest
@route('/just/testing/sessions')
def session_test_pagelet():
s = session.start()
session.put(s, 'test', 'hello, world!')
return f'{s}'
class TestSession(AbstractHttpdTest):
"""
Test session handling of the Matemat webserver.
"""
def test_create_new_session(self):
# Reference date to make sure the session expiry lies in the future
refdate: datetime = datetime.utcnow() + timedelta(seconds=3500)
# Send a mock GET request for '/just/testing/sessions'
packet = self.request('/just/testing/sessions')
# Make sure the request was served by the test pagelet
self.assertEqual(200, packet.code)
session_id: str = packet.read().decode()
# Make sure a cookie was set - assuming that only one was set
self.assertIn('Set-Cookie', packet.headers)
# Split into the cookie itself
cookie, expiry = packet.headers['Set-Cookie'].split(';')
cookie: str = cookie.strip()
expiry: str = expiry.strip()
# Make sure the 'matemat_session_id' cookie was set to the session ID string
self.assertEqual(f'matemat_session_id={session_id}', cookie)
# Make sure the session expires in about one hour
self.assertTrue(expiry.startswith('expires='))
_, expdatestr = expiry.split('=', 1)
expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
self.assertTrue(expdate > refdate)
def test_resume_session(self):
# Test session expiry date
refdate: datetime = datetime.utcnow() + timedelta(hours=1)
# Session ID for testing
session_id: str = 'testsessionid'
# Insert test session
self.server.session_vars[session_id] = refdate, {'test': 'bar'}
sleep(2)
# Send a mock GET request for '/just/testing/sessions' with a matemat session cookie
self.client_sock.set_request(
f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
# Trigger request handling
handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
# Fetch the parsed response
packet = self.client_sock.get_response()
# Make sure a full HTTP response was parsed
self.assertEqual('done', packet.parse_phase)
# Make sure the request was served by the test pagelet
self.assertEqual('session_test_pagelet', packet.pagelet)
self.assertEqual(200, packet.statuscode)
response_session_id: str = list(handler.server.session_vars.keys())[0]
# Make sure a cookie was set - assuming that only one was set
self.assertIn('Set-Cookie', packet.headers)
# Split into the cookie itself
cookie, expiry = packet.headers['Set-Cookie'].split(';')
cookie: str = cookie.strip()
expiry: str = expiry.strip()
# Make sure the 'matemat_session_id' cookie was set to the session ID string
self.assertEqual(f'matemat_session_id={response_session_id}', cookie)
# Make sure the session ID matches the one we sent along
self.assertEqual(session_id, response_session_id)
# Make sure the session timeout was postponed
self.assertTrue(expiry.startswith('expires='))
_, expdatestr = expiry.split('=', 1)
expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
self.assertTrue(expdate > refdate)
# Make sure the session exists on the server
self.assertIn('test', handler.session_vars)
self.assertEqual('hello, world!', handler.session_vars['test'])
def test_unknown_session_id(self):
# Unknown session ID
session_id: str = 'theserverdoesnotknowthisid'
refdate: datetime = datetime.utcnow() + timedelta(seconds=3500)
# Send a mock GET request for '/just/testing/sessions' with a session cookie not known to the server
self.client_sock.set_request(
f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
# Trigger request handling
handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
# Fetch the parsed response
packet = self.client_sock.get_response()
# Make sure a full HTTP response was parsed
self.assertEqual('done', packet.parse_phase)
# Make sure the request was served by the test pagelet
self.assertEqual('session_test_pagelet', packet.pagelet)
self.assertEqual(200, packet.statuscode)
server_session_id: str = list(handler.server.session_vars.keys())[0]
self.assertNotEqual(session_id, server_session_id)
# Make sure a cookie was set - assuming that only one was set
self.assertIn('Set-Cookie', packet.headers)
# Split into the cookie itself
cookie, expiry = packet.headers['Set-Cookie'].split(';')
cookie: str = cookie.strip()
expiry: str = expiry.strip()
# Make sure the 'matemat_session_id' cookie was set to the session ID string
self.assertEqual(f'matemat_session_id={server_session_id}', cookie)
# Make sure the session expires in about one hour
self.assertTrue(expiry.startswith('expires='))
_, expdatestr = expiry.split('=', 1)
expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
self.assertTrue(expdate > refdate)
# Make sure the session exists on the server
self.assertIn('test', handler.session_vars)
self.assertEqual('hello, world!', handler.session_vars['test'])
def test_session_expired(self):
# Test session expiry date
refdate: datetime = datetime.utcnow() - timedelta(hours=1)
# Session ID for testing
session_id: str = 'testsessionid'
# Insert test session
self.server.session_vars[session_id] = refdate, {'test': 'bar'}
# Send a mock GET request for '/just/testing/sessions' with a matemat session cookie
self.client_sock.set_request(
f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
# Trigger request handling
handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
# Fetch the parsed response
packet = self.client_sock.get_response()
# Make sure a full HTTP response was parsed
self.assertEqual('done', packet.parse_phase)
# Make sure the server redirects to /
self.assertEqual(302, packet.statuscode)
self.assertIn('Location', packet.headers)
self.assertEqual('/', packet.headers['Location'])
# Make sure the session was terminated
self.assertNotIn(session_id, self.server.session_vars)

4
package/docker/Dockerfile
View file

@ -3,11 +3,15 @@ FROM python:3.7-alpine
ADD . /
RUN mkdir -p /var/matemat/db /var/matemat/upload \
&& chown 1000:0 -R /var/matemat/db /var/matemat/upload \
&& chmod g=u -R /var/matemat/db /var/matemat/upload \
&& apk --update add libmagic zlib jpeg zlib-dev jpeg-dev build-base \
&& pip3 install -e . \
&& apk del zlib-dev jpeg-dev build-base \
&& rm -rf /var/cache/apk /root/.cache/pip \
&& rm -rf /package
USER 1000
EXPOSE 80/tcp
CMD [ "/run.sh" ]

12
testing/Dockerfile
View file

@ -1,15 +1,13 @@
# There is no buster image yet and stretch doesn't have a docker package. So let's just "upgrade" the image to buster.
FROM python:3.6-stretch
FROM python:3.7-buster
RUN sed -re 's/stretch/buster/g' -i /etc/apt/sources.list \
&& useradd -d /home/matemat -m matemat \
RUN useradd -d /home/matemat -m matemat \
&& mkdir -p /var/matemat/db /var/matemat/upload \
&& chown matemat:matemat -R /var/matemat/db \
&& chown matemat:matemat -R /var/matemat/upload \
&& chown matemat:0 -R /var/matemat/db /var/matemat/upload \
&& chmod g=u -R /var/matemat/db /var/matemat/upload \
&& apt-get update -qy \
&& apt-get install -y --no-install-recommends file sudo openssh-client git docker.io build-essential lintian rsync \
&& python3.6 -m pip install coverage wheel pycodestyle mypy \
&& python3.7 -m pip install coverage wheel pycodestyle mypy \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /home/matemat