From 18c720fdab1145322e6305db08bd87e064513a79 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 3 Feb 2020 23:39:35 +0100 Subject: [PATCH] Update docker images --- .gitlab-ci.yml | 8 +- matemat/webserver/test/abstract_httpd_test.py | 73 --------- matemat/webserver/test/test_config.py | 20 ++- matemat/webserver/test/test_pagelet_cron.py | 65 -------- matemat/webserver/test/test_session.py | 148 ------------------ package/docker/Dockerfile | 4 + testing/Dockerfile | 12 +- 7 files changed, 26 insertions(+), 304 deletions(-) delete mode 100644 matemat/webserver/test/abstract_httpd_test.py delete mode 100644 matemat/webserver/test/test_pagelet_cron.py delete mode 100644 matemat/webserver/test/test_session.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73cbd52..933cf08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ --- -image: s3lph/matemat-ci:20181107-02 +image: s3lph/matemat-ci:20200203-01 stages: - test @@ -17,9 +17,9 @@ test: stage: test script: - pip3 install -e . - - sudo -u matemat python3.6 -m coverage run --rcfile=setup.cfg -m unittest discover matemat - - sudo -u matemat python3.6 -m coverage combine - - sudo -u matemat python3.6 -m coverage report --rcfile=setup.cfg + - sudo -u matemat python3.7 -m coverage run --rcfile=setup.cfg -m unittest discover matemat + - sudo -u matemat python3.7 -m coverage combine + - sudo -u matemat python3.7 -m coverage report --rcfile=setup.cfg codestyle: stage: test diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py deleted file mode 100644 index 373f39d..0000000 --- a/matemat/webserver/test/abstract_httpd_test.py +++ /dev/null @@ -1,73 +0,0 @@ -import threading -from http.client import HTTPResponse -from typing import Any, Callable, Dict, Tuple, Union - -import unittest.mock -import urllib.request -from tempfile import TemporaryDirectory - -from abc import ABC -from datetime import datetime - -import logging -import jinja2 - -import bottle - - -class MockServer: - """ - A mock implementation of http.server.HTTPServer. Only used for matemat-specific storage. - """ - - def __init__(self, webroot: str = '/var/matemat/webroot') -> None: - # Session timeout and variables for all sessions - self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() - # Webroot for statically served content - self.webroot: str = webroot - # Variables to pass to pagelets - self.pagelet_variables: Dict[str, str] = dict() - # Static response headers - self.static_headers: Dict[str, str] = { - 'X-Static-Testing-Header': 'helloworld!' - } - # Jinja environment with a single, static template - self.jinja_env = jinja2.Environment( - loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'}) - ) - # Set up logger - self.logger: logging.Logger = logging.getLogger('matemat unit test') - self.logger.setLevel(logging.DEBUG) - # Disable logging - self.logger.addHandler(logging.NullHandler()) - # Initalize a log handler to stderr and set the log format - # sh: logging.StreamHandler = logging.StreamHandler() - # sh.setFormatter(logging.Formatter('%(asctime)s %(name)s [%(levelname)s]: %(message)s')) - # self.logger.addHandler(sh) - bottle.run(host='::1', port='8888') - - -class AbstractHttpdTest(ABC, unittest.TestCase): - """ - An abstract test case that can be inherited by test case classes that want to test part of the webserver's core - functionality. - - Usage (subclass test method): - - self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n') - handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) - packet = self.client_sock.get_response() - """ - - def setUp(self) -> None: - self.tempdir: TemporaryDirectory = TemporaryDirectory(prefix='matemat.', dir='/tmp/') - self.server = threading.Thread(target=bottle.run, kwargs={'host': '::1', 'port': 8888, 'debug': True}) - self.server.start() - - def tearDown(self): - self.server._stop() - self.tempdir.cleanup() - - @staticmethod - def request(query) -> HTTPResponse: - return urllib.request.urlopen(f'http://[::1]:8888{query}') diff --git a/matemat/webserver/test/test_config.py b/matemat/webserver/test/test_config.py index cdbf28b..ef8feda 100644 --- a/matemat/webserver/test/test_config.py +++ b/matemat/webserver/test/test_config.py @@ -8,7 +8,7 @@ from io import StringIO import logging import sys -from matemat.webserver.config import parse_config_file +from matemat.webserver.config import parse_config_file, get_config _EMPTY_CONFIG = '' @@ -112,7 +112,8 @@ class TestConfig(TestCase): # Mock the open() function to return an empty config file example with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)): # The filename is only a placeholder, file content is determined by mocking open - config = parse_config_file('test') + parse_config_file('test') + config = get_config() # Make sure all mandatory values are present self.assertIn('listen', config) self.assertIn('port', config) @@ -141,7 +142,8 @@ class TestConfig(TestCase): # Mock the open() function to return a full config file example with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)): # The filename is only a placeholder, file content is determined by mocking open - config = parse_config_file('test') + parse_config_file('test') + config = get_config() # Make sure all mandatory values are present self.assertIn('listen', config) self.assertIn('port', config) @@ -186,7 +188,8 @@ class TestConfig(TestCase): # second call with patch('builtins.open', return_value=IterOpenMock([_FULL_CONFIG, _PARTIAL_CONFIG])): # These filenames are only placeholders, file content is determined by mocking open - config = parse_config_file(['full', 'partial']) + parse_config_file(['full', 'partial']) + config = get_config() # Make sure all mandatory values are present self.assertIn('listen', config) self.assertIn('port', config) @@ -216,7 +219,8 @@ class TestConfig(TestCase): # Mock the open() function to return a config file example with disabled logging with patch('builtins.open', return_value=StringIO(_LOG_NONE_CONFIG)): # The filename is only a placeholder, file content is determined by mocking open - config = parse_config_file('test') + parse_config_file('test') + config = get_config() # Make sure the returned log handler is a null handler self.assertIsInstance(config['log_handler'], logging.NullHandler) @@ -227,7 +231,8 @@ class TestConfig(TestCase): # Mock the open() function to return a config file example with stdout logging with patch('builtins.open', return_value=StringIO(_LOG_STDOUT_CONFIG)): # The filename is only a placeholder, file content is determined by mocking open - config = parse_config_file('test') + parse_config_file('test') + config = get_config() # Make sure the returned log handler is a stdout handler self.assertIsInstance(config['log_handler'], logging.StreamHandler) self.assertEqual(sys.stdout, config['log_handler'].stream) @@ -239,7 +244,8 @@ class TestConfig(TestCase): # Mock the open() function to return a config file example with stdout logging with patch('builtins.open', return_value=StringIO(_LOG_STDERR_CONFIG)): # The filename is only a placeholder, file content is determined by mocking open - config = parse_config_file('test') + parse_config_file('test') + config = get_config() # Make sure the returned log handler is a stdout handler self.assertIsInstance(config['log_handler'], logging.StreamHandler) self.assertEqual(sys.stderr, config['log_handler'].stream) diff --git a/matemat/webserver/test/test_pagelet_cron.py b/matemat/webserver/test/test_pagelet_cron.py deleted file mode 100644 index 162f91b..0000000 --- a/matemat/webserver/test/test_pagelet_cron.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Dict - -import unittest - -import logging -from threading import Lock, Thread, Timer -from time import sleep -import jinja2 - -from matemat.webserver.cron import cron - -lock: Lock = Lock() - -cron1called: int = 0 -cron2called: int = 0 - - -@cron(seconds=4) -def cron1(config: Dict[str, str], - jinja_env: jinja2.Environment, - logger: logging.Logger) -> None: - global cron1called - with lock: - cron1called += 1 - - -@cron(seconds=3) -def cron2(config: Dict[str, str], - jinja_env: jinja2.Environment, - logger: logging.Logger) -> None: - global cron2called - with lock: - cron2called += 1 - - -class TestPageletCron(unittest.TestCase): - - def setUp(self): - self.srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, - logging.NOTSET, logging.NullHandler()) - self.srv_port = int(self.srv._httpd.socket.getsockname()[1]) - self.timer = Timer(10.0, self.srv._httpd.shutdown) - self.timer.start() - - def tearDown(self): - self.timer.cancel() - if self.srv is not None: - self.srv._httpd.socket.close() - - def test_cron(self): - """ - Test that the cron functions are called properly. - """ - thread = Thread(target=self.srv.start) - thread.start() - sleep(12) - self.srv._httpd.shutdown() - with lock: - self.assertEqual(2, cron1called) - self.assertEqual(3, cron2called) - # Make sure the cron threads were stopped - sleep(5) - with lock: - self.assertEqual(2, cron1called) - self.assertEqual(3, cron2called) diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py deleted file mode 100644 index 0ada390..0000000 --- a/matemat/webserver/test/test_session.py +++ /dev/null @@ -1,148 +0,0 @@ - -from datetime import datetime, timedelta -from time import sleep - -from bottle import route, run - -from matemat.webserver import session -from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest - - -@route('/just/testing/sessions') -def session_test_pagelet(): - s = session.start() - session.put(s, 'test', 'hello, world!') - return f'{s}' - - -class TestSession(AbstractHttpdTest): - - """ - Test session handling of the Matemat webserver. - """ - - def test_create_new_session(self): - # Reference date to make sure the session expiry lies in the future - refdate: datetime = datetime.utcnow() + timedelta(seconds=3500) - # Send a mock GET request for '/just/testing/sessions' - - packet = self.request('/just/testing/sessions') - # Make sure the request was served by the test pagelet - self.assertEqual(200, packet.code) - - session_id: str = packet.read().decode() - # Make sure a cookie was set - assuming that only one was set - self.assertIn('Set-Cookie', packet.headers) - # Split into the cookie itself - cookie, expiry = packet.headers['Set-Cookie'].split(';') - cookie: str = cookie.strip() - expiry: str = expiry.strip() - # Make sure the 'matemat_session_id' cookie was set to the session ID string - self.assertEqual(f'matemat_session_id={session_id}', cookie) - # Make sure the session expires in about one hour - self.assertTrue(expiry.startswith('expires=')) - _, expdatestr = expiry.split('=', 1) - expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') - self.assertTrue(expdate > refdate) - - def test_resume_session(self): - # Test session expiry date - refdate: datetime = datetime.utcnow() + timedelta(hours=1) - # Session ID for testing - session_id: str = 'testsessionid' - # Insert test session - self.server.session_vars[session_id] = refdate, {'test': 'bar'} - sleep(2) - - # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie - self.client_sock.set_request( - f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) - # Trigger request handling - handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) - # Fetch the parsed response - packet = self.client_sock.get_response() - # Make sure a full HTTP response was parsed - self.assertEqual('done', packet.parse_phase) - # Make sure the request was served by the test pagelet - self.assertEqual('session_test_pagelet', packet.pagelet) - self.assertEqual(200, packet.statuscode) - - response_session_id: str = list(handler.server.session_vars.keys())[0] - # Make sure a cookie was set - assuming that only one was set - self.assertIn('Set-Cookie', packet.headers) - # Split into the cookie itself - cookie, expiry = packet.headers['Set-Cookie'].split(';') - cookie: str = cookie.strip() - expiry: str = expiry.strip() - # Make sure the 'matemat_session_id' cookie was set to the session ID string - self.assertEqual(f'matemat_session_id={response_session_id}', cookie) - # Make sure the session ID matches the one we sent along - self.assertEqual(session_id, response_session_id) - # Make sure the session timeout was postponed - self.assertTrue(expiry.startswith('expires=')) - _, expdatestr = expiry.split('=', 1) - expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') - self.assertTrue(expdate > refdate) - # Make sure the session exists on the server - self.assertIn('test', handler.session_vars) - self.assertEqual('hello, world!', handler.session_vars['test']) - - def test_unknown_session_id(self): - # Unknown session ID - session_id: str = 'theserverdoesnotknowthisid' - refdate: datetime = datetime.utcnow() + timedelta(seconds=3500) - # Send a mock GET request for '/just/testing/sessions' with a session cookie not known to the server - self.client_sock.set_request( - f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) - # Trigger request handling - handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) - # Fetch the parsed response - packet = self.client_sock.get_response() - # Make sure a full HTTP response was parsed - self.assertEqual('done', packet.parse_phase) - # Make sure the request was served by the test pagelet - self.assertEqual('session_test_pagelet', packet.pagelet) - self.assertEqual(200, packet.statuscode) - - server_session_id: str = list(handler.server.session_vars.keys())[0] - self.assertNotEqual(session_id, server_session_id) - # Make sure a cookie was set - assuming that only one was set - self.assertIn('Set-Cookie', packet.headers) - # Split into the cookie itself - cookie, expiry = packet.headers['Set-Cookie'].split(';') - cookie: str = cookie.strip() - expiry: str = expiry.strip() - # Make sure the 'matemat_session_id' cookie was set to the session ID string - self.assertEqual(f'matemat_session_id={server_session_id}', cookie) - # Make sure the session expires in about one hour - self.assertTrue(expiry.startswith('expires=')) - _, expdatestr = expiry.split('=', 1) - expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') - self.assertTrue(expdate > refdate) - # Make sure the session exists on the server - self.assertIn('test', handler.session_vars) - self.assertEqual('hello, world!', handler.session_vars['test']) - - def test_session_expired(self): - # Test session expiry date - refdate: datetime = datetime.utcnow() - timedelta(hours=1) - # Session ID for testing - session_id: str = 'testsessionid' - # Insert test session - self.server.session_vars[session_id] = refdate, {'test': 'bar'} - - # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie - self.client_sock.set_request( - f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) - # Trigger request handling - handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) - # Fetch the parsed response - packet = self.client_sock.get_response() - # Make sure a full HTTP response was parsed - self.assertEqual('done', packet.parse_phase) - # Make sure the server redirects to / - self.assertEqual(302, packet.statuscode) - self.assertIn('Location', packet.headers) - self.assertEqual('/', packet.headers['Location']) - # Make sure the session was terminated - self.assertNotIn(session_id, self.server.session_vars) diff --git a/package/docker/Dockerfile b/package/docker/Dockerfile index d85e4f4..fb2166d 100644 --- a/package/docker/Dockerfile +++ b/package/docker/Dockerfile @@ -3,11 +3,15 @@ FROM python:3.7-alpine ADD . / RUN mkdir -p /var/matemat/db /var/matemat/upload \ + && chown 1000:0 -R /var/matemat/db /var/matemat/upload \ + && chmod g=u -R /var/matemat/db /var/matemat/upload \ && apk --update add libmagic zlib jpeg zlib-dev jpeg-dev build-base \ && pip3 install -e . \ && apk del zlib-dev jpeg-dev build-base \ && rm -rf /var/cache/apk /root/.cache/pip \ && rm -rf /package +USER 1000 + EXPOSE 80/tcp CMD [ "/run.sh" ] diff --git a/testing/Dockerfile b/testing/Dockerfile index ca8831a..f0af7cc 100644 --- a/testing/Dockerfile +++ b/testing/Dockerfile @@ -1,15 +1,13 @@ -# There is no buster image yet and stretch doesn't have a docker package. So let's just "upgrade" the image to buster. -FROM python:3.6-stretch +FROM python:3.7-buster -RUN sed -re 's/stretch/buster/g' -i /etc/apt/sources.list \ - && useradd -d /home/matemat -m matemat \ +RUN useradd -d /home/matemat -m matemat \ && mkdir -p /var/matemat/db /var/matemat/upload \ - && chown matemat:matemat -R /var/matemat/db \ - && chown matemat:matemat -R /var/matemat/upload \ + && chown matemat:0 -R /var/matemat/db /var/matemat/upload \ + && chmod g=u -R /var/matemat/db /var/matemat/upload \ && apt-get update -qy \ && apt-get install -y --no-install-recommends file sudo openssh-client git docker.io build-essential lintian rsync \ - && python3.6 -m pip install coverage wheel pycodestyle mypy \ + && python3.7 -m pip install coverage wheel pycodestyle mypy \ && rm -rf /var/lib/apt/lists/* WORKDIR /home/matemat