Update docker images

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 ---
-image: s3lph/matemat-ci:20181107-02
+image: s3lph/matemat-ci:20200203-01
 
 stages:
 - test
@@ -17,9 +17,9 @@ test:
   stage: test
   script:
   - pip3 install -e .
-  - sudo -u matemat python3.6 -m coverage run --rcfile=setup.cfg -m unittest discover matemat
+  - sudo -u matemat python3.7 -m coverage run --rcfile=setup.cfg -m unittest discover matemat
-  - sudo -u matemat python3.6 -m coverage combine
+  - sudo -u matemat python3.7 -m coverage combine
-  - sudo -u matemat python3.6 -m coverage report --rcfile=setup.cfg
+  - sudo -u matemat python3.7 -m coverage report --rcfile=setup.cfg
 
 codestyle:
   stage: test

matemat/webserver/test/abstract_httpd_test.py

@@ -1,73 +0,0 @@
-import threading
-from http.client import HTTPResponse
-from typing import Any, Callable, Dict, Tuple, Union
-
-import unittest.mock
-import urllib.request
-from tempfile import TemporaryDirectory
-
-from abc import ABC
-from datetime import datetime
-
-import logging
-import jinja2
-
-import bottle
-
-
-class MockServer:
-    """
-    A mock implementation of http.server.HTTPServer.  Only used for matemat-specific storage.
-    """
-
-    def __init__(self, webroot: str = '/var/matemat/webroot') -> None:
-        # Session timeout and variables for all sessions
-        self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict()
-        # Webroot for statically served content
-        self.webroot: str = webroot
-        # Variables to pass to pagelets
-        self.pagelet_variables: Dict[str, str] = dict()
-        # Static response headers
-        self.static_headers: Dict[str, str] = {
-            'X-Static-Testing-Header': 'helloworld!'
-        }
-        # Jinja environment with a single, static template
-        self.jinja_env = jinja2.Environment(
-            loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'})
-        )
-        # Set up logger
-        self.logger: logging.Logger = logging.getLogger('matemat unit test')
-        self.logger.setLevel(logging.DEBUG)
-        # Disable logging
-        self.logger.addHandler(logging.NullHandler())
-        # Initalize a log handler to stderr and set the log format
-        # sh: logging.StreamHandler = logging.StreamHandler()
-        # sh.setFormatter(logging.Formatter('%(asctime)s %(name)s [%(levelname)s]: %(message)s'))
-        # self.logger.addHandler(sh)
-        bottle.run(host='::1', port='8888')
-
-
-class AbstractHttpdTest(ABC, unittest.TestCase):
-    """
-    An abstract test case that can be inherited by test case classes that want to test part of the webserver's core
-    functionality.
-
-    Usage (subclass test method):
-
-        self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n')
-        handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
-        packet = self.client_sock.get_response()
-    """
-
-    def setUp(self) -> None:
-        self.tempdir: TemporaryDirectory = TemporaryDirectory(prefix='matemat.', dir='/tmp/')
-        self.server = threading.Thread(target=bottle.run, kwargs={'host': '::1', 'port': 8888, 'debug': True})
-        self.server.start()
-
-    def tearDown(self):
-        self.server._stop()
-        self.tempdir.cleanup()
-
-    @staticmethod
-    def request(query) -> HTTPResponse:
-        return urllib.request.urlopen(f'http://[::1]:8888{query}')

matemat/webserver/test/test_config.py

@@ -8,7 +8,7 @@ from io import StringIO
 import logging
 import sys
 
-from matemat.webserver.config import parse_config_file
+from matemat.webserver.config import parse_config_file, get_config
 
 _EMPTY_CONFIG = ''
 
@@ -112,7 +112,8 @@ class TestConfig(TestCase):
         # Mock the open() function to return an empty config file example
         with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)):
             # The filename is only a placeholder, file content is determined by mocking open
-            config = parse_config_file('test')
+            parse_config_file('test')
+        config = get_config()
         # Make sure all mandatory values are present
         self.assertIn('listen', config)
         self.assertIn('port', config)
@@ -141,7 +142,8 @@ class TestConfig(TestCase):
         # Mock the open() function to return a full config file example
         with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)):
             # The filename is only a placeholder, file content is determined by mocking open
-            config = parse_config_file('test')
+            parse_config_file('test')
+        config = get_config()
         # Make sure all mandatory values are present
         self.assertIn('listen', config)
         self.assertIn('port', config)
@@ -186,7 +188,8 @@ class TestConfig(TestCase):
         # second call
         with patch('builtins.open', return_value=IterOpenMock([_FULL_CONFIG, _PARTIAL_CONFIG])):
             # These filenames are only placeholders, file content is determined by mocking open
-            config = parse_config_file(['full', 'partial'])
+            parse_config_file(['full', 'partial'])
+        config = get_config()
         # Make sure all mandatory values are present
         self.assertIn('listen', config)
         self.assertIn('port', config)
@@ -216,7 +219,8 @@ class TestConfig(TestCase):
         # Mock the open() function to return a config file example with disabled logging
         with patch('builtins.open', return_value=StringIO(_LOG_NONE_CONFIG)):
             # The filename is only a placeholder, file content is determined by mocking open
-            config = parse_config_file('test')
+            parse_config_file('test')
+        config = get_config()
         # Make sure the returned log handler is a null handler
         self.assertIsInstance(config['log_handler'], logging.NullHandler)
 
@@ -227,7 +231,8 @@ class TestConfig(TestCase):
         # Mock the open() function to return a config file example with stdout logging
         with patch('builtins.open', return_value=StringIO(_LOG_STDOUT_CONFIG)):
             # The filename is only a placeholder, file content is determined by mocking open
-            config = parse_config_file('test')
+            parse_config_file('test')
+        config = get_config()
         # Make sure the returned log handler is a stdout handler
         self.assertIsInstance(config['log_handler'], logging.StreamHandler)
         self.assertEqual(sys.stdout, config['log_handler'].stream)
@@ -239,7 +244,8 @@ class TestConfig(TestCase):
         # Mock the open() function to return a config file example with stdout logging
         with patch('builtins.open', return_value=StringIO(_LOG_STDERR_CONFIG)):
             # The filename is only a placeholder, file content is determined by mocking open
-            config = parse_config_file('test')
+            parse_config_file('test')
+        config = get_config()
         # Make sure the returned log handler is a stdout handler
         self.assertIsInstance(config['log_handler'], logging.StreamHandler)
         self.assertEqual(sys.stderr, config['log_handler'].stream)

matemat/webserver/test/test_pagelet_cron.py

@@ -1,65 +0,0 @@
-from typing import Dict
-
-import unittest
-
-import logging
-from threading import Lock, Thread, Timer
-from time import sleep
-import jinja2
-
-from matemat.webserver.cron import cron
-
-lock: Lock = Lock()
-
-cron1called: int = 0
-cron2called: int = 0
-
-
-@cron(seconds=4)
-def cron1(config: Dict[str, str],
-          jinja_env: jinja2.Environment,
-          logger: logging.Logger) -> None:
-    global cron1called
-    with lock:
-        cron1called += 1
-
-
-@cron(seconds=3)
-def cron2(config: Dict[str, str],
-          jinja_env: jinja2.Environment,
-          logger: logging.Logger) -> None:
-    global cron2called
-    with lock:
-        cron2called += 1
-
-
-class TestPageletCron(unittest.TestCase):
-
-    def setUp(self):
-        self.srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {},
-                                    logging.NOTSET, logging.NullHandler())
-        self.srv_port = int(self.srv._httpd.socket.getsockname()[1])
-        self.timer = Timer(10.0, self.srv._httpd.shutdown)
-        self.timer.start()
-
-    def tearDown(self):
-        self.timer.cancel()
-        if self.srv is not None:
-            self.srv._httpd.socket.close()
-
-    def test_cron(self):
-        """
-        Test that the cron functions are called properly.
-        """
-        thread = Thread(target=self.srv.start)
-        thread.start()
-        sleep(12)
-        self.srv._httpd.shutdown()
-        with lock:
-            self.assertEqual(2, cron1called)
-            self.assertEqual(3, cron2called)
-        # Make sure the cron threads were stopped
-        sleep(5)
-        with lock:
-            self.assertEqual(2, cron1called)
-            self.assertEqual(3, cron2called)

matemat/webserver/test/test_session.py

@@ -1,148 +0,0 @@
-
-from datetime import datetime, timedelta
-from time import sleep
-
-from bottle import route, run
-
-from matemat.webserver import session
-from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest
-
-
-@route('/just/testing/sessions')
-def session_test_pagelet():
-    s = session.start()
-    session.put(s, 'test', 'hello, world!')
-    return f'{s}'
-
-
-class TestSession(AbstractHttpdTest):
-
-    """
-    Test session handling of the Matemat webserver.
-    """
-
-    def test_create_new_session(self):
-        # Reference date to make sure the session expiry lies in the future
-        refdate: datetime = datetime.utcnow() + timedelta(seconds=3500)
-        # Send a mock GET request for '/just/testing/sessions'
-
-        packet = self.request('/just/testing/sessions')
-        # Make sure the request was served by the test pagelet
-        self.assertEqual(200, packet.code)
-
-        session_id: str = packet.read().decode()
-        # Make sure a cookie was set - assuming that only one was set
-        self.assertIn('Set-Cookie', packet.headers)
-        # Split into the cookie itself
-        cookie, expiry = packet.headers['Set-Cookie'].split(';')
-        cookie: str = cookie.strip()
-        expiry: str = expiry.strip()
-        # Make sure the 'matemat_session_id' cookie was set to the session ID string
-        self.assertEqual(f'matemat_session_id={session_id}', cookie)
-        # Make sure the session expires in about one hour
-        self.assertTrue(expiry.startswith('expires='))
-        _, expdatestr = expiry.split('=', 1)
-        expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
-        self.assertTrue(expdate > refdate)
-
-    def test_resume_session(self):
-        # Test session expiry date
-        refdate: datetime = datetime.utcnow() + timedelta(hours=1)
-        # Session ID for testing
-        session_id: str = 'testsessionid'
-        # Insert test session
-        self.server.session_vars[session_id] = refdate, {'test': 'bar'}
-        sleep(2)
-
-        # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie
-        self.client_sock.set_request(
-            f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
-        # Trigger request handling
-        handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
-        # Fetch the parsed response
-        packet = self.client_sock.get_response()
-        # Make sure a full HTTP response was parsed
-        self.assertEqual('done', packet.parse_phase)
-        # Make sure the request was served by the test pagelet
-        self.assertEqual('session_test_pagelet', packet.pagelet)
-        self.assertEqual(200, packet.statuscode)
-
-        response_session_id: str = list(handler.server.session_vars.keys())[0]
-        # Make sure a cookie was set - assuming that only one was set
-        self.assertIn('Set-Cookie', packet.headers)
-        # Split into the cookie itself
-        cookie, expiry = packet.headers['Set-Cookie'].split(';')
-        cookie: str = cookie.strip()
-        expiry: str = expiry.strip()
-        # Make sure the 'matemat_session_id' cookie was set to the session ID string
-        self.assertEqual(f'matemat_session_id={response_session_id}', cookie)
-        # Make sure the session ID matches the one we sent along
-        self.assertEqual(session_id, response_session_id)
-        # Make sure the session timeout was postponed
-        self.assertTrue(expiry.startswith('expires='))
-        _, expdatestr = expiry.split('=', 1)
-        expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
-        self.assertTrue(expdate > refdate)
-        # Make sure the session exists on the server
-        self.assertIn('test', handler.session_vars)
-        self.assertEqual('hello, world!', handler.session_vars['test'])
-
-    def test_unknown_session_id(self):
-        # Unknown session ID
-        session_id: str = 'theserverdoesnotknowthisid'
-        refdate: datetime = datetime.utcnow() + timedelta(seconds=3500)
-        # Send a mock GET request for '/just/testing/sessions' with a session cookie not known to the server
-        self.client_sock.set_request(
-            f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
-        # Trigger request handling
-        handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
-        # Fetch the parsed response
-        packet = self.client_sock.get_response()
-        # Make sure a full HTTP response was parsed
-        self.assertEqual('done', packet.parse_phase)
-        # Make sure the request was served by the test pagelet
-        self.assertEqual('session_test_pagelet', packet.pagelet)
-        self.assertEqual(200, packet.statuscode)
-
-        server_session_id: str = list(handler.server.session_vars.keys())[0]
-        self.assertNotEqual(session_id, server_session_id)
-        # Make sure a cookie was set - assuming that only one was set
-        self.assertIn('Set-Cookie', packet.headers)
-        # Split into the cookie itself
-        cookie, expiry = packet.headers['Set-Cookie'].split(';')
-        cookie: str = cookie.strip()
-        expiry: str = expiry.strip()
-        # Make sure the 'matemat_session_id' cookie was set to the session ID string
-        self.assertEqual(f'matemat_session_id={server_session_id}', cookie)
-        # Make sure the session expires in about one hour
-        self.assertTrue(expiry.startswith('expires='))
-        _, expdatestr = expiry.split('=', 1)
-        expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT')
-        self.assertTrue(expdate > refdate)
-        # Make sure the session exists on the server
-        self.assertIn('test', handler.session_vars)
-        self.assertEqual('hello, world!', handler.session_vars['test'])
-
-    def test_session_expired(self):
-        # Test session expiry date
-        refdate: datetime = datetime.utcnow() - timedelta(hours=1)
-        # Session ID for testing
-        session_id: str = 'testsessionid'
-        # Insert test session
-        self.server.session_vars[session_id] = refdate, {'test': 'bar'}
-
-        # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie
-        self.client_sock.set_request(
-            f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8'))
-        # Trigger request handling
-        handler = HttpHandler(self.client_sock, ('::1', 45678), self.server)
-        # Fetch the parsed response
-        packet = self.client_sock.get_response()
-        # Make sure a full HTTP response was parsed
-        self.assertEqual('done', packet.parse_phase)
-        # Make sure the server redirects to /
-        self.assertEqual(302, packet.statuscode)
-        self.assertIn('Location', packet.headers)
-        self.assertEqual('/', packet.headers['Location'])
-        # Make sure the session was terminated
-        self.assertNotIn(session_id, self.server.session_vars)

package/docker/Dockerfile

@@ -3,11 +3,15 @@ FROM python:3.7-alpine
 
 ADD . /
 RUN mkdir -p /var/matemat/db /var/matemat/upload \
+  && chown 1000:0 -R /var/matemat/db /var/matemat/upload \
+  && chmod g=u -R /var/matemat/db /var/matemat/upload \
   && apk --update add libmagic zlib jpeg zlib-dev jpeg-dev build-base \
   && pip3 install -e . \
   && apk del zlib-dev jpeg-dev build-base \
   && rm -rf /var/cache/apk /root/.cache/pip \
   && rm -rf /package
 
+USER 1000
+
 EXPOSE 80/tcp
 CMD [ "/run.sh" ]

testing/Dockerfile

@@ -1,15 +1,13 @@
 
-# There is no buster image yet and stretch doesn't have a docker package.  So let's just "upgrade" the image to buster.
+FROM python:3.7-buster
-FROM python:3.6-stretch
 
-RUN sed -re 's/stretch/buster/g' -i /etc/apt/sources.list \
+RUN useradd -d /home/matemat -m matemat \
-  && useradd -d /home/matemat -m matemat \
   && mkdir -p /var/matemat/db /var/matemat/upload \
-  && chown matemat:matemat -R /var/matemat/db \
+  && chown matemat:0 -R /var/matemat/db /var/matemat/upload \
-  && chown matemat:matemat -R /var/matemat/upload \
+  && chmod g=u -R /var/matemat/db /var/matemat/upload \
   && apt-get update -qy \
   && apt-get install -y --no-install-recommends file sudo openssh-client git docker.io build-essential lintian rsync \
-  && python3.6 -m pip install coverage wheel pycodestyle mypy \
+  && python3.7 -m pip install coverage wheel pycodestyle mypy \
   && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/matemat