From 5bace1a3f82546dad416ad80789440900797a12b Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 17 Aug 2018 20:44:07 +0200 Subject: [PATCH 1/5] Implemented pagelet initialization. --- matemat/webserver/__init__.py | 2 +- matemat/webserver/httpd.py | 41 +++++++++++++++++++- matemat/webserver/pagelets/__init__.py | 1 + matemat/webserver/pagelets/initialization.py | 28 +++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 matemat/webserver/pagelets/initialization.py diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index bf3f2f8..bd62400 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -8,5 +8,5 @@ server will attempt to serve the request with a static resource in a previously from .requestargs import RequestArgument, RequestArguments from .responses import PageletResponse, RedirectResponse, TemplateResponse -from .httpd import MatematWebserver, HttpHandler, pagelet +from .httpd import MatematWebserver, HttpHandler, pagelet, pagelet_init from .config import parse_config_file diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 20f5bc8..24916be 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,7 +1,8 @@ -from typing import Any, Callable, Dict, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Tuple, Type, Union import logging +import sys import os import socket import mimetypes @@ -38,6 +39,8 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) bytes, str, # Response body: will assign HTTP/1.0 200 OK PageletResponse, # A generic response ]]] = dict() +# The pagelet initialization functions, to be executed upon startup +_PAGELET_INIT_FUNCTIONS: List[Callable[[Dict[str, str], logging.Logger], None]] = [] # Inactivity timeout for client sessions _SESSION_TIMEOUT: int = 3600 @@ -94,6 +97,28 @@ def pagelet(path: str): return http_handler +def pagelet_init(fun: Callable[[Dict[str, str], logging.Logger], None]): + """ + Annotate a function to act as a pagelet initialization function. The function will be called when the webserver is + started. The function should set up everything required for pagelets to operate correctly, e.g. providing default + values for configuration items, or performing database schema migrations. + + Any exception thrown by an initialization function will cause the web server to seize operation. Multiple + initialization functions can be used. + + The function must have the following signature: + + (config: Dict[str, str], logger: logging.Logger) -> None + + config: The mutable dictionary of variables read from the [Pagelets] section of the configuration file. + logger: The server's logger instance. + returns: Nothing. + + :param fun: The function to annotate + """ + _PAGELET_INIT_FUNCTIONS.append(fun) + + class MatematHTTPServer(HTTPServer): """ A http.server.HTTPServer subclass that acts as a container for data that must be persistent between requests. @@ -184,8 +209,20 @@ class MatematWebserver(object): def start(self) -> None: """ - Start the web server. This call blocks while the server is running. + Call all pagelet initialization functions and start the web server. This call blocks while the server is + running. If any exception is raised in the initialization phase, the program is terminated with a non-zero + exit code. """ + try: + # Run all pagelet initialization functions + for fun in _PAGELET_INIT_FUNCTIONS: + fun(self._httpd.pagelet_variables, self._httpd.logger) + except BaseException as e: + # If an error occurs, log it and terminate + self._httpd.logger.exception(e) + self._httpd.logger.critical('An initialization pagelet raised an error. Stopping.') + sys.exit(1) + # If pagelet initialization went fine, start the HTTP server self._httpd.serve_forever() diff --git a/matemat/webserver/pagelets/__init__.py b/matemat/webserver/pagelets/__init__.py index 4ef6fd5..43b7f31 100644 --- a/matemat/webserver/pagelets/__init__.py +++ b/matemat/webserver/pagelets/__init__.py @@ -4,6 +4,7 @@ This package contains the pagelet functions served by the Matemat software. A new pagelet function must be imported here to be automatically loaded when the server is started. """ +from .initialization import initialization from .main import main_page from .login import login_page from .logout import logout diff --git a/matemat/webserver/pagelets/initialization.py b/matemat/webserver/pagelets/initialization.py new file mode 100644 index 0000000..27636b6 --- /dev/null +++ b/matemat/webserver/pagelets/initialization.py @@ -0,0 +1,28 @@ +from typing import Dict + +import logging + +from matemat.webserver import pagelet_init +from matemat.db import MatematDatabase + + +@pagelet_init +def initialization(config: Dict[str, str], + logger: logging.Logger) -> None: + """ + The pagelet initialization function. Makes sure everything is ready for operation. If anything fails here, the web + server won't resume operation. + """ + # Set default values for missing config items + if 'InstanceName' not in config: + config['InstanceName'] = 'Matemat' + logger.warning('Property \'InstanceName\' not set, using \'Matemat\'') + if 'UploadDir' not in config: + config['UploadDir'] = './static/upload/' + logger.warning('Property \'UploadDir\' not set, using \'./static/upload/\'') + if 'DatabaseFile' not in config: + config['DatabaseFile'] = './matemat.db' + logger.warning('Property \'DatabaseFile\' not set, using \'./matemat.db\'') + with MatematDatabase(config['DatabaseFile']): + # Connect to the database to create it and perform any schema migrations + pass From 8adb3402d0440596eb0e980566ab39e92d6dc4c4 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 17 Aug 2018 20:53:19 +0200 Subject: [PATCH 2/5] CI: Exclude abstract_httpd_test from coverage. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61f518d..77e0975 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ test: script: - pip3 install -r requirements.txt - sudo -u matemat python3 -m coverage run --branch -m unittest discover matemat - - sudo -u matemat python3 -m coverage report -m --include 'matemat/*' --omit '*/test_*.py' + - sudo -u matemat python3 -m coverage report -m --include 'matemat/*' --omit '*/test_*.py' --omit 'matemat/webserver/test/abstract_httpd_test.py' codestyle: stage: test From cd87695dd1c835006b03231fdb293f33acf6c8a0 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 18 Aug 2018 02:44:09 +0200 Subject: [PATCH 3/5] Pagelet initialization test. --- .gitlab-ci.yml | 5 +- matemat/webserver/httpd.py | 9 ++- matemat/webserver/test/test_pagelet_init.py | 63 +++++++++++++++++++++ setup.cfg | 19 +++++++ 4 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 matemat/webserver/test/test_pagelet_init.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77e0975..122e4f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,8 +10,9 @@ test: stage: test script: - pip3 install -r requirements.txt - - sudo -u matemat python3 -m coverage run --branch -m unittest discover matemat - - sudo -u matemat python3 -m coverage report -m --include 'matemat/*' --omit '*/test_*.py' --omit 'matemat/webserver/test/abstract_httpd_test.py' + - sudo -u matemat python3 -m coverage run --rcfile=setup.cfg -m unittest discover matemat + - sudo -u matemat python3 -m coverage combine + - sudo -u matemat python3 -m coverage report --rcfile=setup.cfg codestyle: stage: test diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 24916be..c0a351d 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,8 +1,7 @@ -from typing import Any, Callable, Dict, List, Tuple, Type, Union +from typing import Any, Callable, Dict, Set, Tuple, Type, Union import logging -import sys import os import socket import mimetypes @@ -40,7 +39,7 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) PageletResponse, # A generic response ]]] = dict() # The pagelet initialization functions, to be executed upon startup -_PAGELET_INIT_FUNCTIONS: List[Callable[[Dict[str, str], logging.Logger], None]] = [] +_PAGELET_INIT_FUNCTIONS: Set[Callable[[Dict[str, str], logging.Logger], None]] = set() # Inactivity timeout for client sessions _SESSION_TIMEOUT: int = 3600 @@ -116,7 +115,7 @@ def pagelet_init(fun: Callable[[Dict[str, str], logging.Logger], None]): :param fun: The function to annotate """ - _PAGELET_INIT_FUNCTIONS.append(fun) + _PAGELET_INIT_FUNCTIONS.add(fun) class MatematHTTPServer(HTTPServer): @@ -221,7 +220,7 @@ class MatematWebserver(object): # If an error occurs, log it and terminate self._httpd.logger.exception(e) self._httpd.logger.critical('An initialization pagelet raised an error. Stopping.') - sys.exit(1) + raise e # If pagelet initialization went fine, start the HTTP server self._httpd.serve_forever() diff --git a/matemat/webserver/test/test_pagelet_init.py b/matemat/webserver/test/test_pagelet_init.py new file mode 100644 index 0000000..21c30e4 --- /dev/null +++ b/matemat/webserver/test/test_pagelet_init.py @@ -0,0 +1,63 @@ +from typing import Any, Dict + +import unittest +import http.client + +import logging +import threading + +from matemat.webserver import MatematWebserver, RequestArguments, pagelet_init, pagelet + + +@pagelet('/just/testing/init') +def init_test_pagelet(method: str, + path: str, + args: RequestArguments, + session_vars: Dict[str, Any], + headers: Dict[str, str], + pagelet_variables: Dict[str, str]): + return pagelet_variables['Unit-Test'] + + +_INIT_FAIL = False + + +@pagelet_init +def init(config: Dict[str, str], + logger: logging.Logger): + if _INIT_FAIL: + raise ValueError('This error should be raised!') + config['Unit-Test'] = 'Pagelet Init Test' + + +class TestPageletInitialization(unittest.TestCase): + + def tearDown(self): + global _INIT_FAIL + _INIT_FAIL = False + + def test_pagelet_init_ok(self): + """ + Test successful pagelet initialization + """ + srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, logging.NOTSET, logging.NullHandler()) + port = int(srv._httpd.socket.getsockname()[1]) + thread = threading.Thread(target=srv.start) + thread.start() + con = http.client.HTTPConnection(f'[::1]:{port}') + con.request('GET', '/just/testing/init') + response = con.getresponse().read() + srv._httpd.shutdown() + srv._httpd.socket.close() + self.assertEqual(b'Pagelet Init Test', response) + + def test_pagelet_init_fail(self): + """ + Test unsuccessful pagelet initialization + """ + global _INIT_FAIL + _INIT_FAIL = True + srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, logging.NOTSET, logging.NullHandler()) + with self.assertRaises(ValueError): + srv.start() + srv._httpd.socket.close() diff --git a/setup.cfg b/setup.cfg index 62acd3a..2dcea66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,22 @@ + +# +# PyCodestyle +# + [pycodestyle] max-line-length = 120 statistics = True + +# +# Coverage +# + +[run] +branch = True +parallel = True +source = matemat/ + +[report] +show_missing = True +include = matemat/* +omit = */test/*.py From 0fe8ef55a2418e1853a64a7ac0b8826d8cc96250 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 19 Aug 2018 01:06:38 +0200 Subject: [PATCH 4/5] Pagelet initialization tests: Added safeguards against non-terminating threads. --- matemat/webserver/test/test_pagelet_init.py | 23 +++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/matemat/webserver/test/test_pagelet_init.py b/matemat/webserver/test/test_pagelet_init.py index 21c30e4..0b2c45a 100644 --- a/matemat/webserver/test/test_pagelet_init.py +++ b/matemat/webserver/test/test_pagelet_init.py @@ -32,7 +32,17 @@ def init(config: Dict[str, str], class TestPageletInitialization(unittest.TestCase): + def setUp(self): + self.srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, + logging.NOTSET, logging.NullHandler()) + self.srv_port = int(self.srv._httpd.socket.getsockname()[1]) + self.timer = threading.Timer(5.0, self.srv._httpd.shutdown) + self.timer.start() + def tearDown(self): + self.timer.cancel() + if self.srv is not None: + self.srv._httpd.socket.close() global _INIT_FAIL _INIT_FAIL = False @@ -40,15 +50,12 @@ class TestPageletInitialization(unittest.TestCase): """ Test successful pagelet initialization """ - srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, logging.NOTSET, logging.NullHandler()) - port = int(srv._httpd.socket.getsockname()[1]) - thread = threading.Thread(target=srv.start) + thread = threading.Thread(target=self.srv.start) thread.start() - con = http.client.HTTPConnection(f'[::1]:{port}') + con = http.client.HTTPConnection(f'[::1]:{self.srv_port}') con.request('GET', '/just/testing/init') response = con.getresponse().read() - srv._httpd.shutdown() - srv._httpd.socket.close() + self.srv._httpd.shutdown() self.assertEqual(b'Pagelet Init Test', response) def test_pagelet_init_fail(self): @@ -57,7 +64,5 @@ class TestPageletInitialization(unittest.TestCase): """ global _INIT_FAIL _INIT_FAIL = True - srv = MatematWebserver('::1', 0, '/nonexistent', '/nonexistent', {}, {}, logging.NOTSET, logging.NullHandler()) with self.assertRaises(ValueError): - srv.start() - srv._httpd.socket.close() + self.srv.start() From b1af98409ec82a4aab7f907d3372c7e530773e44 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 21 Aug 2018 00:41:52 +0200 Subject: [PATCH 5/5] Added pagelet initialization documentation --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 9449a6d..0cf3d59 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 9449a6dc39843969d3b549f05848b5857c23cfa3 +Subproject commit 0cf3d59c8b37f84e915f5e30e7447f0611cc1238