From a52f09fc58b9078eed02e24aa08a9d77bc8afba4 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 11 Jul 2018 22:06:06 +0200 Subject: [PATCH 1/8] First, horrible config file support. --- .gitignore | 1 + matemat/__init__.py | 2 ++ matemat/__main__.py | 15 ++++++++--- matemat/config.py | 29 +++++++++++++++++++++ matemat/webserver/httpd.py | 35 ++++++++++++++++++-------- matemat/webserver/pagelets/login.py | 7 +++--- matemat/webserver/pagelets/logout.py | 3 ++- matemat/webserver/pagelets/main.py | 9 ++++--- matemat/webserver/pagelets/touchkey.py | 9 ++++--- templates/base.html | 4 +-- 10 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 matemat/config.py diff --git a/.gitignore b/.gitignore index cca262e..fef1fd9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ *.sqlite3 *.db +**/matemat.conf diff --git a/matemat/__init__.py b/matemat/__init__.py index a3332a5..2d3934d 100644 --- a/matemat/__init__.py +++ b/matemat/__init__.py @@ -1,2 +1,4 @@ __version__ = '2.0' + +from .config import parse_config_file diff --git a/matemat/__main__.py b/matemat/__main__.py index 9654b64..cc69d4a 100644 --- a/matemat/__main__.py +++ b/matemat/__main__.py @@ -1,16 +1,23 @@ +from typing import Any, Dict + import sys +from matemat import parse_config_file + if __name__ == '__main__': # Those imports are actually needed, as they implicitly register pagelets. # noinspection PyUnresolvedReferences from matemat.webserver.pagelets import * from matemat.webserver import MatematWebserver - # Read HTTP port from command line - port: int = 8080 + # Use config file name from command line, if present + configfile: str = '/etc/matemat.conf' if len(sys.argv) > 1: - port = int(sys.argv[1]) + configfile = sys.argv[1] + + # Parse the config file + config: Dict[str, Any] = parse_config_file(configfile) # Start the web server - MatematWebserver(port=port).start() + MatematWebserver(**config).start() diff --git a/matemat/config.py b/matemat/config.py new file mode 100644 index 0000000..48963dd --- /dev/null +++ b/matemat/config.py @@ -0,0 +1,29 @@ + +from typing import Any, Dict + +from configparser import ConfigParser, NoSectionError + + +def parse_config_file(path: str) -> Dict[str, Any]: + config: Dict[str, Any] = dict() + + parser: ConfigParser = ConfigParser() + parser.read(path, 'utf-8') + if 'Matemat' not in parser.sections(): + raise NoSectionError(f'Section "matemat" missing in file {path}') + config['listen'] = parser['Matemat'].get('Address', '::') + config['port'] = int(parser['Matemat'].get('Port', '8080')) + config['dbpath'] = parser['Matemat'].get('DatabaseFile', '/var/matemat/db/matemat.sqlite3') + config['staticroot'] = parser['Matemat'].get('StaticPath', '/var/matemat/static/') + config['templateroot'] = parser['Matemat'].get('TemplatePath', '/var/matemat/templates/') + + config['pagelet_arguments']: Dict[str, str] = dict() + config['pagelet_arguments']['DatabaseFile'] = config['dbpath'] + config['pagelet_arguments']['StaticPath'] = config['staticroot'] + config['pagelet_arguments']['TemplatePath'] = config['templateroot'] + + if 'Pagelets' in parser.sections(): + for k, v in parser['Pagelets'].items(): + config['pagelet_arguments'][k] = v + + return config diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 41866de..c3a35c6 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -34,7 +34,8 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) str, # Request path RequestArguments, # HTTP Request arguments Dict[str, Any], # Session vars - Dict[str, str]], # Response headers + Dict[str, str], # Response headers + Dict[str, str]], # Items from the [Pagelets] section in the config file Union[ # Return type: either a response body, or a redirect bytes, str, # Response body: will assign HTTP/1.0 200 OK PageletResponse, # A generic response @@ -77,6 +78,7 @@ def pagelet(path: str): str, RequestArguments, Dict[str, Any], + Dict[str, str], Dict[str, str]], Union[ bytes, str, @@ -102,12 +104,17 @@ class MatematHTTPServer(HTTPServer): handler: Type[BaseHTTPRequestHandler], staticroot: str, templateroot: str, + dbpath: str, + pagelet_arguments: Dict[str, str], bind_and_activate: bool = True) -> None: super().__init__(server_address, handler, bind_and_activate) - # Resolve webroot directory + # Resolve webroot directory and database file self.webroot = os.path.abspath(staticroot) + self.database_file = os.path.abspath(dbpath) # Set up session vars dict self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + # Set up pagelet arguments dict + self.pagelet_arguments = pagelet_arguments # Set up the Jinja2 environment self.jinja_env: jinja2.Environment = jinja2.Environment( loader=jinja2.FileSystemLoader(os.path.abspath(templateroot)) @@ -130,10 +137,12 @@ class MatematWebserver(object): """ def __init__(self, - listen: str = '::', - port: int = 80, - staticroot: str = './static', - templateroot: str = './templates') -> None: + listen: str, + port: int, + staticroot: str, + templateroot: str, + dbpath: str, + pagelet_arguments: Dict[str, str]) -> None: """ Instantiate a MatematWebserver. @@ -141,16 +150,14 @@ class MatematWebserver(object): :param port: The TCP port to listen on. :param staticroot: Path to the static webroot directory. :param templateroot: Path to the Jinja2 templates root directory. + :param dbpath: Path to the database file. """ - if len(listen) == 0: - # Empty string should be interpreted as all addresses - listen = '::' # IPv4 address detection heuristic if ':' not in listen and '.' in listen: # Rewrite IPv4 address to IPv6-mapped form listen = f'::ffff:{listen}' # Create the http server - self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot) + self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot, dbpath, pagelet_arguments) def start(self) -> None: """ @@ -289,8 +296,14 @@ class HttpHandler(BaseHTTPRequestHandler): 'Content-Type': 'text/html', 'Cache-Control': 'no-cache' } + print(self.server.pagelet_arguments) # Call the pagelet function - pagelet_res = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) + pagelet_res = _PAGELET_PATHS[path](method, + path, + args, + self.session_vars, + headers, + self.server.pagelet_arguments) # Parse the pagelet's return value, vielding a HTTP status code and a response body hsc, data = self._parse_pagelet_result(pagelet_res, headers) # Send the HTTP status code diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index 6ef9a9c..a36dda2 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -12,14 +12,15 @@ def login_page(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str])\ + headers: Dict[str, str], + config: Dict[str, str])\ -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: return RedirectResponse('/') if method == 'GET': - return TemplateResponse('login.html') + return TemplateResponse('login.html', setupname=config['name']) elif method == 'POST': - with MatematDatabase('test.db') as db: + with MatematDatabase(config['DatabaseFile']) as db: try: user: User = db.login(str(args.username), str(args.password)) except AuthenticationError: diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index cbd7cad..4e18258 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -9,7 +9,8 @@ def logout(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str])\ + headers: Dict[str, str], + config: Dict[str, str])\ -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: del session_vars['user'] diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index be0bd60..534cf61 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -11,13 +11,14 @@ def main_page(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str])\ + headers: Dict[str, str], + config: Dict[str, str])\ -> Union[bytes, str, PageletResponse]: - with MatematDatabase('test.db') as db: + with MatematDatabase(config['DatabaseFile']) as db: if 'user' in session_vars: user: User = session_vars['user'] products = db.list_products() - return TemplateResponse('main.html', user=user, list=products) + return TemplateResponse('main.html', user=user, list=products, setupname=config['name']) else: users = db.list_users() - return TemplateResponse('main.html', list=users) + return TemplateResponse('main.html', list=users, setupname=config['name']) diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 2b81c89..26c481e 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -14,14 +14,17 @@ def touchkey_page(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str])\ + headers: Dict[str, str], + config: Dict[str, str])\ -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: return RedirectResponse('/') if method == 'GET': - return TemplateResponse('touchkey.html', username=str(args.username) if 'username' in args else None) + return TemplateResponse('touchkey.html', + username=str(args.username) if 'username' in args else None, + setupname=config['name']) elif method == 'POST': - with MatematDatabase('test.db') as db: + with MatematDatabase(config['DatabaseFile']) as db: try: user: User = db.login(str(args.username), touchkey=str(args.touchkey)) except AuthenticationError: diff --git a/templates/base.html b/templates/base.html index 1ddd113..5bae28f 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,7 +1,7 @@ - Matemat + {{ setupname }} -

Matemat {{__version__}}

+

{{ setupname }}

{% block main %} {% endblock %} From 2dd57dcfd698033ab0e3f484d6ffa1a89d30d551 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 00:14:48 +0200 Subject: [PATCH 2/8] Reworked and cleaned up config file integration. --- matemat/__init__.py | 2 -- matemat/__main__.py | 2 +- matemat/config.py | 29 ---------------- matemat/webserver/__init__.py | 1 + matemat/webserver/config.py | 47 ++++++++++++++++++++++++++ matemat/webserver/httpd.py | 22 ++++++------ matemat/webserver/pagelets/login.py | 2 +- matemat/webserver/pagelets/main.py | 4 +-- matemat/webserver/pagelets/touchkey.py | 2 +- 9 files changed, 63 insertions(+), 48 deletions(-) delete mode 100644 matemat/config.py create mode 100644 matemat/webserver/config.py diff --git a/matemat/__init__.py b/matemat/__init__.py index 2d3934d..a3332a5 100644 --- a/matemat/__init__.py +++ b/matemat/__init__.py @@ -1,4 +1,2 @@ __version__ = '2.0' - -from .config import parse_config_file diff --git a/matemat/__main__.py b/matemat/__main__.py index cc69d4a..b461855 100644 --- a/matemat/__main__.py +++ b/matemat/__main__.py @@ -3,7 +3,7 @@ from typing import Any, Dict import sys -from matemat import parse_config_file +from matemat.webserver import parse_config_file if __name__ == '__main__': # Those imports are actually needed, as they implicitly register pagelets. diff --git a/matemat/config.py b/matemat/config.py deleted file mode 100644 index 48963dd..0000000 --- a/matemat/config.py +++ /dev/null @@ -1,29 +0,0 @@ - -from typing import Any, Dict - -from configparser import ConfigParser, NoSectionError - - -def parse_config_file(path: str) -> Dict[str, Any]: - config: Dict[str, Any] = dict() - - parser: ConfigParser = ConfigParser() - parser.read(path, 'utf-8') - if 'Matemat' not in parser.sections(): - raise NoSectionError(f'Section "matemat" missing in file {path}') - config['listen'] = parser['Matemat'].get('Address', '::') - config['port'] = int(parser['Matemat'].get('Port', '8080')) - config['dbpath'] = parser['Matemat'].get('DatabaseFile', '/var/matemat/db/matemat.sqlite3') - config['staticroot'] = parser['Matemat'].get('StaticPath', '/var/matemat/static/') - config['templateroot'] = parser['Matemat'].get('TemplatePath', '/var/matemat/templates/') - - config['pagelet_arguments']: Dict[str, str] = dict() - config['pagelet_arguments']['DatabaseFile'] = config['dbpath'] - config['pagelet_arguments']['StaticPath'] = config['staticroot'] - config['pagelet_arguments']['TemplatePath'] = config['templateroot'] - - if 'Pagelets' in parser.sections(): - for k, v in parser['Pagelets'].items(): - config['pagelet_arguments'][k] = v - - return config diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index 6059687..bf3f2f8 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -9,3 +9,4 @@ server will attempt to serve the request with a static resource in a previously from .requestargs import RequestArgument, RequestArguments from .responses import PageletResponse, RedirectResponse, TemplateResponse from .httpd import MatematWebserver, HttpHandler, pagelet +from .config import parse_config_file diff --git a/matemat/webserver/config.py b/matemat/webserver/config.py new file mode 100644 index 0000000..c220adc --- /dev/null +++ b/matemat/webserver/config.py @@ -0,0 +1,47 @@ + +from typing import Any, Dict + +from configparser import ConfigParser + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse the configuration file at the given path. + + :param path: The config file to parse. + :return: A dictionary containing the parsed configuration. + """ + # Set up default values + config: Dict[str, Any] = { + # Address to listen on + 'listen': '::', + # TCP port to listen on + 'port': 80, + # Root directory of statically served content + 'staticroot': '/var/matemat/static', + # Root directory of Jinja2 templates + 'templateroot': '/var/matemat/templates', + # Variables passed to pagelets + 'pagelet_variables': dict() + } + + # Initialize the config parser + parser: ConfigParser = ConfigParser() + # Replace the original option transformation by a string constructor to preserve the case of config keys + parser.optionxform = str + # Read the configuration file + parser.read(path, 'utf-8') + + # Read values from the [Matemat] section, if present, falling back to default values + if 'Matemat' in parser.sections(): + config['listen'] = parser['Matemat'].get('Address', config['listen']) + config['port'] = int(parser['Matemat'].get('Port', config['port'])) + config['staticroot'] = parser['Matemat'].get('StaticPath', config['staticroot']) + config['templateroot'] = parser['Matemat'].get('TemplatePath', config['templateroot']) + + # Read all values from the [Pagelets] section, if present. These values are passed to pagelet functions + if 'Pagelets' in parser.sections(): + for k, v in parser['Pagelets'].items(): + config['pagelet_variables'][k] = v + + return config diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index c3a35c6..2d5194d 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -57,7 +57,8 @@ def pagelet(path: str): path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) + headers: Dict[str, str], + config: Dict[str, str]) -> Union[bytes, str, Tuple[int, str]] method: The HTTP method (GET, POST) that was used. @@ -65,6 +66,7 @@ def pagelet(path: str): args: The arguments that were passed with the request (as GET or POST arguments). session_vars: The session storage. May be read from and written to. headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. + config: The dictionary of variables read from the [Pagelets] section of the configuration file. returns: One of the following: - A HTTP Response body as str or bytes - A PageletResponse class instance: An instance of (a subclass of) @@ -104,17 +106,15 @@ class MatematHTTPServer(HTTPServer): handler: Type[BaseHTTPRequestHandler], staticroot: str, templateroot: str, - dbpath: str, - pagelet_arguments: Dict[str, str], + pagelet_variables: Dict[str, str], bind_and_activate: bool = True) -> None: super().__init__(server_address, handler, bind_and_activate) - # Resolve webroot directory and database file + # Resolve webroot directory self.webroot = os.path.abspath(staticroot) - self.database_file = os.path.abspath(dbpath) # Set up session vars dict self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() # Set up pagelet arguments dict - self.pagelet_arguments = pagelet_arguments + self.pagelet_variables = pagelet_variables # Set up the Jinja2 environment self.jinja_env: jinja2.Environment = jinja2.Environment( loader=jinja2.FileSystemLoader(os.path.abspath(templateroot)) @@ -141,8 +141,7 @@ class MatematWebserver(object): port: int, staticroot: str, templateroot: str, - dbpath: str, - pagelet_arguments: Dict[str, str]) -> None: + pagelet_variables: Dict[str, str]) -> None: """ Instantiate a MatematWebserver. @@ -150,14 +149,14 @@ class MatematWebserver(object): :param port: The TCP port to listen on. :param staticroot: Path to the static webroot directory. :param templateroot: Path to the Jinja2 templates root directory. - :param dbpath: Path to the database file. + :param pagelet_variables: Dictionary of variables to pass to pagelet functions. """ # IPv4 address detection heuristic if ':' not in listen and '.' in listen: # Rewrite IPv4 address to IPv6-mapped form listen = f'::ffff:{listen}' # Create the http server - self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot, dbpath, pagelet_arguments) + self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot, pagelet_variables) def start(self) -> None: """ @@ -296,14 +295,13 @@ class HttpHandler(BaseHTTPRequestHandler): 'Content-Type': 'text/html', 'Cache-Control': 'no-cache' } - print(self.server.pagelet_arguments) # Call the pagelet function pagelet_res = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers, - self.server.pagelet_arguments) + self.server.pagelet_variables) # Parse the pagelet's return value, vielding a HTTP status code and a response body hsc, data = self._parse_pagelet_result(pagelet_res, headers) # Send the HTTP status code diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index a36dda2..e956795 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -18,7 +18,7 @@ def login_page(method: str, if 'user' in session_vars: return RedirectResponse('/') if method == 'GET': - return TemplateResponse('login.html', setupname=config['name']) + return TemplateResponse('login.html', setupname=config['InstanceName']) elif method == 'POST': with MatematDatabase(config['DatabaseFile']) as db: try: diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index 534cf61..b96364d 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -18,7 +18,7 @@ def main_page(method: str, if 'user' in session_vars: user: User = session_vars['user'] products = db.list_products() - return TemplateResponse('main.html', user=user, list=products, setupname=config['name']) + return TemplateResponse('main.html', user=user, list=products, setupname=config['InstanceName']) else: users = db.list_users() - return TemplateResponse('main.html', list=users, setupname=config['name']) + return TemplateResponse('main.html', list=users, setupname=config['InstanceName']) diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 26c481e..2470ffd 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -22,7 +22,7 @@ def touchkey_page(method: str, if method == 'GET': return TemplateResponse('touchkey.html', username=str(args.username) if 'username' in args else None, - setupname=config['name']) + setupname=config['InstanceName']) elif method == 'POST': with MatematDatabase(config['DatabaseFile']) as db: try: From 23e6662ca82653e2973f79a6d89cf27e665dc695 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 00:15:08 +0200 Subject: [PATCH 3/8] Fixed previous unit tests and implemented tests for config parsing. --- matemat/webserver/test/abstract_httpd_test.py | 8 ++- matemat/webserver/test/test_config.py | 63 +++++++++++++++++++ matemat/webserver/test/test_post.py | 3 +- matemat/webserver/test/test_serve.py | 15 +++-- matemat/webserver/test/test_session.py | 3 +- 5 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 matemat/webserver/test/test_config.py diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index b121dfd..4b61417 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -109,6 +109,8 @@ class MockServer: self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() # Webroot for statically served content self.webroot: str = webroot + # Variables to pass to pagelets + self.pagelet_variables: Dict[str, str] = dict() # Jinja environment with a single, static template self.jinja_env = jinja2.Environment( loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'}) @@ -168,6 +170,7 @@ def test_pagelet(path: str): str, RequestArguments, Dict[str, Any], + Dict[str, str], Dict[str, str]], Union[bytes, str, Tuple[int, str]]]): @pagelet(path) @@ -175,9 +178,10 @@ def test_pagelet(path: str): path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]): + headers: Dict[str, str], + pagelet_variables: Dict[str, str]): headers['X-Test-Pagelet'] = fun.__name__ - result = fun(method, path, args, session_vars, headers) + result = fun(method, path, args, session_vars, headers, pagelet_variables) return result return testing_wrapper return with_testing_headers diff --git a/matemat/webserver/test/test_config.py b/matemat/webserver/test/test_config.py new file mode 100644 index 0000000..6ddcc83 --- /dev/null +++ b/matemat/webserver/test/test_config.py @@ -0,0 +1,63 @@ + +from unittest import TestCase +from unittest.mock import patch + +from io import StringIO + +from matemat.webserver import parse_config_file + +_EMPTY_CONFIG = '' + +_FULL_CONFIG = ''' +[Matemat] + +Address=fe80::0123:45ff:fe67:89ab +Port = 8080 + +StaticPath =/var/test/static +TemplatePath= /var/test/templates + +[Pagelets] +Name=Matemat + (Unit Test) +UploadDir= /var/test/static/upload +DatabaseFile=/var/test/db/test.db +''' + + +class TestConfig(TestCase): + + def test_parse_config_empty_defualt_values(self): + with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)): + config = parse_config_file('test') + self.assertIn('listen', config) + self.assertIn('port', config) + self.assertIn('staticroot', config) + self.assertIn('templateroot', config) + self.assertIn('pagelet_variables', config) + self.assertEqual('::', config['listen']) + self.assertEqual(80, config['port']) + self.assertEqual('/var/matemat/static', config['staticroot']) + self.assertEqual('/var/matemat/templates', config['templateroot']) + self.assertIsInstance(config['pagelet_variables'], dict) + self.assertEqual(0, len(config['pagelet_variables'])) + + def test_parse_config_full(self): + with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)): + config = parse_config_file('test') + self.assertIn('listen', config) + self.assertIn('port', config) + self.assertIn('staticroot', config) + self.assertIn('templateroot', config) + self.assertIn('pagelet_variables', config) + self.assertIn('Name', config['pagelet_variables']) + self.assertIn('UploadDir', config['pagelet_variables']) + self.assertIn('DatabaseFile', config['pagelet_variables']) + + self.assertEqual('fe80::0123:45ff:fe67:89ab', config['listen']) + self.assertEqual(8080, config['port']) + self.assertEqual('/var/test/static', config['staticroot']) + self.assertEqual('/var/test/templates', config['templateroot']) + self.assertEqual('Matemat\n(Unit Test)', config['pagelet_variables']['Name']) + self.assertEqual('/var/test/static/upload', config['pagelet_variables']['UploadDir']) + self.assertEqual('/var/test/db/test.db', config['pagelet_variables']['DatabaseFile']) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index e105354..a006221 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -12,7 +12,8 @@ def post_test_pagelet(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]): + headers: Dict[str, str], + pagelet_variables: Dict[str, str]): """ Test pagelet that simply prints the parsed arguments as response body. """ diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 7b0b61d..32853de 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -13,7 +13,8 @@ def serve_test_pagelet_str(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers: Dict[str, str], + pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]: headers['Content-Type'] = 'text/plain' return 'serve test pagelet str' @@ -23,7 +24,8 @@ def serve_test_pagelet_bytes(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers: Dict[str, str], + pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]: headers['Content-Type'] = 'application/octet-stream' return b'serve\x80test\xffpagelet\xfebytes' @@ -33,7 +35,8 @@ def serve_test_pagelet_redirect(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers: Dict[str, str], + pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]: return RedirectResponse('/foo/bar') @@ -42,7 +45,8 @@ def serve_test_pagelet_template(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers: Dict[str, str], + pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]: headers['Content-Type'] = 'text/plain' return TemplateResponse('test.txt', what='World') @@ -53,7 +57,8 @@ def serve_test_pagelet_fail(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers: Dict[str, str], + pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]: session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' raise HttpException() diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index 2e64ceb..224d18f 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -13,7 +13,8 @@ def session_test_pagelet(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]): + headers: Dict[str, str], + pagelet_variables: Dict[str, str]): session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' return 'session test' From 758a2eb0182d27d181bf89a18cbb13194c9a9b17 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 01:22:17 +0200 Subject: [PATCH 4/8] Fixed: Expand ~ in config file parsing. --- matemat/webserver/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/matemat/webserver/config.py b/matemat/webserver/config.py index c220adc..53386b5 100644 --- a/matemat/webserver/config.py +++ b/matemat/webserver/config.py @@ -1,6 +1,7 @@ from typing import Any, Dict +import os from configparser import ConfigParser @@ -30,14 +31,14 @@ def parse_config_file(path: str) -> Dict[str, Any]: # Replace the original option transformation by a string constructor to preserve the case of config keys parser.optionxform = str # Read the configuration file - parser.read(path, 'utf-8') + parser.read(os.path.expanduser(path), 'utf-8') # Read values from the [Matemat] section, if present, falling back to default values if 'Matemat' in parser.sections(): config['listen'] = parser['Matemat'].get('Address', config['listen']) config['port'] = int(parser['Matemat'].get('Port', config['port'])) - config['staticroot'] = parser['Matemat'].get('StaticPath', config['staticroot']) - config['templateroot'] = parser['Matemat'].get('TemplatePath', config['templateroot']) + config['staticroot'] = parser['Matemat'].get('StaticPath', os.path.expanduser(config['staticroot'])) + config['templateroot'] = parser['Matemat'].get('TemplatePath', os.path.expanduser(config['templateroot'])) # Read all values from the [Pagelets] section, if present. These values are passed to pagelet functions if 'Pagelets' in parser.sections(): From eff8135acfed1e2d9a5097c75493ff4e53a42717 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 01:43:21 +0200 Subject: [PATCH 5/8] Added configuration file documentation to wiki --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 9634785..e09ae61 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 9634785e5621b324f72598b3cee83bbc45c25d7b +Subproject commit e09ae61a38ec0085c7191a673ada57b4d8217e9f From 150588e6a93541560e3e6767b8129e3325f19f37 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 01:47:15 +0200 Subject: [PATCH 6/8] Fixed a minor issue in the wiki --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index e09ae61..ccaabe8 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit e09ae61a38ec0085c7191a673ada57b4d8217e9f +Subproject commit ccaabe8ac743aac83ce1c1806412e3b6d330b16a From 95c81608c3116fb546a29f8d3509ae1d8dbb745f Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 01:49:24 +0200 Subject: [PATCH 7/8] Fixed a second minor issue in the wiki --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index ccaabe8..4599a33 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit ccaabe8ac743aac83ce1c1806412e3b6d330b16a +Subproject commit 4599a339dbd4a2299a8eca575a7b801686d9fc5a From 8fab13e13aa359fe688f7280671975a7fe6efff1 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 13 Jul 2018 16:02:30 +0200 Subject: [PATCH 8/8] Added support for multiple config files --- doc | 2 +- matemat/__main__.py | 6 +-- matemat/webserver/config.py | 19 +++++-- matemat/webserver/test/test_config.py | 75 ++++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/doc b/doc index 4599a33..d5dc5f7 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 4599a339dbd4a2299a8eca575a7b801686d9fc5a +Subproject commit d5dc5f794ad1b959b9d4dce47eeb6068e5a75115 diff --git a/matemat/__main__.py b/matemat/__main__.py index b461855..8876386 100644 --- a/matemat/__main__.py +++ b/matemat/__main__.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, Iterable, Union import sys @@ -12,9 +12,9 @@ if __name__ == '__main__': from matemat.webserver import MatematWebserver # Use config file name from command line, if present - configfile: str = '/etc/matemat.conf' + configfile: Union[str, Iterable[str]] = '/etc/matemat.conf' if len(sys.argv) > 1: - configfile = sys.argv[1] + configfile = sys.argv[1:] # Parse the config file config: Dict[str, Any] = parse_config_file(configfile) diff --git a/matemat/webserver/config.py b/matemat/webserver/config.py index 53386b5..c2e1c5f 100644 --- a/matemat/webserver/config.py +++ b/matemat/webserver/config.py @@ -1,15 +1,15 @@ -from typing import Any, Dict +from typing import Any, Dict, Iterable, List, Union import os from configparser import ConfigParser -def parse_config_file(path: str) -> Dict[str, Any]: +def parse_config_file(paths: Union[str, Iterable[str]]) -> Dict[str, Any]: """ Parse the configuration file at the given path. - :param path: The config file to parse. + :param paths: The config file(s) to parse. :return: A dictionary containing the parsed configuration. """ # Set up default values @@ -30,8 +30,17 @@ def parse_config_file(path: str) -> Dict[str, Any]: parser: ConfigParser = ConfigParser() # Replace the original option transformation by a string constructor to preserve the case of config keys parser.optionxform = str - # Read the configuration file - parser.read(os.path.expanduser(path), 'utf-8') + # Normalize the input argument (turn a scalar into a list and expand ~ in paths) + files: List[str] = list() + if isinstance(paths, str): + files.append(os.path.expanduser(paths)) + else: + for path in paths: + if not isinstance(path, str): + raise TypeError(f'Not a string: {path}') + files.append(os.path.expanduser(path)) + # Read the configuration files + parser.read(files, 'utf-8') # Read values from the [Matemat] section, if present, falling back to default values if 'Matemat' in parser.sections(): diff --git a/matemat/webserver/test/test_config.py b/matemat/webserver/test/test_config.py index 6ddcc83..2d37635 100644 --- a/matemat/webserver/test/test_config.py +++ b/matemat/webserver/test/test_config.py @@ -1,4 +1,6 @@ +from typing import List + from unittest import TestCase from unittest.mock import patch @@ -24,17 +26,54 @@ UploadDir= /var/test/static/upload DatabaseFile=/var/test/db/test.db ''' +_PARTIAL_CONFIG = ''' +[Matemat] +Port=443 + +[Pagelets] +Name=Matemat (Unit Test 2) +''' + + +class IterOpenMock: + """ + Enable mocking of subsequent open() class for different files. Usage: + + with mock.patch('builtins.open', IterOpenMock(['content 1', 'content 2'])): + ... + with open('foo') as f: + # Reading from f will yield 'content 1' + with open('foo') as f: + # Reading from f will yield 'content 2' + """ + + def __init__(self, files: List[str]): + self.files = files + + def __enter__(self): + return StringIO(self.files[0]) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.files = self.files[1:] + class TestConfig(TestCase): def test_parse_config_empty_defualt_values(self): + """ + Test that default values are set when reading an empty config file. + """ + # Mock the open() function to return an empty config file example with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)): + # The filename is only a placeholder, file content is determined by mocking open config = parse_config_file('test') + # Make sure all mandatory values are present self.assertIn('listen', config) self.assertIn('port', config) self.assertIn('staticroot', config) self.assertIn('templateroot', config) self.assertIn('pagelet_variables', config) + # Make sure all mandatory values are set to their default self.assertEqual('::', config['listen']) self.assertEqual(80, config['port']) self.assertEqual('/var/matemat/static', config['staticroot']) @@ -43,8 +82,14 @@ class TestConfig(TestCase): self.assertEqual(0, len(config['pagelet_variables'])) def test_parse_config_full(self): + """ + Test that all default values are overridden by the values provided in the config file. + """ + # Mock the open() function to return a full config file example with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)): + # The filename is only a placeholder, file content is determined by mocking open config = parse_config_file('test') + # Make sure all mandatory values are present self.assertIn('listen', config) self.assertIn('port', config) self.assertIn('staticroot', config) @@ -53,7 +98,7 @@ class TestConfig(TestCase): self.assertIn('Name', config['pagelet_variables']) self.assertIn('UploadDir', config['pagelet_variables']) self.assertIn('DatabaseFile', config['pagelet_variables']) - + # Make sure all values are set as described in the config file self.assertEqual('fe80::0123:45ff:fe67:89ab', config['listen']) self.assertEqual(8080, config['port']) self.assertEqual('/var/test/static', config['staticroot']) @@ -61,3 +106,31 @@ class TestConfig(TestCase): self.assertEqual('Matemat\n(Unit Test)', config['pagelet_variables']['Name']) self.assertEqual('/var/test/static/upload', config['pagelet_variables']['UploadDir']) self.assertEqual('/var/test/db/test.db', config['pagelet_variables']['DatabaseFile']) + + def test_parse_config_precedence(self): + """ + Test that config items from files with higher precedence replace items with the same key from files with lower + precedence. + """ + # Mock the open() function to return a full config file on the first call, and a partial config file on the + # second call + with patch('builtins.open', return_value=IterOpenMock([_FULL_CONFIG, _PARTIAL_CONFIG])): + # These filenames are only placeholders, file content is determined by mocking open + config = parse_config_file(['full', 'partial']) + # Make sure all mandatory values are present + self.assertIn('listen', config) + self.assertIn('port', config) + self.assertIn('staticroot', config) + self.assertIn('templateroot', config) + self.assertIn('pagelet_variables', config) + self.assertIn('Name', config['pagelet_variables']) + self.assertIn('UploadDir', config['pagelet_variables']) + self.assertIn('DatabaseFile', config['pagelet_variables']) + # Make sure all values are set as described in the config files, values from the partial file take precedence + self.assertEqual('fe80::0123:45ff:fe67:89ab', config['listen']) + self.assertEqual(443, config['port']) + self.assertEqual('/var/test/static', config['staticroot']) + self.assertEqual('/var/test/templates', config['templateroot']) + self.assertEqual('Matemat (Unit Test 2)', config['pagelet_variables']['Name']) + self.assertEqual('/var/test/static/upload', config['pagelet_variables']['UploadDir']) + self.assertEqual('/var/test/db/test.db', config['pagelet_variables']['DatabaseFile'])