Merge branch '7-configuration-file' into 'master'

Configuration File Support

Closes #7

See merge request s3lph/matemat!10

.gitignore

@@ -9,3 +9,4 @@
 
 *.sqlite3
 *.db
+**/matemat.conf

doc

@@ -1 +1 @@
-Subproject commit 9634785e5621b324f72598b3cee83bbc45c25d7b
+Subproject commit d5dc5f794ad1b959b9d4dce47eeb6068e5a75115

matemat/__main__.py

@@ -1,16 +1,23 @@
 
+from typing import Any, Dict, Iterable, Union
+
 import sys
 
+from matemat.webserver import parse_config_file
+
 if __name__ == '__main__':
     # Those imports are actually needed, as they implicitly register pagelets.
     # noinspection PyUnresolvedReferences
     from matemat.webserver.pagelets import *
     from matemat.webserver import MatematWebserver
 
-    # Read HTTP port from command line
+    # Use config file name from command line, if present
-    port: int = 8080
+    configfile: Union[str, Iterable[str]] = '/etc/matemat.conf'
     if len(sys.argv) > 1:
-        port = int(sys.argv[1])
+        configfile = sys.argv[1:]
+
+    # Parse the config file
+    config: Dict[str, Any] = parse_config_file(configfile)
 
     # Start the web server
-    MatematWebserver(port=port).start()
+    MatematWebserver(**config).start()

matemat/webserver/__init__.py

@@ -9,3 +9,4 @@ server will attempt to serve the request with a static resource in a previously
 from .requestargs import RequestArgument, RequestArguments
 from .responses import PageletResponse, RedirectResponse, TemplateResponse
 from .httpd import MatematWebserver, HttpHandler, pagelet
+from .config import parse_config_file

matemat/webserver/config.py

@@ -0,0 +1,57 @@
+
+from typing import Any, Dict, Iterable, List, Union
+
+import os
+from configparser import ConfigParser
+
+
+def parse_config_file(paths: Union[str, Iterable[str]]) -> Dict[str, Any]:
+    """
+    Parse the configuration file at the given path.
+
+    :param paths: The config file(s) to parse.
+    :return: A dictionary containing the parsed configuration.
+    """
+    # Set up default values
+    config: Dict[str, Any] = {
+        # Address to listen on
+        'listen': '::',
+        # TCP port to listen on
+        'port': 80,
+        # Root directory of statically served content
+        'staticroot': '/var/matemat/static',
+        # Root directory of Jinja2 templates
+        'templateroot': '/var/matemat/templates',
+        # Variables passed to pagelets
+        'pagelet_variables': dict()
+    }
+
+    # Initialize the config parser
+    parser: ConfigParser = ConfigParser()
+    # Replace the original option transformation by a string constructor to preserve the case of config keys
+    parser.optionxform = str
+    # Normalize the input argument (turn a scalar into a list and expand ~ in paths)
+    files: List[str] = list()
+    if isinstance(paths, str):
+        files.append(os.path.expanduser(paths))
+    else:
+        for path in paths:
+            if not isinstance(path, str):
+                raise TypeError(f'Not a string: {path}')
+            files.append(os.path.expanduser(path))
+    # Read the configuration files
+    parser.read(files, 'utf-8')
+
+    # Read values from the [Matemat] section, if present, falling back to default values
+    if 'Matemat' in parser.sections():
+        config['listen'] = parser['Matemat'].get('Address', config['listen'])
+        config['port'] = int(parser['Matemat'].get('Port', config['port']))
+        config['staticroot'] = parser['Matemat'].get('StaticPath', os.path.expanduser(config['staticroot']))
+        config['templateroot'] = parser['Matemat'].get('TemplatePath', os.path.expanduser(config['templateroot']))
+
+    # Read all values from the [Pagelets] section, if present.  These values are passed to pagelet functions
+    if 'Pagelets' in parser.sections():
+        for k, v in parser['Pagelets'].items():
+            config['pagelet_variables'][k] = v
+
+    return config

matemat/webserver/httpd.py

@@ -34,7 +34,8 @@ _PAGELET_PATHS: Dict[str, Callable[[str,  # HTTP method (GET, POST, ...)
                                     str,  # Request path
                                     RequestArguments,  # HTTP Request arguments
                                     Dict[str, Any],  # Session vars
-                                    Dict[str, str]],  # Response headers
+                                    Dict[str, str],  # Response headers
+                                    Dict[str, str]],  # Items from the [Pagelets] section in the config file
                                    Union[  # Return type: either a response body, or a redirect
                                        bytes, str,  # Response body: will assign HTTP/1.0 200 OK
                                        PageletResponse,  # A generic response
@@ -56,7 +57,8 @@ def pagelet(path: str):
          path: str,
          args: RequestArguments,
          session_vars: Dict[str, Any],
-         headers: Dict[str, str])
+         headers: Dict[str, str],
+         config: Dict[str, str])
         -> Union[bytes, str, Tuple[int, str]]
 
         method:       The HTTP method (GET, POST) that was used.
@@ -64,6 +66,7 @@ def pagelet(path: str):
         args:         The arguments that were passed with the request (as GET or POST arguments).
         session_vars: The session storage.  May be read from and written to.
         headers:      The dictionary of HTTP response headers.  Add headers you wish to send with the response.
+        config:       The dictionary of variables read from the [Pagelets] section of the configuration file.
         returns:      One of the following:
                         - A HTTP Response body as str or bytes
                         - A PageletResponse class instance: An instance of (a subclass of)
@@ -77,6 +80,7 @@ def pagelet(path: str):
                                     str,
                                     RequestArguments,
                                     Dict[str, Any],
+                                    Dict[str, str],
                                     Dict[str, str]],
                                    Union[
                                        bytes, str,
@@ -102,12 +106,15 @@ class MatematHTTPServer(HTTPServer):
                  handler: Type[BaseHTTPRequestHandler],
                  staticroot: str,
                  templateroot: str,
+                 pagelet_variables: Dict[str, str],
                  bind_and_activate: bool = True) -> None:
         super().__init__(server_address, handler, bind_and_activate)
         # Resolve webroot directory
         self.webroot = os.path.abspath(staticroot)
         # Set up session vars dict
         self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict()
+        # Set up pagelet arguments dict
+        self.pagelet_variables = pagelet_variables
         # Set up the Jinja2 environment
         self.jinja_env: jinja2.Environment = jinja2.Environment(
             loader=jinja2.FileSystemLoader(os.path.abspath(templateroot))
@@ -130,10 +137,11 @@ class MatematWebserver(object):
     """
 
     def __init__(self,
-                 listen: str = '::',
+                 listen: str,
-                 port: int = 80,
+                 port: int,
-                 staticroot: str = './static',
+                 staticroot: str,
-                 templateroot: str = './templates') -> None:
+                 templateroot: str,
+                 pagelet_variables: Dict[str, str]) -> None:
         """
         Instantiate a MatematWebserver.
 
@@ -141,16 +149,14 @@ class MatematWebserver(object):
         :param port: The TCP port to listen on.
         :param staticroot: Path to the static webroot directory.
         :param templateroot: Path to the Jinja2 templates root directory.
+        :param pagelet_variables: Dictionary of variables to pass to pagelet functions.
         """
-        if len(listen) == 0:
-            # Empty string should be interpreted as all addresses
-            listen = '::'
         # IPv4 address detection heuristic
         if ':' not in listen and '.' in listen:
             # Rewrite IPv4 address to IPv6-mapped form
             listen = f'::ffff:{listen}'
         # Create the http server
-        self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot)
+        self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot, pagelet_variables)
 
     def start(self) -> None:
         """
@@ -290,7 +296,12 @@ class HttpHandler(BaseHTTPRequestHandler):
                 'Cache-Control': 'no-cache'
             }
             # Call the pagelet function
-            pagelet_res = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers)
+            pagelet_res = _PAGELET_PATHS[path](method,
+                                               path,
+                                               args,
+                                               self.session_vars,
+                                               headers,
+                                               self.server.pagelet_variables)
             # Parse the pagelet's return value, vielding a HTTP status code and a response body
             hsc, data = self._parse_pagelet_result(pagelet_res, headers)
             # Send the HTTP status code

matemat/webserver/pagelets/login.py

@@ -12,14 +12,15 @@ def login_page(method: str,
                path: str,
                args: RequestArguments,
                session_vars: Dict[str, Any],
-               headers: Dict[str, str])\
+               headers: Dict[str, str],
+               config: Dict[str, str])\
         -> Union[bytes, str, PageletResponse]:
     if 'user' in session_vars:
         return RedirectResponse('/')
     if method == 'GET':
-        return TemplateResponse('login.html')
+        return TemplateResponse('login.html', setupname=config['InstanceName'])
     elif method == 'POST':
-        with MatematDatabase('test.db') as db:
+        with MatematDatabase(config['DatabaseFile']) as db:
             try:
                 user: User = db.login(str(args.username), str(args.password))
             except AuthenticationError:

matemat/webserver/pagelets/logout.py

@@ -9,7 +9,8 @@ def logout(method: str,
            path: str,
            args: RequestArguments,
            session_vars: Dict[str, Any],
-           headers: Dict[str, str])\
+           headers: Dict[str, str],
+           config: Dict[str, str])\
         -> Union[bytes, str, PageletResponse]:
     if 'user' in session_vars:
         del session_vars['user']

matemat/webserver/pagelets/main.py

@@ -11,13 +11,14 @@ def main_page(method: str,
               path: str,
               args: RequestArguments,
               session_vars: Dict[str, Any],
-              headers: Dict[str, str])\
+              headers: Dict[str, str],
+              config: Dict[str, str])\
         -> Union[bytes, str, PageletResponse]:
-    with MatematDatabase('test.db') as db:
+    with MatematDatabase(config['DatabaseFile']) as db:
         if 'user' in session_vars:
             user: User = session_vars['user']
             products = db.list_products()
-            return TemplateResponse('main.html', user=user, list=products)
+            return TemplateResponse('main.html', user=user, list=products, setupname=config['InstanceName'])
         else:
             users = db.list_users()
-            return TemplateResponse('main.html', list=users)
+            return TemplateResponse('main.html', list=users, setupname=config['InstanceName'])

matemat/webserver/pagelets/touchkey.py

@@ -14,14 +14,17 @@ def touchkey_page(method: str,
                   path: str,
                   args: RequestArguments,
                   session_vars: Dict[str, Any],
-                  headers: Dict[str, str])\
+                  headers: Dict[str, str],
+                  config: Dict[str, str])\
         -> Union[bytes, str, PageletResponse]:
     if 'user' in session_vars:
         return RedirectResponse('/')
     if method == 'GET':
-        return TemplateResponse('touchkey.html', username=str(args.username) if 'username' in args else None)
+        return TemplateResponse('touchkey.html',
+                                username=str(args.username) if 'username' in args else None,
+                                setupname=config['InstanceName'])
     elif method == 'POST':
-        with MatematDatabase('test.db') as db:
+        with MatematDatabase(config['DatabaseFile']) as db:
             try:
                 user: User = db.login(str(args.username), touchkey=str(args.touchkey))
             except AuthenticationError:

matemat/webserver/test/abstract_httpd_test.py

@@ -109,6 +109,8 @@ class MockServer:
         self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict()
         # Webroot for statically served content
         self.webroot: str = webroot
+        # Variables to pass to pagelets
+        self.pagelet_variables: Dict[str, str] = dict()
         # Jinja environment with a single, static template
         self.jinja_env = jinja2.Environment(
             loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'})
@@ -168,6 +170,7 @@ def test_pagelet(path: str):
                                             str,
                                             RequestArguments,
                                             Dict[str, Any],
+                                            Dict[str, str],
                                             Dict[str, str]],
                                            Union[bytes, str, Tuple[int, str]]]):
         @pagelet(path)
@@ -175,9 +178,10 @@ def test_pagelet(path: str):
                             path: str,
                             args: RequestArguments,
                             session_vars: Dict[str, Any],
-                            headers: Dict[str, str]):
+                            headers: Dict[str, str],
+                            pagelet_variables: Dict[str, str]):
             headers['X-Test-Pagelet'] = fun.__name__
-            result = fun(method, path, args, session_vars, headers)
+            result = fun(method, path, args, session_vars, headers, pagelet_variables)
             return result
         return testing_wrapper
     return with_testing_headers

matemat/webserver/test/test_config.py

@@ -0,0 +1,136 @@
+
+from typing import List
+
+from unittest import TestCase
+from unittest.mock import patch
+
+from io import StringIO
+
+from matemat.webserver import parse_config_file
+
+_EMPTY_CONFIG = ''
+
+_FULL_CONFIG = '''
+[Matemat]
+
+Address=fe80::0123:45ff:fe67:89ab
+Port = 8080
+
+StaticPath   =/var/test/static
+TemplatePath=   /var/test/templates
+
+[Pagelets]
+Name=Matemat
+  (Unit Test)
+UploadDir= /var/test/static/upload
+DatabaseFile=/var/test/db/test.db
+'''
+
+_PARTIAL_CONFIG = '''
+[Matemat]
+Port=443
+
+[Pagelets]
+Name=Matemat (Unit Test 2)
+'''
+
+
+class IterOpenMock:
+    """
+    Enable mocking of subsequent open() class for different files.  Usage:
+
+    with mock.patch('builtins.open', IterOpenMock(['content 1', 'content 2'])):
+        ...
+            with open('foo') as f:
+                # Reading from f will yield 'content 1'
+            with open('foo') as f:
+                # Reading from f will yield 'content 2'
+    """
+
+    def __init__(self, files: List[str]):
+        self.files = files
+
+    def __enter__(self):
+        return StringIO(self.files[0])
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.files = self.files[1:]
+
+
+class TestConfig(TestCase):
+
+    def test_parse_config_empty_defualt_values(self):
+        """
+        Test that default values are set when reading an empty config file.
+        """
+        # Mock the open() function to return an empty config file example
+        with patch('builtins.open', return_value=StringIO(_EMPTY_CONFIG)):
+            # The filename is only a placeholder, file content is determined by mocking open
+            config = parse_config_file('test')
+        # Make sure all mandatory values are present
+        self.assertIn('listen', config)
+        self.assertIn('port', config)
+        self.assertIn('staticroot', config)
+        self.assertIn('templateroot', config)
+        self.assertIn('pagelet_variables', config)
+        # Make sure all mandatory values are set to their default
+        self.assertEqual('::', config['listen'])
+        self.assertEqual(80, config['port'])
+        self.assertEqual('/var/matemat/static', config['staticroot'])
+        self.assertEqual('/var/matemat/templates', config['templateroot'])
+        self.assertIsInstance(config['pagelet_variables'], dict)
+        self.assertEqual(0, len(config['pagelet_variables']))
+
+    def test_parse_config_full(self):
+        """
+        Test that all default values are overridden by the values provided in the config file.
+        """
+        # Mock the open() function to return a full config file example
+        with patch('builtins.open', return_value=StringIO(_FULL_CONFIG)):
+            # The filename is only a placeholder, file content is determined by mocking open
+            config = parse_config_file('test')
+        # Make sure all mandatory values are present
+        self.assertIn('listen', config)
+        self.assertIn('port', config)
+        self.assertIn('staticroot', config)
+        self.assertIn('templateroot', config)
+        self.assertIn('pagelet_variables', config)
+        self.assertIn('Name', config['pagelet_variables'])
+        self.assertIn('UploadDir', config['pagelet_variables'])
+        self.assertIn('DatabaseFile', config['pagelet_variables'])
+        # Make sure all values are set as described in the config file
+        self.assertEqual('fe80::0123:45ff:fe67:89ab', config['listen'])
+        self.assertEqual(8080, config['port'])
+        self.assertEqual('/var/test/static', config['staticroot'])
+        self.assertEqual('/var/test/templates', config['templateroot'])
+        self.assertEqual('Matemat\n(Unit Test)', config['pagelet_variables']['Name'])
+        self.assertEqual('/var/test/static/upload', config['pagelet_variables']['UploadDir'])
+        self.assertEqual('/var/test/db/test.db', config['pagelet_variables']['DatabaseFile'])
+
+    def test_parse_config_precedence(self):
+        """
+        Test that config items from files with higher precedence replace items with the same key from files with lower
+        precedence.
+        """
+        # Mock the open() function to return a full config file on the first call, and a partial config file on the
+        # second call
+        with patch('builtins.open', return_value=IterOpenMock([_FULL_CONFIG, _PARTIAL_CONFIG])):
+            # These filenames are only placeholders, file content is determined by mocking open
+            config = parse_config_file(['full', 'partial'])
+        # Make sure all mandatory values are present
+        self.assertIn('listen', config)
+        self.assertIn('port', config)
+        self.assertIn('staticroot', config)
+        self.assertIn('templateroot', config)
+        self.assertIn('pagelet_variables', config)
+        self.assertIn('Name', config['pagelet_variables'])
+        self.assertIn('UploadDir', config['pagelet_variables'])
+        self.assertIn('DatabaseFile', config['pagelet_variables'])
+        # Make sure all values are set as described in the config files, values from the partial file take precedence
+        self.assertEqual('fe80::0123:45ff:fe67:89ab', config['listen'])
+        self.assertEqual(443, config['port'])
+        self.assertEqual('/var/test/static', config['staticroot'])
+        self.assertEqual('/var/test/templates', config['templateroot'])
+        self.assertEqual('Matemat (Unit Test 2)', config['pagelet_variables']['Name'])
+        self.assertEqual('/var/test/static/upload', config['pagelet_variables']['UploadDir'])
+        self.assertEqual('/var/test/db/test.db', config['pagelet_variables']['DatabaseFile'])

matemat/webserver/test/test_post.py

@@ -12,7 +12,8 @@ def post_test_pagelet(method: str,
                       path: str,
                       args: RequestArguments,
                       session_vars: Dict[str, Any],
-                      headers: Dict[str, str]):
+                      headers: Dict[str, str],
+                      pagelet_variables: Dict[str, str]):
     """
     Test pagelet that simply prints the parsed arguments as response body.
     """

matemat/webserver/test/test_serve.py

@@ -13,7 +13,8 @@ def serve_test_pagelet_str(method: str,
                            path: str,
                            args: RequestArguments,
                            session_vars: Dict[str, Any],
-                           headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
+                           headers: Dict[str, str],
+                           pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
     headers['Content-Type'] = 'text/plain'
     return 'serve test pagelet str'
 
@@ -23,7 +24,8 @@ def serve_test_pagelet_bytes(method: str,
                              path: str,
                              args: RequestArguments,
                              session_vars: Dict[str, Any],
-                             headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
+                             headers: Dict[str, str],
+                             pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
     headers['Content-Type'] = 'application/octet-stream'
     return b'serve\x80test\xffpagelet\xfebytes'
 
@@ -33,7 +35,8 @@ def serve_test_pagelet_redirect(method: str,
                                 path: str,
                                 args: RequestArguments,
                                 session_vars: Dict[str, Any],
-                                headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
+                                headers: Dict[str, str],
+                                pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
     return RedirectResponse('/foo/bar')
 
 
@@ -42,7 +45,8 @@ def serve_test_pagelet_template(method: str,
                                 path: str,
                                 args: RequestArguments,
                                 session_vars: Dict[str, Any],
-                                headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
+                                headers: Dict[str, str],
+                                pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
     headers['Content-Type'] = 'text/plain'
     return TemplateResponse('test.txt', what='World')
 
@@ -53,7 +57,8 @@ def serve_test_pagelet_fail(method: str,
                             path: str,
                             args: RequestArguments,
                             session_vars: Dict[str, Any],
-                            headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
+                            headers: Dict[str, str],
+                            pagelet_variables: Dict[str, str]) -> Union[bytes, str, PageletResponse]:
     session_vars['test'] = 'hello, world!'
     headers['Content-Type'] = 'text/plain'
     raise HttpException()

matemat/webserver/test/test_session.py

@@ -13,7 +13,8 @@ def session_test_pagelet(method: str,
                          path: str,
                          args: RequestArguments,
                          session_vars: Dict[str, Any],
-                         headers: Dict[str, str]):
+                         headers: Dict[str, str],
+                         pagelet_variables: Dict[str, str]):
     session_vars['test'] = 'hello, world!'
     headers['Content-Type'] = 'text/plain'
     return 'session test'

templates/base.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
     <head>
-        <title>Matemat</title>
+        <title>{{ setupname }}</title>
         <style>
             body {
                 color: #f0f0f0;
@@ -10,7 +10,7 @@
         </style>
     </head>
     <body>
-        <h1>Matemat {{__version__}}</h1>
+        <h1>{{ setupname }}</h1>
         {% block main %}
         {% endblock %}
     </body>