From 700af6883fa6b098d7766cf5259e8e3f7d9b6ae5 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 12 Jun 2018 21:45:50 +0200 Subject: [PATCH 01/46] Initial commit for webserver code. Still needs a lot of documentation, and even more, test coverage. --- .gitignore | 2 +- matemat/__main__.py | 16 ++ matemat/db/facade.py | 4 +- matemat/webserver/__init__.py | 2 + matemat/webserver/httpd.py | 201 ++++++++++++++++++++++--- matemat/webserver/pagelets/__init__.py | 5 + matemat/webserver/pagelets/login.py | 50 ++++++ matemat/webserver/pagelets/logout.py | 12 ++ matemat/webserver/pagelets/main.py | 53 +++++++ matemat/webserver/pagelets/touchkey.py | 48 ++++++ 10 files changed, 373 insertions(+), 20 deletions(-) create mode 100644 matemat/__main__.py create mode 100644 matemat/webserver/pagelets/__init__.py create mode 100644 matemat/webserver/pagelets/login.py create mode 100644 matemat/webserver/pagelets/logout.py create mode 100644 matemat/webserver/pagelets/main.py create mode 100644 matemat/webserver/pagelets/touchkey.py diff --git a/.gitignore b/.gitignore index 95dc460..cca262e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ **/.mypy_cache/ *.sqlite3 -*.db \ No newline at end of file +*.db diff --git a/matemat/__main__.py b/matemat/__main__.py new file mode 100644 index 0000000..819ae2b --- /dev/null +++ b/matemat/__main__.py @@ -0,0 +1,16 @@ + +import sys + +if __name__ == '__main__': + # Those imports are actually needed, as they implicitly register pagelets. + # noinspection PyUnresolvedReferences + from matemat.webserver.pagelets import * + from matemat.webserver import MatematWebserver + + # Read HTTP port from command line + port: int = 8080 + if len(sys.argv) > 1: + port = int(sys.argv[1]) + + # Start the web server + MatematWebserver(port).start() diff --git a/matemat/db/facade.py b/matemat/db/facade.py index f91bf4a..027c62f 100644 --- a/matemat/db/facade.py +++ b/matemat/db/facade.py @@ -140,8 +140,10 @@ class MatematDatabase(object): user_id, username, email, pwhash, tkhash, admin, member = row if password is not None and not bcrypt.checkpw(password.encode('utf-8'), pwhash): raise AuthenticationError('Password mismatch') - elif touchkey is not None and not bcrypt.checkpw(touchkey.encode('utf-8'), tkhash): + elif touchkey is not None and tkhash is not None and not bcrypt.checkpw(touchkey.encode('utf-8'), tkhash): raise AuthenticationError('Touchkey mismatch') + elif touchkey is not None and tkhash is None: + raise AuthenticationError('Touchkey not set') return User(user_id, username, email, admin, member) def change_password(self, user: User, oldpass: str, newpass: str, verify_password: bool = True) -> None: diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index e69de29..bb51b73 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -0,0 +1,2 @@ + +from .httpd import MatematWebserver, HttpHandler, pagelet diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 2ae1685..ff6c7d4 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,6 +1,13 @@ -from typing import Tuple, Dict +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import traceback + +import os +import socket +import mimetypes +import urllib.parse +from socketserver import TCPServer from http.server import HTTPServer, BaseHTTPRequestHandler from http.cookies import SimpleCookie from uuid import uuid4 @@ -9,10 +16,52 @@ from datetime import datetime, timedelta from matemat import __version__ as matemat_version +# Enable IPv6 support (with implicit DualStack). +TCPServer.address_family = socket.AF_INET6 + + +# Dictionary to hold registered pagelet paths and their handler functions. +_PAGELET_PATHS: Dict[str, Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], + Tuple[int, Union[bytes, str]]]] = dict() + + +def pagelet(path: str): + """ + Annotate a function to act as a pagelet (part of a website). The function will be called if a request is made to + the path specified as argument to the annotation. + + The function must have the following signature: + + (method: str, path: str, args: Dict[str, Union[str, List[str]], session_vars: Dict[str, Any], + headers: Dict[str, str]) -> (int, Optional[Union[str, bytes]]) + + method: The HTTP method (GET, POST) that was used. + path: The path that was requested. + args: The arguments that were passed with the request (as GET or POST arguments). + session_vars: The session storage. May be read from and written to. + headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. + returns: A tuple consisting of the HTTP status code (as an int) and the response body (as str or bytes, + may be None) + + :param path: The path to register the function for. + """ + def http_handler(fun: Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], + Tuple[int, Union[bytes, str]]]): + # Add the function to the dict of pagelets. + _PAGELET_PATHS[path] = fun + # Don't change the function itself at all. + return fun + # Return the inner function (Python requires a "real" function annotation to not have any arguments except + # the function itself). + return http_handler + + class MatematWebserver(object): - def __init__(self) -> None: - self._httpd = HTTPServer(('', 8080), HttpHandler) + def __init__(self, port: int = 80, webroot: str = './webroot') -> None: + self._httpd = HTTPServer(('::', port), HttpHandler) + self._httpd.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + self._httpd.webroot = os.path.abspath(webroot) def start(self) -> None: self._httpd.serve_forever() @@ -22,8 +71,6 @@ class HttpHandler(BaseHTTPRequestHandler): def __init__(self, request: bytes, client_address: Tuple[str, int], server: HTTPServer) -> None: super().__init__(request, client_address, server) - self._session_vars: Dict[str, Tuple[datetime, Dict[str, object]]] = dict() - print(self._session_vars) @property def server_version(self) -> str: @@ -34,28 +81,146 @@ class HttpHandler(BaseHTTPRequestHandler): cookiestring = '\n'.join(self.headers.get_all('Cookie', failobj=[])) cookie = SimpleCookie() cookie.load(cookiestring) - session_id = cookie['matemat_session_id'] if 'matemat_session_id' in cookie else str(uuid4()) + session_id = str(cookie['matemat_session_id'].value) if 'matemat_session_id' in cookie else None + if session_id is None or session_id not in self.server.session_vars: + session_id = str(uuid4()) - if session_id in self._session_vars and self._session_vars[session_id][0] < now: + if session_id in self.server.session_vars and self.server.session_vars[session_id][0] < now: self.end_session(session_id) - raise TimeoutError('Session timed out') - elif session_id not in self._session_vars: - self._session_vars[session_id] = (now + timedelta(hours=1)), dict() - return session_id, now + raise TimeoutError('Session timed out.') + elif session_id not in self.server.session_vars: + self.server.session_vars[session_id] = (now + timedelta(seconds=10)), dict() + return session_id, self.server.session_vars[session_id][0] def end_session(self, session_id: str) -> None: - if session_id in self._session_vars: - del self._session_vars[session_id] + if session_id in self.server.session_vars: + del self.server.session_vars[session_id] - def do_GET(self) -> None: + def _handle(self, method: str, path: str, args: Dict[str, str]) -> None: try: session_id, timeout = self.start_session() except TimeoutError: + self.send_error(599, 'Session Timed Out', 'Session Timed Out.') self.send_header('Set-Cookie', 'matemat_session_id=; expires=Thu, 01 Jan 1970 00:00:00 GMT') - self.send_error(599, 'Session Timed Out.', 'Please log in again.') + self.end_headers() return - self.send_response(200, 'Welcome!') + self.session_id: str = session_id + if path in _PAGELET_PATHS: + headers: Dict[str, str] = { + 'Content-Type': 'text/html', + 'Cache-Control': 'no-cache' + } + hsc, data = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) + if data is None: + data = bytes() + if isinstance(data, str): + data = data.encode('utf-8') + self.send_response(hsc) + expires = timeout.strftime("%a, %d %b %Y %H:%M:%S GMT") + self.send_header('Set-Cookie', + f'matemat_session_id={session_id}; expires={expires}') + headers['Content-Length'] = str(len(data)) + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(data) + else: + if method != 'GET': + self.send_error(405) + self.end_headers() + return + filepath: str = os.path.abspath(os.path.join(self.server.webroot, path[1:])) + if os.path.commonpath([filepath, self.server.webroot]) == self.server.webroot and os.path.exists(filepath): + with open(filepath, 'rb') as f: + data = f.read() + self.send_response(200) + mimetype, _ = mimetypes.guess_type(filepath) + if mimetype is not None: + self.send_header('Content-Type', mimetype) + self.end_headers() + if method == 'GET': + self.wfile.write(data) + else: + self.send_response(404) + self.end_headers() -if __name__ == '__main__': - MatematWebserver().start() + @staticmethod + def _parse_args(request: str, postbody: Optional[str] = None) -> Tuple[str, Dict[str, Union[str, List[str]]]]: + """ + Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded form, parse the + arguments and return them as a dictionary. + If a key is used both in GET and in POST, the POST value takes precedence, and the GET value is discarded. + :param request: The request string to parse. + :param postbody: The POST body to parse, defaults to None. + :return: A tuple consisting of the base path and a dictionary with the parsed key/value pairs. + """ + # Parse the request "URL" (i.e. only the path). + tokens = urllib.parse.urlparse(request) + # Parse the GET arguments. + args = urllib.parse.parse_qs(tokens.query) + + if postbody is not None: + # Parse the POST body. + postargs = urllib.parse.parse_qs(postbody) + # Write all POST values into the dict, overriding potential duplicates from GET. + for k, v in postargs.items(): + args[k] = v + # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values. + for k, v in args.items(): + if len(v) == 1: + args[k] = v[0] + # Return the path and the parsed arguments. + return tokens.path, args + + # noinspection PyPep8Naming + def do_GET(self) -> None: + try: + path, args = self._parse_args(self.path) + self._handle('GET', path, args) + except PermissionError as e: + self.send_error(403, 'Forbidden') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + except ValueError as e: + self.send_header(400, 'Bad Request') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + except BaseException as e: + self.send_error(500, 'Internal Server Error') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + + # noinspection PyPep8Naming + def do_POST(self) -> None: + try: + clen: str = self.headers.get('Content-Length', failobj='0') + ctype: str = self.headers.get('Content-Type', failobj='application/octet-stream') + post = '' + if ctype == 'application/x-www-form-urlencoded': + post: str = self.rfile.read(int(clen)).decode('utf-8') + + path, args = self._parse_args(self.path, postbody=post) + self._handle('POST', path, args) + except PermissionError as e: + self.send_error(403, 'Forbidden') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + except ValueError as e: + self.send_header(400, 'Bad Request') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + except BaseException as e: + self.send_error(500, 'Internal Server Error') + self.end_headers() + print(type(e)) + traceback.print_tb(e.__traceback__) + + @property + def session_vars(self) -> Dict[str, Any]: + return self.server.session_vars[self.session_id][1] diff --git a/matemat/webserver/pagelets/__init__.py b/matemat/webserver/pagelets/__init__.py new file mode 100644 index 0000000..dabd91d --- /dev/null +++ b/matemat/webserver/pagelets/__init__.py @@ -0,0 +1,5 @@ + +from .main import main_page +from .login import login_page +from .logout import logout +from .touchkey import touchkey_page diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py new file mode 100644 index 0000000..876fd71 --- /dev/null +++ b/matemat/webserver/pagelets/login.py @@ -0,0 +1,50 @@ + +from typing import Any, Dict + +from matemat.exceptions import AuthenticationError +from matemat.webserver import pagelet +from matemat.primitives import User +from matemat.db import MatematDatabase + + +@pagelet('/login') +def login_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): + if 'user' in session_vars: + headers['Location'] = '/' + return 301, None + if method == 'GET': + data = ''' + + + + Matemat + + + +

Matemat

+ {msg} +
+ Username:
+ Password:
+ +
+ + + ''' + return 200, data.format(msg=args['msg'] if 'msg' in args else '') + elif method == 'POST': + print(args) + with MatematDatabase('test.db') as db: + try: + user: User = db.login(args['username'], args['password']) + except AuthenticationError: + headers['Location'] = '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' + return 301, bytes() + session_vars['user'] = user + headers['Location'] = '/' + return 301, bytes() diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py new file mode 100644 index 0000000..86095b0 --- /dev/null +++ b/matemat/webserver/pagelets/logout.py @@ -0,0 +1,12 @@ + +from typing import Any, Dict + +from matemat.webserver import pagelet + + +@pagelet('/logout') +def logout(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): + if 'user' in session_vars: + del session_vars['user'] + headers['Location'] = '/' + return 301, None diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py new file mode 100644 index 0000000..2ead15d --- /dev/null +++ b/matemat/webserver/pagelets/main.py @@ -0,0 +1,53 @@ + +from typing import Any, Dict, Optional, Tuple, Union + +from matemat.webserver import MatematWebserver, pagelet +from matemat.primitives import User +from matemat.db import MatematDatabase + + +@pagelet('/') +def main_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str])\ + -> Tuple[int, Optional[Union[str, bytes]]]: + data = ''' + + + + Matemat + + + +

Matemat

+ {user} +
    + {list} +
+ + + ''' + try: + with MatematDatabase('test.db') as db: + if 'user' in session_vars: + user: User = session_vars['user'] + products = db.list_products() + plist = '\n'.join([f'
  • {p.name} ' + + f'{p.price_member//100 if user.is_member else p.price_non_member//100}' + + f'.{p.price_member%100 if user.is_member else p.price_non_member%100}' + for p in products]) + uname = f'{user.name} (Logout)' + data = data.format(user=uname, list=plist) + else: + users = db.list_users() + ulist = '\n'.join([f'
  • {u.name}' for u in users]) + ulist = ulist + '
  • Password login' + data = data.format(user='', list=ulist) + return 200, data + except BaseException as e: + import traceback + traceback.print_tb(e.__traceback__) + return 500, None diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py new file mode 100644 index 0000000..fd99fea --- /dev/null +++ b/matemat/webserver/pagelets/touchkey.py @@ -0,0 +1,48 @@ + +from typing import Any, Dict + +from matemat.exceptions import AuthenticationError +from matemat.webserver import pagelet +from matemat.primitives import User +from matemat.db import MatematDatabase + + +@pagelet('/touchkey') +def touchkey_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): + if 'user' in session_vars: + headers['Location'] = '/' + return 301, bytes() + if method == 'GET': + data = ''' + + + + Matemat + + + +

    Matemat

    +
    +
    + Touchkey:
    + +
    + + + ''' + return 200, data.format(username=args['username'] if 'username' in args else '') + elif method == 'POST': + with MatematDatabase('test.db') as db: + try: + user: User = db.login(args['username'], touchkey=args['touchkey']) + except AuthenticationError: + headers['Location'] = f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' + return 301, bytes() + session_vars['user'] = user + headers['Location'] = '/' + return 301, None From 97b8d8b05480eedd686cf6d84b1afe6753e79a54 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 13 Jun 2018 00:00:41 +0200 Subject: [PATCH 02/46] Added lots of code documentation to the httpd module. --- matemat/__main__.py | 2 +- matemat/webserver/httpd.py | 171 ++++++++++++++++++++++++++++++------- 2 files changed, 142 insertions(+), 31 deletions(-) diff --git a/matemat/__main__.py b/matemat/__main__.py index 819ae2b..9654b64 100644 --- a/matemat/__main__.py +++ b/matemat/__main__.py @@ -13,4 +13,4 @@ if __name__ == '__main__': port = int(sys.argv[1]) # Start the web server - MatematWebserver(port).start() + MatematWebserver(port=port).start() diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index ff6c7d4..e776151 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -16,15 +16,18 @@ from datetime import datetime, timedelta from matemat import __version__ as matemat_version -# Enable IPv6 support (with implicit DualStack). +# Enable IPv6 support (with implicit DualStack) TCPServer.address_family = socket.AF_INET6 -# Dictionary to hold registered pagelet paths and their handler functions. +# Dictionary to hold registered pagelet paths and their handler functions _PAGELET_PATHS: Dict[str, Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], Tuple[int, Union[bytes, str]]]] = dict() +_SESSION_TIMEOUT: int = 3600 + + def pagelet(path: str): """ Annotate a function to act as a pagelet (part of a website). The function will be called if a request is made to @@ -47,27 +50,66 @@ def pagelet(path: str): """ def http_handler(fun: Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], Tuple[int, Union[bytes, str]]]): - # Add the function to the dict of pagelets. + # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun - # Don't change the function itself at all. + # Don't change the function itself at all return fun # Return the inner function (Python requires a "real" function annotation to not have any arguments except - # the function itself). + # the function itself) return http_handler class MatematWebserver(object): + """ + Then main webserver class, internally uses Python's http.server. - def __init__(self, port: int = 80, webroot: str = './webroot') -> None: - self._httpd = HTTPServer(('::', port), HttpHandler) + The server will serve a pagelet, if one is defined for a request path, else it will attempt to serve a static + resource from the webroot. + + Usage: + + # Listen on all interfaces on port 80 (dual-stack IPv6/IPv4) + server = MatematWebserver('::', 80, webroot='/var/www/matemat') + # Start the server. This call blocks while the server is running. + server.start() + """ + + def __init__(self, listen: str = '::', port: int = 80, webroot: str = './webroot') -> None: + """ + Instantiate a MatematWebserver. + + :param listen: The IPv4 or IPv6 address to listen on + :param port: The TCP port to listen on + :param webroot: Path to the webroot directory + """ + if len(listen) == 0: + # Empty string should be interpreted as all addresses + listen = '::' + # IPv4 address detection heuristic + if ':' not in listen and '.' in listen: + # Rewrite IPv4 address to IPv6-mapped form + listen = f'::ffff:{listen}' + # Create the http server + self._httpd = HTTPServer((listen, port), HttpHandler) + # Set up session vars dict self._httpd.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + # Resolve webroot directory self._httpd.webroot = os.path.abspath(webroot) def start(self) -> None: + """ + Start the web server. This call blocks while the server is running. + """ self._httpd.serve_forever() class HttpHandler(BaseHTTPRequestHandler): + """ + HTTP Request handler. + + This class parses HTTP requests, and calls the appropriate pagelets, or fetches a static resource from the webroot + directory. + """ def __init__(self, request: bytes, client_address: Tuple[str, int], server: HTTPServer) -> None: super().__init__(request, client_address, server) @@ -76,29 +118,58 @@ class HttpHandler(BaseHTTPRequestHandler): def server_version(self) -> str: return f'matemat/{matemat_version}' - def start_session(self) -> Tuple[str, datetime]: + def _start_session(self) -> Tuple[str, datetime]: + """ + Start a new session, or resume the session identified by the session cookie sent in the HTTP request. + + :return: A tuple consisting of the session ID (a UUID string), and the session timeout date. + """ + # Reference date for session timeout now = datetime.utcnow() + # Parse cookies sent by the client cookiestring = '\n'.join(self.headers.get_all('Cookie', failobj=[])) cookie = SimpleCookie() cookie.load(cookiestring) + # Read the client's session ID, if any session_id = str(cookie['matemat_session_id'].value) if 'matemat_session_id' in cookie else None + # If there is no active session, create a new session ID if session_id is None or session_id not in self.server.session_vars: session_id = str(uuid4()) + # Check for session timeout if session_id in self.server.session_vars and self.server.session_vars[session_id][0] < now: - self.end_session(session_id) + self._end_session(session_id) raise TimeoutError('Session timed out.') - elif session_id not in self.server.session_vars: - self.server.session_vars[session_id] = (now + timedelta(seconds=10)), dict() + # Update or initialize the session timeout + if session_id not in self.server.session_vars: + self.server.session_vars[session_id] = (now + timedelta(seconds=_SESSION_TIMEOUT)), dict() + else: + self.server.session_vars[session_id] =\ + (now + timedelta(seconds=_SESSION_TIMEOUT), self.server.session_vars[session_id][1]) + # Return the session ID and timeout return session_id, self.server.session_vars[session_id][0] - def end_session(self, session_id: str) -> None: + def _end_session(self, session_id: str) -> None: + """ + Destroy a session identified by the session ID. + + :param session_id: ID of the session to destroy. + """ if session_id in self.server.session_vars: del self.server.session_vars[session_id] def _handle(self, method: str, path: str, args: Dict[str, str]) -> None: + """ + Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. + + :param method: The HTTP request method (GET, POST). + :param path: The request path without GET arguments. + :param args: Arguments sent with the request. This includes GET and POST arguments, where the POST arguments + take precedence. + """ + # Start or resume a session; report an error on session timeout try: - session_id, timeout = self.start_session() + session_id, timeout = self._start_session() except TimeoutError: self.send_error(599, 'Session Timed Out', 'Session Timed Out.') self.send_header('Set-Cookie', 'matemat_session_id=; expires=Thu, 01 Jan 1970 00:00:00 GMT') @@ -106,42 +177,63 @@ class HttpHandler(BaseHTTPRequestHandler): return self.session_id: str = session_id + # Call a pagelet function, if one is registered for the requested path if path in _PAGELET_PATHS: + # Prepare some headers. Those can still be overwritten by the pagelet headers: Dict[str, str] = { 'Content-Type': 'text/html', 'Cache-Control': 'no-cache' } + # Call the pagelet function hsc, data = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) + # The pagelet may return None as data as a shorthand for an empty response if data is None: data = bytes() + # If the pagelet returns a Python str, convert it to an UTF-8 encoded bytes object if isinstance(data, str): data = data.encode('utf-8') + # Send the HTTP status code self.send_response(hsc) + # Format the session cookie timeout string and send the session cookie header expires = timeout.strftime("%a, %d %b %Y %H:%M:%S GMT") self.send_header('Set-Cookie', f'matemat_session_id={session_id}; expires={expires}') + # Compute the body length and add the appropriate header headers['Content-Length'] = str(len(data)) + # Send all headers set by the pagelet for name, value in headers.items(): self.send_header(name, value) + # End the header section and write the body self.end_headers() self.wfile.write(data) else: + # No pagelet function for this path, try a static serve instead + # Only HTTP GET is allowed, else reply with a 'Method Not Allowed' header if method != 'GET': self.send_error(405) self.end_headers() return + # Create the absolute resource path, resolving '..' filepath: str = os.path.abspath(os.path.join(self.server.webroot, path[1:])) + # Make sure the file is actually inside the webroot directory and that it exists if os.path.commonpath([filepath, self.server.webroot]) == self.server.webroot and os.path.exists(filepath): + # Open and read the file with open(filepath, 'rb') as f: data = f.read() + # File read successfully, send 'OK' header self.send_response(200) + # TODO: Guess the MIME type. Unfortunately this call solely relies on the file extension, not ideal? mimetype, _ = mimetypes.guess_type(filepath) - if mimetype is not None: - self.send_header('Content-Type', mimetype) + # Fall back to octet-stream type, if unknown + if mimetype is None: + mimetype = 'application/octet-stream' + # Send content type header + self.send_header('Content-Type', mimetype) self.end_headers() - if method == 'GET': - self.wfile.write(data) + # Send the requested resource as response body + self.wfile.write(data) else: + # File does not exist or path points outside the webroot directory self.send_response(404) self.end_headers() @@ -150,77 +242,96 @@ class HttpHandler(BaseHTTPRequestHandler): """ Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded form, parse the arguments and return them as a dictionary. + If a key is used both in GET and in POST, the POST value takes precedence, and the GET value is discarded. + :param request: The request string to parse. :param postbody: The POST body to parse, defaults to None. :return: A tuple consisting of the base path and a dictionary with the parsed key/value pairs. """ - # Parse the request "URL" (i.e. only the path). + # Parse the request "URL" (i.e. only the path) tokens = urllib.parse.urlparse(request) - # Parse the GET arguments. + # Parse the GET arguments args = urllib.parse.parse_qs(tokens.query) if postbody is not None: - # Parse the POST body. + # Parse the POST body postargs = urllib.parse.parse_qs(postbody) - # Write all POST values into the dict, overriding potential duplicates from GET. + # Write all POST values into the dict, overriding potential duplicates from GET for k, v in postargs.items(): args[k] = v - # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values. + # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values for k, v in args.items(): if len(v) == 1: args[k] = v[0] - # Return the path and the parsed arguments. + # Return the path and the parsed arguments return tokens.path, args # noinspection PyPep8Naming def do_GET(self) -> None: + """ + Called by BasicHTTPRequestHandler for GET requests. + """ try: + # Parse the request and hand it to the handle function path, args = self._parse_args(self.path) self._handle('GET', path, args) + # Special handling for some errors except PermissionError as e: self.send_error(403, 'Forbidden') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) except ValueError as e: self.send_header(400, 'Bad Request') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) except BaseException as e: + # Generic error handling self.send_error(500, 'Internal Server Error') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) # noinspection PyPep8Naming def do_POST(self) -> None: + """ + Called by BasicHTTPRequestHandler for POST requests. + """ try: + # Read the POST body, if it exists, and its MIME type is application/x-www-form-urlencoded clen: str = self.headers.get('Content-Length', failobj='0') ctype: str = self.headers.get('Content-Type', failobj='application/octet-stream') post = '' if ctype == 'application/x-www-form-urlencoded': post: str = self.rfile.read(int(clen)).decode('utf-8') - + # Parse the request and hand it to the handle function path, args = self._parse_args(self.path, postbody=post) self._handle('POST', path, args) + # Special handling for some errors except PermissionError as e: self.send_error(403, 'Forbidden') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) except ValueError as e: self.send_header(400, 'Bad Request') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) except BaseException as e: + # Generic error handling self.send_error(500, 'Internal Server Error') self.end_headers() - print(type(e)) + print(e) traceback.print_tb(e.__traceback__) @property def session_vars(self) -> Dict[str, Any]: + """ + Get the session variables for the current session. + + :return: Dictionary of named session variables. + """ return self.server.session_vars[self.session_id][1] From 1d995f24624211ee87b9b6719c922a14f66064a6 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 13 Jun 2018 00:52:47 +0200 Subject: [PATCH 03/46] Added some more documentation, especially package docs. --- README.md | 31 +++++++++++++++++++++----- matemat/db/__init__.py | 3 +++ matemat/db/test/test_facade.py | 1 + matemat/db/test/test_wrapper.py | 1 + matemat/exceptions/__init__.py | 3 +++ matemat/primitives/__init__.py | 3 +++ matemat/webserver/__init__.py | 7 ++++++ matemat/webserver/httpd.py | 3 ++- matemat/webserver/pagelets/__init__.py | 5 +++++ 9 files changed, 51 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8082659..80b8d89 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,22 @@ -# matemat +# Matemat -[![pipeline status](https://gitlab.com/s3lph/matemat/badges/master/pipeline.svg)](https://gitlab.com/s3lph/matemat/commits/master) -[![coverage report](https://gitlab.com/s3lph/matemat/badges/master/coverage.svg)](https://gitlab.com/s3lph/matemat/commits/master) +[![pipeline status](https://gitlab.com/s3lph/matemat/badges/master/pipeline.svg)][master] +[![coverage report](https://gitlab.com/s3lph/matemat/badges/master/coverage.svg)][master] + +A web service for automated stock-keeping of a soda machine written in Python. +It provides a touch-input-friendly user interface (as most input happens through the +soda machine's touch screen). + +This project intends to provide a well-tested and maintainable alternative to +[TODO][todo] (discontinued). + +## Further Documentation + +[Wiki][wiki] ## Dependencies -- Python 3.6 +- Python 3 (>=3.6) - Python dependencies: - apsw - bcrypt @@ -16,6 +27,16 @@ python -m matemat ``` +## Contributors + +- s3lph +- SPiNNiX + ## License -[MIT License](https://gitlab.com/s3lph/matemat/blob/master/LICENSE) +[MIT License][mit-license] + + +[mit-license]: https://gitlab.com/s3lph/matemat/blob/master/LICENSE +[master]: https://gitlab.com/s3lph/matemat/commits/master +[wiki]: https://gitlab.com/s3lph/matemat/wiki \ No newline at end of file diff --git a/matemat/db/__init__.py b/matemat/db/__init__.py index 93b6c8c..4f80a25 100644 --- a/matemat/db/__init__.py +++ b/matemat/db/__init__.py @@ -1,3 +1,6 @@ +""" +This package provides a developer-friendly API to the SQLite3 database backend of the Matemat software. +""" from .wrapper import DatabaseWrapper from .facade import MatematDatabase diff --git a/matemat/db/test/test_facade.py b/matemat/db/test/test_facade.py index c266feb..d7236b6 100644 --- a/matemat/db/test/test_facade.py +++ b/matemat/db/test/test_facade.py @@ -10,6 +10,7 @@ from matemat.exceptions import AuthenticationError, DatabaseConsistencyError class DatabaseTest(unittest.TestCase): def setUp(self) -> None: + # Create an in-memory database for testing self.db = MatematDatabase(':memory:') def test_create_user(self) -> None: diff --git a/matemat/db/test/test_wrapper.py b/matemat/db/test/test_wrapper.py index 152066a..dafff65 100644 --- a/matemat/db/test/test_wrapper.py +++ b/matemat/db/test/test_wrapper.py @@ -7,6 +7,7 @@ from matemat.db import DatabaseWrapper class DatabaseTest(unittest.TestCase): def setUp(self) -> None: + # Create an in-memory database for testing self.db = DatabaseWrapper(':memory:') def test_create_schema(self) -> None: diff --git a/matemat/exceptions/__init__.py b/matemat/exceptions/__init__.py index cacbbb9..ebae437 100644 --- a/matemat/exceptions/__init__.py +++ b/matemat/exceptions/__init__.py @@ -1,3 +1,6 @@ +""" +This package provides custom exception classes used in the Matemat codebase. +""" from .AuthenticatonError import AuthenticationError from .DatabaseConsistencyError import DatabaseConsistencyError diff --git a/matemat/primitives/__init__.py b/matemat/primitives/__init__.py index f380d86..a18a142 100644 --- a/matemat/primitives/__init__.py +++ b/matemat/primitives/__init__.py @@ -1,3 +1,6 @@ +""" +This package provides the 'primitive types' the Matemat software deals with - namely users and products. +""" from .User import User from .Product import Product diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index bb51b73..f4d86f3 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -1,2 +1,9 @@ +""" +The Matemat Webserver. + +This package provides the webserver for the Matemat software. It uses Python's http.server and extends it with an event +API that can be used by 'pagelets' - single pages of a web service. If a request cannot be handled by a pagelet, the +server will attempt to serve the request with a static resource in a previously configured webroot directory. +""" from .httpd import MatematWebserver, HttpHandler, pagelet diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index e776151..20af55c 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta from matemat import __version__ as matemat_version -# Enable IPv6 support (with implicit DualStack) +# Enable IPv6 support (IPv6/IPv4 dual-stack support should be implicitly enabled) TCPServer.address_family = socket.AF_INET6 @@ -25,6 +25,7 @@ _PAGELET_PATHS: Dict[str, Callable[[str, str, Dict[str, str], Dict[str, Any], Di Tuple[int, Union[bytes, str]]]] = dict() +# Inactivity timeout for client sessions _SESSION_TIMEOUT: int = 3600 diff --git a/matemat/webserver/pagelets/__init__.py b/matemat/webserver/pagelets/__init__.py index dabd91d..9b926d6 100644 --- a/matemat/webserver/pagelets/__init__.py +++ b/matemat/webserver/pagelets/__init__.py @@ -1,3 +1,8 @@ +""" +This package contains the pagelet functions served by the Matemat software. + +A new pagelet function must be imported here to be automatically loaded when the server is started. +""" from .main import main_page from .login import login_page From 1314699d11324995512c873454b1aa6de9a3233c Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 13 Jun 2018 01:38:37 +0200 Subject: [PATCH 04/46] README: Fixed wiki URL. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80b8d89..f162e95 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,4 @@ python -m matemat [mit-license]: https://gitlab.com/s3lph/matemat/blob/master/LICENSE [master]: https://gitlab.com/s3lph/matemat/commits/master -[wiki]: https://gitlab.com/s3lph/matemat/wiki \ No newline at end of file +[wiki]: https://gitlab.com/s3lph/matemat/wikis/home \ No newline at end of file From 8547dd76ac157e6b8bf5ede197210b3e72b7fd43 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 13 Jun 2018 01:46:30 +0200 Subject: [PATCH 05/46] Added wiki repo as a git submodule in /doc. --- .gitmodules | 3 +++ doc | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 doc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..55531b0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "doc"] + path = doc + url = gitlab.com:s3lph/matemat.wiki.git diff --git a/doc b/doc new file mode 160000 index 0000000..8f85927 --- /dev/null +++ b/doc @@ -0,0 +1 @@ +Subproject commit 8f859277827574f80e392d818c25dcf24afaf5b0 From 23132377737d87a1d679093fe30d83b3e382ddde Mon Sep 17 00:00:00 2001 From: s3lph Date: Thu, 14 Jun 2018 01:13:50 +0200 Subject: [PATCH 06/46] Implemented the first webserver unit test. --- matemat/webserver/httpd.py | 25 ++- matemat/webserver/test/__init__.py | 0 matemat/webserver/test/abstract_httpd_test.py | 163 ++++++++++++++++++ matemat/webserver/test/test_session.py | 53 ++++++ 4 files changed, 232 insertions(+), 9 deletions(-) create mode 100644 matemat/webserver/test/__init__.py create mode 100644 matemat/webserver/test/abstract_httpd_test.py create mode 100644 matemat/webserver/test/test_session.py diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 20af55c..2f3f3cb 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -16,8 +16,15 @@ from datetime import datetime, timedelta from matemat import __version__ as matemat_version +# +# Python internal class hacks +# + # Enable IPv6 support (IPv6/IPv4 dual-stack support should be implicitly enabled) TCPServer.address_family = socket.AF_INET6 +# Redirect internal logging to somewhere else, or, for now, silently discard (TODO: logger will come later) +BaseHTTPRequestHandler.log_request = lambda self, code='-', size='-': None +BaseHTTPRequestHandler.log_error = lambda self, fstring='', *args: None # Dictionary to hold registered pagelet paths and their handler functions @@ -159,7 +166,7 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] - def _handle(self, method: str, path: str, args: Dict[str, str]) -> None: + def _handle(self, method: str, path: str, args: Dict[str, Union[str, List[str]]]) -> None: """ Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. @@ -279,18 +286,18 @@ class HttpHandler(BaseHTTPRequestHandler): self._handle('GET', path, args) # Special handling for some errors except PermissionError as e: - self.send_error(403, 'Forbidden') + self.send_response(403, 'Forbidden') self.end_headers() print(e) traceback.print_tb(e.__traceback__) except ValueError as e: - self.send_header(400, 'Bad Request') + self.send_response(400, 'Bad Request') self.end_headers() print(e) traceback.print_tb(e.__traceback__) except BaseException as e: # Generic error handling - self.send_error(500, 'Internal Server Error') + self.send_response(500, 'Internal Server Error') self.end_headers() print(e) traceback.print_tb(e.__traceback__) @@ -304,26 +311,26 @@ class HttpHandler(BaseHTTPRequestHandler): # Read the POST body, if it exists, and its MIME type is application/x-www-form-urlencoded clen: str = self.headers.get('Content-Length', failobj='0') ctype: str = self.headers.get('Content-Type', failobj='application/octet-stream') - post = '' + post: str = '' if ctype == 'application/x-www-form-urlencoded': - post: str = self.rfile.read(int(clen)).decode('utf-8') + post = self.rfile.read(int(clen)).decode('utf-8') # Parse the request and hand it to the handle function path, args = self._parse_args(self.path, postbody=post) self._handle('POST', path, args) # Special handling for some errors except PermissionError as e: - self.send_error(403, 'Forbidden') + self.send_response(403, 'Forbidden') self.end_headers() print(e) traceback.print_tb(e.__traceback__) except ValueError as e: - self.send_header(400, 'Bad Request') + self.send_response(400, 'Bad Request') self.end_headers() print(e) traceback.print_tb(e.__traceback__) except BaseException as e: # Generic error handling - self.send_error(500, 'Internal Server Error') + self.send_response(500, 'Internal Server Error') self.end_headers() print(e) traceback.print_tb(e.__traceback__) diff --git a/matemat/webserver/test/__init__.py b/matemat/webserver/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py new file mode 100644 index 0000000..806a312 --- /dev/null +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -0,0 +1,163 @@ + +from typing import Any, Dict, Tuple + +import unittest.mock +from io import BytesIO + +from abc import ABC +from datetime import datetime +from http.server import HTTPServer + + +class HttpResponse: + """ + A really basic HTTP response container and parser class, just good enough for unit testing a HTTP server, if even. + + Usage: + response = HttpResponse() + while response.parse_phase != 'done' + response.parse() + print(response.statuscode) + """ + + def __init__(self) -> None: + # The HTTP status code of the response + self.statuscode: int = 0 + # HTTP headers set in the response + self.headers: Dict[str, str] = { + 'Content-Length': 0 + } + # The response body. Only UTF-8 strings are supported + self.body: str = '' + # Parsing phase, one of 'begin', 'hdr', 'body' or 'done' + self.parse_phase = 'begin' + # Buffer for uncompleted lines + self.buffer: bytes = bytes() + + def parse(self, fragment: bytes) -> None: + """ + Parse a new fragment of data. This function does nothing if the parsed HTTP response is already complete. + + :param fragment: The data fragment to parse. + """ + # response packet complete, nothing to do + if self.parse_phase == 'done': + return + # If in the body phase, simply decode and append to the body, while the body is not complete yet + elif self.parse_phase == 'body': + self.body += fragment.decode('utf-8') + if len(self.body) >= int(self.headers['Content-Length']): + self.parse_phase = 'done' + return + if b'\r\n' not in fragment: + # If the fragment does not contain a CR-LF, add it to the buffer, we only want to parse whole lines + self.buffer = self.buffer + fragment + else: + if not fragment.endswith(b'\r\n'): + # Special treatment for no trailing CR-LF: Add remainder to buffer + head, tail = fragment.rsplit(b'\r\n', 1) + data: str = (self.buffer + head).decode('utf-8') + self.buffer = tail + else: + data: str = (self.buffer + fragment).decode('utf-8') + self.buffer = bytes() + # Iterate the lines that are ready to be parsed + for line in data.split('\r\n'): + # The 'begin' phase indicates that the parser is waiting for the HTTP status line + if self.parse_phase == 'begin': + if line.startswith('HTTP/'): + # Parse the statuscode and advance to header parsing + _, statuscode, _ = line.split(' ', 2) + self.statuscode = int(statuscode) + self.parse_phase = 'hdr' + elif self.parse_phase == 'hdr': + # Parse a header line and add it to the header dict + if len(line) > 0: + k, v = line.split(':', 1) + self.headers[k.strip()] = v.strip() + else: + # Empty line separates header from body + self.parse_phase = 'body' + elif self.parse_phase == 'body': + # if there is a remainder in the data packet, it is (part of) the body, add to body string + self.body += line + if len(self.body) >= int(self.headers['Content-Length']): + self.parse_phase = 'done' + + +class MockServer: + """ + A mock implementation of http.server.HTTPServer. Only used for matemat-specific storage. + """ + + def __init__(self, webroot: str = '/var/matemat/webroot') -> None: + # Session timeout and variables for all sessions + self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + # Webroot for statically served content + self.webroot: str = webroot + + +class MockSocket(bytes): + """ + A mock implementation of a socket.socket for http.server.BaseHTTPRequestHandler. + + The bytes inheritance is due to a broken type annotation in BaseHTTPRequestHandler. + """ + + def __init__(self) -> None: + super().__init__() + # The request string + self.__request = bytes() + # The parsed response + self.__packet = HttpResponse() + + def set_request(self, request: bytes) -> None: + """ + Sets the HTTP request to send to the server. + + :param request: The request + """ + self.__request: bytes = request + + def makefile(self, mode: str, size: int) -> BytesIO: + """ + Required by http.server.HTTPServer. + + :return: A dummy buffer IO object instead of a network socket file handle. + """ + return BytesIO(self.__request) + + def sendall(self, b: bytes) -> None: + """ + Required by http.server.HTTPServer. + + :param b: The data to send to the client. Will be parsed directly instead. + """ + self.__packet.parse(b) + + def get_response(self) -> HttpResponse: + """ + Fetches the parsed HTTP response generated by the server. + + :return: The response object. + """ + return self.__packet + + +class AbstractHttpdTest(ABC, unittest.TestCase): + """ + An abstract test case that can be inherited by test case classes that want to test part of the webserver's core + functionality. + + Usage (subclass test method): + + self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n') + handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + TODO(s3lph): This could probably go here instead. + """ + + def setUp(self) -> None: + self.server: HTTPServer = MockServer() + self.client_sock: MockSocket = MockSocket() diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py new file mode 100644 index 0000000..af51a75 --- /dev/null +++ b/matemat/webserver/test/test_session.py @@ -0,0 +1,53 @@ + +from typing import Any, Dict + +from datetime import datetime, timedelta + +from matemat.webserver.httpd import HttpHandler, pagelet +from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest + + +@pagelet('/just/testing/sessions') +def test_pagelet(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): + session_vars['test'] = 'hello, world!' + headers['Content-Type'] = 'text/plain' + return 200, 'session test' + + +class TestSession(AbstractHttpdTest): + """ + Test session handling of the Matemat webserver. + """ + + def test_create_new_session(self): + # Reference date to make sure the session expiry lies in the future + refdate = datetime.utcnow() + timedelta(seconds=3500) + # Send a mock GET request for '/just/testing/sessions' + self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n') + # Trigger request handling + handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) + # Fetch the parsed response + packet = self.client_sock.get_response() + # Make sure a full HTTP response was parsed + self.assertEqual('done', packet.parse_phase) + # Make sure the request was served by the test pagelet + self.assertEqual('session test', packet.body) + self.assertEqual(200, packet.statuscode) + + session_id: str = list(handler.server.session_vars.keys())[0] + # Make sure a cookie was set - assuming that only one was set + self.assertIn('Set-Cookie', packet.headers) + # Split into the cookie itself + cookie, expiry = packet.headers['Set-Cookie'].split(';') + cookie: str = cookie.strip() + expiry: str = expiry.strip() + # Make sure the 'matemat_session_id' cookie was set to the session ID string + self.assertEqual(f'matemat_session_id={session_id}', cookie) + # Make sure the session expires in about one hour + self.assertTrue(expiry.startswith('expires=')) + _, expdatestr = expiry.split('=', 1) + expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') + self.assertTrue(expdate > refdate) + # Make sure the session exists on the server + self.assertIn('test', handler.session_vars) + self.assertEqual('hello, world!', handler.session_vars['test']) From 2f2f6b73e9dfaffdcf9621735036036b0df295e9 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 15 Jun 2018 22:30:35 +0200 Subject: [PATCH 07/46] Use crypt with SHA512 instead of bcrypt. Going to change to BLOWFISH when adopting Python 3.7. --- matemat/db/facade.py | 22 ++++++++++++++-------- matemat/db/test/test_facade.py | 4 ++-- requirements.txt | 1 - 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/matemat/db/facade.py b/matemat/db/facade.py index 027c62f..4bd27cd 100644 --- a/matemat/db/facade.py +++ b/matemat/db/facade.py @@ -1,12 +1,18 @@ from typing import List, Optional, Any, Type -import bcrypt +import crypt from matemat.primitives import User, Product from matemat.exceptions import AuthenticationError, DatabaseConsistencyError from matemat.db import DatabaseWrapper +# TODO: Change to METHOD_BLOWFISH when adopting Python 3.7 +""" +The method to use for password hashing. +""" +_CRYPT_METHOD = crypt.METHOD_SHA512 + class MatematDatabase(object): """ @@ -92,7 +98,7 @@ class MatematDatabase(object): :raises ValueError: If a user with the same name already exists. """ # Hash the password. - pwhash: str = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt(12)) + pwhash: str = crypt.crypt(password, crypt.mksalt(_CRYPT_METHOD)) user_id: int = -1 with self.db.transaction() as c: # Look up whether a user with the same name already exists. @@ -138,9 +144,9 @@ class MatematDatabase(object): if row is None: raise AuthenticationError('User does not exist') user_id, username, email, pwhash, tkhash, admin, member = row - if password is not None and not bcrypt.checkpw(password.encode('utf-8'), pwhash): + if password is not None and crypt.crypt(password, pwhash) != pwhash: raise AuthenticationError('Password mismatch') - elif touchkey is not None and tkhash is not None and not bcrypt.checkpw(touchkey.encode('utf-8'), tkhash): + elif touchkey is not None and tkhash is not None and crypt.crypt(touchkey, tkhash) != tkhash: raise AuthenticationError('Touchkey mismatch') elif touchkey is not None and tkhash is None: raise AuthenticationError('Touchkey not set') @@ -165,10 +171,10 @@ class MatematDatabase(object): if row is None: raise AuthenticationError('User does not exist in database.') # Verify the old password, if it should be verified. - if verify_password and not bcrypt.checkpw(oldpass.encode('utf-8'), row[0]): + if verify_password and crypt.crypt(oldpass, row[0]) != row[0]: raise AuthenticationError('Old password does not match.') # Hash the new password and write it to the database. - pwhash: str = bcrypt.hashpw(newpass.encode('utf-8'), bcrypt.gensalt(12)) + pwhash: str = crypt.crypt(newpass, crypt.mksalt(_CRYPT_METHOD)) c.execute(''' UPDATE users SET password = :pwhash, lastchange = STRFTIME('%s', 'now') WHERE user_id = :user_id ''', { @@ -195,10 +201,10 @@ class MatematDatabase(object): if row is None: raise AuthenticationError('User does not exist in database.') # Verify the password, if it should be verified. - if verify_password and not bcrypt.checkpw(password.encode('utf-8'), row[0]): + if verify_password and crypt.crypt(password, row[0]) != row[0]: raise AuthenticationError('Password does not match.') # Hash the new touchkey and write it to the database. - tkhash: str = bcrypt.hashpw(touchkey.encode('utf-8'), bcrypt.gensalt(12)) if touchkey is not None else None + tkhash: str = crypt.crypt(touchkey, crypt.mksalt(_CRYPT_METHOD)) if touchkey is not None else None c.execute(''' UPDATE users SET touchkey = :tkhash, lastchange = STRFTIME('%s', 'now') WHERE user_id = :user_id ''', { diff --git a/matemat/db/test/test_facade.py b/matemat/db/test/test_facade.py index d7236b6..b17262f 100644 --- a/matemat/db/test/test_facade.py +++ b/matemat/db/test/test_facade.py @@ -1,7 +1,7 @@ import unittest -import bcrypt +import crypt from matemat.db import MatematDatabase from matemat.exceptions import AuthenticationError, DatabaseConsistencyError @@ -58,7 +58,7 @@ class DatabaseTest(unittest.TestCase): u = db.create_user('testuser', 'supersecurepassword', 'testuser@example.com') # Add a touchkey without using the provided function c.execute('''UPDATE users SET touchkey = :tkhash WHERE user_id = :user_id''', { - 'tkhash': bcrypt.hashpw(b'0123', bcrypt.gensalt(12)), + 'tkhash': crypt.crypt('0123', crypt.mksalt(crypt.METHOD_SHA512)), 'user_id': u.id }) user = db.login('testuser', 'supersecurepassword') diff --git a/requirements.txt b/requirements.txt index f2fa21b..b8ab4ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -bcrypt apsw From 9414b19dc21a59c1a65c522cbd5e093b00bcb2df Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 00:03:30 +0200 Subject: [PATCH 08/46] More unit tests for the webserver. --- matemat/webserver/test/abstract_httpd_test.py | 30 +++++++++- matemat/webserver/test/test_session.py | 59 +++++++++++++++++-- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index 806a312..a931628 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -1,5 +1,5 @@ -from typing import Any, Dict, Tuple +from typing import Any, Callable, Dict, Tuple, Union import unittest.mock from io import BytesIO @@ -8,6 +8,8 @@ from abc import ABC from datetime import datetime from http.server import HTTPServer +from matemat.webserver.httpd import pagelet + class HttpResponse: """ @@ -27,6 +29,7 @@ class HttpResponse: self.headers: Dict[str, str] = { 'Content-Length': 0 } + self.pagelet: str = None # The response body. Only UTF-8 strings are supported self.body: str = '' # Parsing phase, one of 'begin', 'hdr', 'body' or 'done' @@ -34,6 +37,10 @@ class HttpResponse: # Buffer for uncompleted lines self.buffer: bytes = bytes() + def __finalize(self): + self.parse_phase = 'done' + self.pagelet = self.headers['X-Test-Pagelet'] + def parse(self, fragment: bytes) -> None: """ Parse a new fragment of data. This function does nothing if the parsed HTTP response is already complete. @@ -47,7 +54,7 @@ class HttpResponse: elif self.parse_phase == 'body': self.body += fragment.decode('utf-8') if len(self.body) >= int(self.headers['Content-Length']): - self.parse_phase = 'done' + self.__finalize() return if b'\r\n' not in fragment: # If the fragment does not contain a CR-LF, add it to the buffer, we only want to parse whole lines @@ -82,7 +89,7 @@ class HttpResponse: # if there is a remainder in the data packet, it is (part of) the body, add to body string self.body += line if len(self.body) >= int(self.headers['Content-Length']): - self.parse_phase = 'done' + self.__finalize() class MockServer: @@ -144,6 +151,23 @@ class MockSocket(bytes): return self.__packet +def test_pagelet(path: str): + + def with_testing_headers(fun: Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str]], + Tuple[int, Union[bytes, str]]]): + @pagelet(path) + def testing_wrapper(method: str, + path: str, + args: Dict[str, str], + session_vars: Dict[str, Any], + headers: Dict[str, str]): + status, body = fun(method, path, args, session_vars, headers) + headers['X-Test-Pagelet'] = fun.__name__ + return status, body + return testing_wrapper + return with_testing_headers + + class AbstractHttpdTest(ABC, unittest.TestCase): """ An abstract test case that can be inherited by test case classes that want to test part of the webserver's core diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index af51a75..6697da3 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -2,13 +2,18 @@ from typing import Any, Dict from datetime import datetime, timedelta +from time import sleep -from matemat.webserver.httpd import HttpHandler, pagelet -from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest +from matemat.webserver.httpd import HttpHandler +from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet -@pagelet('/just/testing/sessions') -def test_pagelet(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): +@test_pagelet('/just/testing/sessions') +def session_test_pagelet(method: str, + path: str, + args: Dict[str, str], + session_vars: Dict[str, Any], + headers: Dict[str, str]): session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' return 200, 'session test' @@ -21,7 +26,7 @@ class TestSession(AbstractHttpdTest): def test_create_new_session(self): # Reference date to make sure the session expiry lies in the future - refdate = datetime.utcnow() + timedelta(seconds=3500) + refdate: datetime = datetime.utcnow() + timedelta(seconds=3500) # Send a mock GET request for '/just/testing/sessions' self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n') # Trigger request handling @@ -31,7 +36,7 @@ class TestSession(AbstractHttpdTest): # Make sure a full HTTP response was parsed self.assertEqual('done', packet.parse_phase) # Make sure the request was served by the test pagelet - self.assertEqual('session test', packet.body) + self.assertEqual('session_test_pagelet', packet.pagelet) self.assertEqual(200, packet.statuscode) session_id: str = list(handler.server.session_vars.keys())[0] @@ -51,3 +56,45 @@ class TestSession(AbstractHttpdTest): # Make sure the session exists on the server self.assertIn('test', handler.session_vars) self.assertEqual('hello, world!', handler.session_vars['test']) + + def test_resume_session(self): + # Test session expiry date + refdate: datetime = datetime.utcnow() + timedelta(hours=1) + # Session ID for testing + session_id: str = 'testsessionid' + # Insert test session + self.server.session_vars[session_id] = refdate, {'test': 'bar'} + sleep(2) + + # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie + self.client_sock.set_request( + f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n'.encode('utf-8')) + # Trigger request handling + handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) + # Fetch the parsed response + packet = self.client_sock.get_response() + # Make sure a full HTTP response was parsed + self.assertEqual('done', packet.parse_phase) + # Make sure the request was served by the test pagelet + self.assertEqual('session_test_pagelet', packet.pagelet) + self.assertEqual(200, packet.statuscode) + + response_session_id: str = list(handler.server.session_vars.keys())[0] + # Make sure a cookie was set - assuming that only one was set + self.assertIn('Set-Cookie', packet.headers) + # Split into the cookie itself + cookie, expiry = packet.headers['Set-Cookie'].split(';') + cookie: str = cookie.strip() + expiry: str = expiry.strip() + # Make sure the 'matemat_session_id' cookie was set to the session ID string + self.assertEqual(f'matemat_session_id={response_session_id}', cookie) + # Make sure the session ID matches the one we sent along + self.assertEqual(session_id, response_session_id) + # Make sure the session timeout was postponed + self.assertTrue(expiry.startswith('expires=')) + _, expdatestr = expiry.split('=', 1) + expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') + self.assertTrue(expdate > refdate) + # Make sure the session exists on the server + self.assertIn('test', handler.session_vars) + self.assertEqual('hello, world!', handler.session_vars['test']) From 2f12403e1f211943bb14e78ed2fce2590c9cb8f3 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 20:16:39 +0200 Subject: [PATCH 09/46] Added two more testcases for session testing. --- matemat/webserver/httpd.py | 5 +- matemat/webserver/test/abstract_httpd_test.py | 2 +- matemat/webserver/test/test_session.py | 62 ++++++++++++++++++- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 2f3f3cb..5c75520 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -175,12 +175,13 @@ class HttpHandler(BaseHTTPRequestHandler): :param args: Arguments sent with the request. This includes GET and POST arguments, where the POST arguments take precedence. """ - # Start or resume a session; report an error on session timeout + # Start or resume a session; redirect to / on session timeout try: session_id, timeout = self._start_session() except TimeoutError: - self.send_error(599, 'Session Timed Out', 'Session Timed Out.') + self.send_response(302) self.send_header('Set-Cookie', 'matemat_session_id=; expires=Thu, 01 Jan 1970 00:00:00 GMT') + self.send_header('Location', '/') self.end_headers() return self.session_id: str = session_id diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index a931628..eda18dd 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -39,7 +39,7 @@ class HttpResponse: def __finalize(self): self.parse_phase = 'done' - self.pagelet = self.headers['X-Test-Pagelet'] + self.pagelet = self.headers.get('X-Test-Pagelet', None) def parse(self, fragment: bytes) -> None: """ diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index 6697da3..b8e21cf 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -68,7 +68,7 @@ class TestSession(AbstractHttpdTest): # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie self.client_sock.set_request( - f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n'.encode('utf-8')) + f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) # Trigger request handling handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) # Fetch the parsed response @@ -98,3 +98,63 @@ class TestSession(AbstractHttpdTest): # Make sure the session exists on the server self.assertIn('test', handler.session_vars) self.assertEqual('hello, world!', handler.session_vars['test']) + + def test_unknown_session_id(self): + # Unknown session ID + session_id: str = 'theserverdoesnotknowthisid' + refdate: datetime = datetime.utcnow() + timedelta(seconds=3500) + # Send a mock GET request for '/just/testing/sessions' with a session cookie not known to the server + self.client_sock.set_request( + f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) + # Trigger request handling + handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) + # Fetch the parsed response + packet = self.client_sock.get_response() + # Make sure a full HTTP response was parsed + self.assertEqual('done', packet.parse_phase) + # Make sure the request was served by the test pagelet + self.assertEqual('session_test_pagelet', packet.pagelet) + self.assertEqual(200, packet.statuscode) + + server_session_id: str = list(handler.server.session_vars.keys())[0] + self.assertNotEqual(session_id, server_session_id) + # Make sure a cookie was set - assuming that only one was set + self.assertIn('Set-Cookie', packet.headers) + # Split into the cookie itself + cookie, expiry = packet.headers['Set-Cookie'].split(';') + cookie: str = cookie.strip() + expiry: str = expiry.strip() + # Make sure the 'matemat_session_id' cookie was set to the session ID string + self.assertEqual(f'matemat_session_id={server_session_id}', cookie) + # Make sure the session expires in about one hour + self.assertTrue(expiry.startswith('expires=')) + _, expdatestr = expiry.split('=', 1) + expdate = datetime.strptime(expdatestr, '%a, %d %b %Y %H:%M:%S GMT') + self.assertTrue(expdate > refdate) + # Make sure the session exists on the server + self.assertIn('test', handler.session_vars) + self.assertEqual('hello, world!', handler.session_vars['test']) + + def test_session_expired(self): + # Test session expiry date + refdate: datetime = datetime.utcnow() - timedelta(hours=1) + # Session ID for testing + session_id: str = 'testsessionid' + # Insert test session + self.server.session_vars[session_id] = refdate, {'test': 'bar'} + + # Send a mock GET request for '/just/testing/sessions' with a matemat session cookie + self.client_sock.set_request( + f'GET /just/testing/sessions HTTP/1.1\r\nCookie: matemat_session_id={session_id}\r\n\r\n'.encode('utf-8')) + # Trigger request handling + handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) + # Fetch the parsed response + packet = self.client_sock.get_response() + # Make sure a full HTTP response was parsed + self.assertEqual('done', packet.parse_phase) + # Make sure the server redirects to / + self.assertEqual(302, packet.statuscode) + self.assertIn('Location', packet.headers) + self.assertEqual('/', packet.headers['Location']) + # Make sure the session was terminated + self.assertNotIn(session_id, self.server.session_vars) From 0bfd4efab9016bc2f43e328a381f12a12d684b0f Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 21:34:48 +0200 Subject: [PATCH 10/46] Added unit tests for pagelet and static resource serving. --- matemat/webserver/httpd.py | 12 +- matemat/webserver/test/abstract_httpd_test.py | 11 +- matemat/webserver/test/test_serve.py | 114 ++++++++++++++++++ 3 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 matemat/webserver/test/test_serve.py diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 5c75520..a22969e 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -227,8 +227,13 @@ class HttpHandler(BaseHTTPRequestHandler): # Make sure the file is actually inside the webroot directory and that it exists if os.path.commonpath([filepath, self.server.webroot]) == self.server.webroot and os.path.exists(filepath): # Open and read the file - with open(filepath, 'rb') as f: - data = f.read() + try: + with open(filepath, 'rb') as f: + data = f.read() + except PermissionError: + self.send_error(403) + self.end_headers() + return # File read successfully, send 'OK' header self.send_response(200) # TODO: Guess the MIME type. Unfortunately this call solely relies on the file extension, not ideal? @@ -236,8 +241,9 @@ class HttpHandler(BaseHTTPRequestHandler): # Fall back to octet-stream type, if unknown if mimetype is None: mimetype = 'application/octet-stream' - # Send content type header + # Send content type and length header self.send_header('Content-Type', mimetype) + self.send_header('Content-Length', len(data)) self.end_headers() # Send the requested resource as response body self.wfile.write(data) diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index eda18dd..de0daf6 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -3,6 +3,7 @@ from typing import Any, Callable, Dict, Tuple, Union import unittest.mock from io import BytesIO +from tempfile import TemporaryDirectory from abc import ABC from datetime import datetime @@ -45,6 +46,8 @@ class HttpResponse: """ Parse a new fragment of data. This function does nothing if the parsed HTTP response is already complete. + DO NOT USE THIS OUTSIDE UNIT TESTING! + :param fragment: The data fragment to parse. """ # response packet complete, nothing to do @@ -178,10 +181,12 @@ class AbstractHttpdTest(ABC, unittest.TestCase): self.client_sock.set_request(b'GET /just/testing/sessions HTTP/1.1\r\n\r\n') handler = HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() - - TODO(s3lph): This could probably go here instead. """ def setUp(self) -> None: - self.server: HTTPServer = MockServer() + self.tempdir: TemporaryDirectory = TemporaryDirectory(prefix='matemat.', dir='/tmp/') + self.server: HTTPServer = MockServer(webroot=self.tempdir.name) self.client_sock: MockSocket = MockSocket() + + def tearDown(self): + self.tempdir.cleanup() diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py new file mode 100644 index 0000000..0c59a0c --- /dev/null +++ b/matemat/webserver/test/test_serve.py @@ -0,0 +1,114 @@ + +from typing import Any, Dict + +import os +import os.path +from matemat.webserver.httpd import HttpHandler +from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet + + +@test_pagelet('/just/testing/serve_pagelet_ok') +def serve_test_pagelet_ok(method: str, + path: str, + args: Dict[str, str], + session_vars: Dict[str, Any], + headers: Dict[str, str]): + headers['Content-Type'] = 'text/plain' + return 200, 'serve test pagelet ok' + + +@test_pagelet('/just/testing/serve_pagelet_fail') +def serve_test_pagelet_fail(method: str, + path: str, + args: Dict[str, str], + session_vars: Dict[str, Any], + headers: Dict[str, str]): + session_vars['test'] = 'hello, world!' + headers['Content-Type'] = 'text/plain' + return 500, 'serve test pagelet fail' + + +class TestServe(AbstractHttpdTest): + """ + Test cases for the content serving of the web server. + """ + + def setUp(self): + super().setUp() + # Create a static resource in the temp dir + with open(os.path.join(self.tempdir.name, 'static_resource.txt'), 'w') as f: + f.write('static resource test') + # Create a second static resource chmodded to 0000, to test 403 Forbidden error + forbidden: str = os.path.join(self.tempdir.name, 'forbidden_static_resource.txt') + with open(forbidden, 'w') as f: + f.write('This should not be readable') + os.chmod(forbidden, 0) + + def test_serve_pagelet_ok(self): + # Call the test pagelet that produces a 200 OK result + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_ok HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure the correct pagelet was called + self.assertEqual('serve_test_pagelet_ok', packet.pagelet) + # Make sure the expected content is served + self.assertEqual(200, packet.statuscode) + self.assertEqual('serve test pagelet ok', packet.body) + + def test_serve_pagelet_fail(self): + # Call the test pagelet that produces a 500 Internal Server Error result + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_fail HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure the correct pagelet was called + self.assertEqual('serve_test_pagelet_fail', packet.pagelet) + # Make sure the expected content is served + self.assertEqual(500, packet.statuscode) + self.assertEqual('serve test pagelet fail', packet.body) + + def test_serve_static_ok(self): + # Request a static resource + self.client_sock.set_request(b'GET /static_resource.txt HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure that no pagelet was called + self.assertIsNone(packet.pagelet) + # Make sure the expected content is served + self.assertEqual(200, packet.statuscode) + self.assertEqual('static resource test', packet.body) + + def test_serve_static_forbidden(self): + # Request a static resource with lacking permissions + self.client_sock.set_request(b'GET /forbidden_static_resource.txt HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure that no pagelet was called + self.assertIsNone(packet.pagelet) + # Make sure a 403 header is served + self.assertEqual(403, packet.statuscode) + + def test_serve_not_found(self): + # Request a nonexistent resource + self.client_sock.set_request(b'GET /nonexistent HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure that no pagelet was called + self.assertIsNone(packet.pagelet) + # Make sure a 404 header is served + self.assertEqual(404, packet.statuscode) + + def test_serve_directory_traversal(self): + # Request a resource outside the webroot + self.client_sock.set_request(b'GET /../../../../../etc/passwd HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure that no pagelet was called + self.assertIsNone(packet.pagelet) + # Make sure a 404 header is served + self.assertEqual(404, packet.statuscode) From 3b2d8ffa935cd4bcd3db7fe7f41588c2f6d79605 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 21:37:56 +0200 Subject: [PATCH 11/46] Added unit tests for pagelet and static resource serving. --- matemat/webserver/test/test_serve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 0c59a0c..5fdc6ec 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -90,6 +90,7 @@ class TestServe(AbstractHttpdTest): self.assertIsNone(packet.pagelet) # Make sure a 403 header is served self.assertEqual(403, packet.statuscode) + self.assertNotEqual('This should not be readable', packet.body) def test_serve_not_found(self): # Request a nonexistent resource From 3e3d544d902671a1be178a2e658e73369a2816f6 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 21:47:40 +0200 Subject: [PATCH 12/46] GitLab CI: Run unit tests as unprivileged user. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d351014..b46ce36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,11 +8,12 @@ stages: test: stage: test script: + - useradd matemat - apt-get update -qy - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - pip3 install wheel - pip3 install -r requirements.txt - - python3-coverage run --branch -m unittest discover matemat + - su - matemat 'python3-coverage run --branch -m unittest discover matemat' - python3-coverage report -m --include 'matemat/*' --omit '*/test_*.py' codestyle: From 27c264278af7f5e5571d104a6c0fb6016d78ff7c Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 21:49:56 +0200 Subject: [PATCH 13/46] GitLab CI: Run unit tests as unprivileged user. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b46ce36..21e44c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ test: - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - pip3 install wheel - pip3 install -r requirements.txt - - su - matemat 'python3-coverage run --branch -m unittest discover matemat' + - su - matemat python3-coverage run --branch -m unittest discover matemat - python3-coverage report -m --include 'matemat/*' --omit '*/test_*.py' codestyle: From 1f0312fcbdb0d71b058cacb726162e0bfd94e9b8 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 21:52:00 +0200 Subject: [PATCH 14/46] GitLab CI: Run unit tests as unprivileged user. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21e44c0..b27f6a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ test: - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - pip3 install wheel - pip3 install -r requirements.txt - - su - matemat python3-coverage run --branch -m unittest discover matemat + - su - matemat -c 'python3-coverage run --branch -m unittest discover matemat' - python3-coverage report -m --include 'matemat/*' --omit '*/test_*.py' codestyle: From 617cc2aa6f543ea487e951a7492c5c958e3badd9 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 22:11:42 +0200 Subject: [PATCH 15/46] GitLab CI: Run unit tests as unprivileged user. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b27f6a9..d70106b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ test: stage: test script: - useradd matemat + - chmod g+r -R . - apt-get update -qy - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - pip3 install wheel From b06c54e7829aae0e2742d740199389aad379edb0 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 22:15:18 +0200 Subject: [PATCH 16/46] GitLab CI: Run unit tests as unprivileged user. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d70106b..05aa099 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ test: stage: test script: - useradd matemat - - chmod g+r -R . + - chmod o+rw -R . - apt-get update -qy - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - pip3 install wheel From 17fe381402bfc842901559674dda74c7605a3367 Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 22:54:07 +0200 Subject: [PATCH 17/46] Use unprivileged docker image. --- .gitlab-ci.yml | 12 ++---------- Dockerfile | 10 ++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05aa099..b0cbf3c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ --- -image: debian:buster +image: s3lph/matemat:20180619-01 stages: - test @@ -8,21 +8,13 @@ stages: test: stage: test script: - - useradd matemat - - chmod o+rw -R . - - apt-get update -qy - - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - - pip3 install wheel - pip3 install -r requirements.txt - - su - matemat -c 'python3-coverage run --branch -m unittest discover matemat' + - python3-coverage run --branch -m unittest discover matemat - python3-coverage report -m --include 'matemat/*' --omit '*/test_*.py' codestyle: stage: codestyle script: - - apt-get update -qy - - apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential - - pip3 install wheel pycodestyle mypy - pip3 install -r requirements.txt - pycodestyle matemat # - mypy --ignore-missing-imports --strict -p matemat diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..126a06f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ + +FROM debian:buster + +RUN useradd -d /home/matemat -m matemat +RUN apt-get update -qy +RUN apt-get install -y --no-install-recommends python3-dev python3-pip python3-coverage python3-setuptools build-essential +RUN pip3 install wheel pycodestyle mypy + +WORKDIR /home/matemat +USER matemat From 78d347f26c2183cc806eac8aefa806cb90fb081a Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 22:55:37 +0200 Subject: [PATCH 18/46] Use unprivileged docker image. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0cbf3c..9adc47d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ --- -image: s3lph/matemat:20180619-01 +image: s3lph/matemat-ci:20180619-01 stages: - test From 67db4c654a54f94270c8e89dec3e53238c00410e Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 23:51:20 +0200 Subject: [PATCH 19/46] Added tests for HTTP POST requests. --- matemat/webserver/httpd.py | 21 ++------ matemat/webserver/test/test_post.py | 77 ++++++++++++++++++++++++++++ matemat/webserver/test/test_serve.py | 11 ++++ 3 files changed, 93 insertions(+), 16 deletions(-) create mode 100644 matemat/webserver/test/test_post.py diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index a22969e..220849c 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -227,13 +227,8 @@ class HttpHandler(BaseHTTPRequestHandler): # Make sure the file is actually inside the webroot directory and that it exists if os.path.commonpath([filepath, self.server.webroot]) == self.server.webroot and os.path.exists(filepath): # Open and read the file - try: - with open(filepath, 'rb') as f: - data = f.read() - except PermissionError: - self.send_error(403) - self.end_headers() - return + with open(filepath, 'rb') as f: + data = f.read() # File read successfully, send 'OK' header self.send_response(200) # TODO: Guess the MIME type. Unfortunately this call solely relies on the file extension, not ideal? @@ -292,22 +287,16 @@ class HttpHandler(BaseHTTPRequestHandler): path, args = self._parse_args(self.path) self._handle('GET', path, args) # Special handling for some errors - except PermissionError as e: + except PermissionError: self.send_response(403, 'Forbidden') self.end_headers() - print(e) - traceback.print_tb(e.__traceback__) - except ValueError as e: + except ValueError: self.send_response(400, 'Bad Request') self.end_headers() - print(e) - traceback.print_tb(e.__traceback__) - except BaseException as e: + except BaseException: # Generic error handling self.send_response(500, 'Internal Server Error') self.end_headers() - print(e) - traceback.print_tb(e.__traceback__) # noinspection PyPep8Naming def do_POST(self) -> None: diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py new file mode 100644 index 0000000..cc92ad1 --- /dev/null +++ b/matemat/webserver/test/test_post.py @@ -0,0 +1,77 @@ + +from typing import Any, Dict, List + +import os +import os.path +from matemat.webserver.httpd import HttpHandler +from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet + + +@test_pagelet('/just/testing/post') +def post_test_pagelet(method: str, + path: str, + args: Dict[str, str], + session_vars: Dict[str, Any], + headers: Dict[str, str]): + dump: str = '' + for k, v in args.items(): + dump += f'{k}: {v}\n' + return 200, dump + + +class TestPost(AbstractHttpdTest): + """ + Test cases for the content serving of the web server. + """ + + def setUp(self): + super().setUp() + # Create a static resource in the temp dir + with open(os.path.join(self.tempdir.name, 'static_resource.txt'), 'w') as f: + f.write('static resource test') + + def test_post_get_only_args(self): + self.client_sock.set_request(b'POST /just/testing/post?foo=bar&test=1 HTTP/1.1\r\n' + b'Content-Length: 0\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() + self.assertEqual('bar', kv['foo']) + self.assertEqual('1', kv['test']) + + def test_post_post_only_args(self): + self.client_sock.set_request(b'POST /just/testing/post HTTP/1.1\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n' + b'Content-Length: 14\r\n\r\n' + b'foo=bar&test=1\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() + self.assertEqual('bar', kv['foo']) + self.assertEqual('1', kv['test']) + + def test_post_mixed_args(self): + self.client_sock.set_request(b'POST /just/testing/post?gettest=1&foo=baz HTTP/1.1\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n' + b'Content-Length: 18\r\n\r\n' + b'foo=bar&posttest=2\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() + self.assertEqual('bar', kv['foo']) + self.assertEqual('1', kv['gettest']) + self.assertEqual('2', kv['posttest']) + diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 5fdc6ec..f3dc6be 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -113,3 +113,14 @@ class TestServe(AbstractHttpdTest): self.assertIsNone(packet.pagelet) # Make sure a 404 header is served self.assertEqual(404, packet.statuscode) + + def test_static_post_not_allowed(self): + # Request a resource outside the webroot + self.client_sock.set_request(b'POST /iwanttouploadthis HTTP/1.1\r\n\r\nq=this%20should%20not%20be%20uploaded') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure that no pagelet was called + self.assertIsNone(packet.pagelet) + # Make sure a 405 Method Not Allowed header is served + self.assertEqual(405, packet.statuscode) From 12f0a0928d645c8ea6d6c57e551a2f4d7ae4f77d Mon Sep 17 00:00:00 2001 From: s3lph Date: Tue, 19 Jun 2018 23:55:53 +0200 Subject: [PATCH 20/46] Removed an additional newline that was in violation of PEP8. --- matemat/webserver/test/test_post.py | 1 - 1 file changed, 1 deletion(-) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index cc92ad1..720baea 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -74,4 +74,3 @@ class TestPost(AbstractHttpdTest): self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['gettest']) self.assertEqual('2', kv['posttest']) - From 3801909408480653d00116ffeac8e0411815426a Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 20 Jun 2018 00:53:06 +0200 Subject: [PATCH 21/46] Added GET and POST array argument tests, and added some docstrings to the tests. --- matemat/webserver/test/test_post.py | 113 +++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 12 deletions(-) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 720baea..059d41b 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -1,8 +1,6 @@ from typing import Any, Dict, List -import os -import os.path from matemat.webserver.httpd import HttpHandler from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @@ -13,9 +11,13 @@ def post_test_pagelet(method: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): + """ + Test pagelet that simply prints the parsed arguments as response body. + """ + headers['Content-Type'] = 'text/plain' dump: str = '' for k, v in args.items(): - dump += f'{k}: {v}\n' + dump += f'{k}: {v if v is str else ",".join(v)}\n' return 200, dump @@ -24,53 +26,140 @@ class TestPost(AbstractHttpdTest): Test cases for the content serving of the web server. """ - def setUp(self): - super().setUp() - # Create a static resource in the temp dir - with open(os.path.join(self.tempdir.name, 'static_resource.txt'), 'w') as f: - f.write('static resource test') - def test_post_get_only_args(self): + """ + Test a POST request that only contains GET arguments. + """ + # Send POST request self.client_sock.set_request(b'POST /just/testing/post?foo=bar&test=1 HTTP/1.1\r\n' b'Content-Length: 0\r\n' b'Content-Type: application/x-www-form-urlencoded\r\n\r\n') HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() + + # Parse response body lines: List[str] = packet.body.split('\n')[:-1] kv: Dict[str, str] = dict() for l in lines: k, v = l.split(':', 1) - kv[k.strip()] = v.strip() + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['test']) def test_post_post_only_args(self): + """ + Test a POST request that only contains POST arguments (urlencoded). + """ + # Send POST request self.client_sock.set_request(b'POST /just/testing/post HTTP/1.1\r\n' b'Content-Type: application/x-www-form-urlencoded\r\n' b'Content-Length: 14\r\n\r\n' b'foo=bar&test=1\r\n') HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() + + # Parse response body lines: List[str] = packet.body.split('\n')[:-1] kv: Dict[str, str] = dict() for l in lines: k, v = l.split(':', 1) - kv[k.strip()] = v.strip() + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['test']) def test_post_mixed_args(self): + """ + Test that mixed POST and GET args are properly parsed, and that POST takes precedence over GET. + """ + # Send POST request self.client_sock.set_request(b'POST /just/testing/post?gettest=1&foo=baz HTTP/1.1\r\n' b'Content-Type: application/x-www-form-urlencoded\r\n' b'Content-Length: 18\r\n\r\n' b'foo=bar&posttest=2\r\n') HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() + + # Parse response body lines: List[str] = packet.body.split('\n')[:-1] kv: Dict[str, str] = dict() for l in lines: k, v = l.split(':', 1) - kv[k.strip()] = v.strip() + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['gettest']) self.assertEqual('2', kv['posttest']) + + def test_post_get_array(self): + """ + Test a POST request that contains GET array arguments. + """ + # Send POST request + self.client_sock.set_request(b'POST /just/testing/post?foo=bar&test=1&foo=baz HTTP/1.1\r\n' + b'Content-Length: 0\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Parse response body + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed + self.assertListEqual(['bar', 'baz'], kv['foo']) + self.assertEqual('1', kv['test']) + + def test_post_post_array(self): + """ + Test a POST request that contains POST array arguments. + """ + # Send POST request + self.client_sock.set_request(b'POST /just/testing/post HTTP/1.1\r\n' + b'Content-Length: 22\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n\r\n' + b'foo=bar&test=1&foo=baz\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Parse response body + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed + self.assertListEqual(['bar', 'baz'], kv['foo']) + self.assertEqual('1', kv['test']) + + def test_post_mixed_array(self): + """ + Test a POST request that contains both GET and POST array arguments. + """ + # Send POST request + self.client_sock.set_request(b'POST /just/testing/post?foo=getbar&gettest=1&gettest=42&foo=getbaz HTTP/1.1\r\n' + b'Content-Length: 45\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n\r\n' + b'foo=postbar&posttest=1&posttest=2&foo=postbaz\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Parse response body + lines: List[str] = packet.body.split('\n')[:-1] + kv: Dict[str, str] = dict() + for l in lines: + k, v = l.split(':', 1) + kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') + + # Make sure the arguments were properly parsed + self.assertListEqual(['postbar', 'postbaz'], kv['foo']) + self.assertListEqual(['1', '42'], kv['gettest']) + self.assertListEqual(['1', '2'], kv['posttest']) From 5ccb2c9304a3649423569c71e8b5d27413838358 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 20 Jun 2018 02:04:35 +0200 Subject: [PATCH 22/46] Fixed failing unit tests. --- matemat/webserver/test/test_post.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 059d41b..ad99247 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -17,7 +17,7 @@ def post_test_pagelet(method: str, headers['Content-Type'] = 'text/plain' dump: str = '' for k, v in args.items(): - dump += f'{k}: {v if v is str else ",".join(v)}\n' + dump += f'{k}: {v if isinstance(v, str) else ",".join(v)}\n' return 200, dump From f702eccc57e38789687175cc118870a8611b999e Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 27 Jun 2018 21:17:18 +0200 Subject: [PATCH 23/46] First implementation of multipart/form-data parsing --- matemat/webserver/httpd.py | 90 ++++++------- matemat/webserver/pagelets/__init__.py | 1 + matemat/webserver/pagelets/login.py | 19 ++- matemat/webserver/pagelets/logout.py | 9 +- matemat/webserver/pagelets/main.py | 8 +- matemat/webserver/pagelets/touchkey.py | 18 ++- matemat/webserver/pagelets/upload_test.py | 28 ++++ matemat/webserver/test/abstract_httpd_test.py | 28 ++-- matemat/webserver/test/test_post.py | 123 ++++++++++++++---- matemat/webserver/test/test_serve.py | 19 +-- matemat/webserver/test/test_session.py | 4 +- matemat/webserver/util.py | 118 +++++++++++++++++ 12 files changed, 357 insertions(+), 108 deletions(-) create mode 100644 matemat/webserver/pagelets/upload_test.py create mode 100644 matemat/webserver/util.py diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 220849c..a4e9cca 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,12 +1,11 @@ -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Tuple, Union import traceback import os import socket import mimetypes -import urllib.parse from socketserver import TCPServer from http.server import HTTPServer, BaseHTTPRequestHandler from http.cookies import SimpleCookie @@ -14,6 +13,7 @@ from uuid import uuid4 from datetime import datetime, timedelta from matemat import __version__ as matemat_version +from matemat.webserver.util import parse_args # @@ -28,12 +28,17 @@ BaseHTTPRequestHandler.log_error = lambda self, fstring='', *args: None # Dictionary to hold registered pagelet paths and their handler functions -_PAGELET_PATHS: Dict[str, Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], - Tuple[int, Union[bytes, str]]]] = dict() +_PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) + str, # Request path + Dict[str, Tuple[str, Union[bytes, str, List[str]]]], # args: (name, (type, value)) + Dict[str, Any], # Session vars + Dict[str, str]], # Response headers + Tuple[int, Union[bytes, str]]]] = dict() # Returns: (status code, response body) # Inactivity timeout for client sessions _SESSION_TIMEOUT: int = 3600 +_MAX_POST: int = 1_000_000 def pagelet(path: str): @@ -43,12 +48,17 @@ def pagelet(path: str): The function must have the following signature: - (method: str, path: str, args: Dict[str, Union[str, List[str]], session_vars: Dict[str, Any], - headers: Dict[str, str]) -> (int, Optional[Union[str, bytes]]) + (method: str, + path: str, + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + session_vars: Dict[str, Any], + headers: Dict[str, str]) + -> (int, Optional[Union[str, bytes]]) method: The HTTP method (GET, POST) that was used. path: The path that was requested. - args: The arguments that were passed with the request (as GET or POST arguments). + args: The arguments that were passed with the request (as GET or POST arguments), each of which may be + either a str or bytes object, or a list of str. session_vars: The session storage. May be read from and written to. headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. returns: A tuple consisting of the HTTP status code (as an int) and the response body (as str or bytes, @@ -56,7 +66,12 @@ def pagelet(path: str): :param path: The path to register the function for. """ - def http_handler(fun: Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str], bytes], + + def http_handler(fun: Callable[[str, + str, + Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + Dict[str, Any], + Dict[str, str]], Tuple[int, Union[bytes, str]]]): # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun @@ -166,7 +181,7 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] - def _handle(self, method: str, path: str, args: Dict[str, Union[str, List[str]]]) -> None: + def _handle(self, method: str, path: str, args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]]) -> None: """ Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. @@ -238,7 +253,7 @@ class HttpHandler(BaseHTTPRequestHandler): mimetype = 'application/octet-stream' # Send content type and length header self.send_header('Content-Type', mimetype) - self.send_header('Content-Length', len(data)) + self.send_header('Content-Length', str(len(data))) self.end_headers() # Send the requested resource as response body self.wfile.write(data) @@ -247,36 +262,6 @@ class HttpHandler(BaseHTTPRequestHandler): self.send_response(404) self.end_headers() - @staticmethod - def _parse_args(request: str, postbody: Optional[str] = None) -> Tuple[str, Dict[str, Union[str, List[str]]]]: - """ - Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded form, parse the - arguments and return them as a dictionary. - - If a key is used both in GET and in POST, the POST value takes precedence, and the GET value is discarded. - - :param request: The request string to parse. - :param postbody: The POST body to parse, defaults to None. - :return: A tuple consisting of the base path and a dictionary with the parsed key/value pairs. - """ - # Parse the request "URL" (i.e. only the path) - tokens = urllib.parse.urlparse(request) - # Parse the GET arguments - args = urllib.parse.parse_qs(tokens.query) - - if postbody is not None: - # Parse the POST body - postargs = urllib.parse.parse_qs(postbody) - # Write all POST values into the dict, overriding potential duplicates from GET - for k, v in postargs.items(): - args[k] = v - # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values - for k, v in args.items(): - if len(v) == 1: - args[k] = v[0] - # Return the path and the parsed arguments - return tokens.path, args - # noinspection PyPep8Naming def do_GET(self) -> None: """ @@ -284,7 +269,7 @@ class HttpHandler(BaseHTTPRequestHandler): """ try: # Parse the request and hand it to the handle function - path, args = self._parse_args(self.path) + path, args = parse_args(self.path) self._handle('GET', path, args) # Special handling for some errors except PermissionError: @@ -305,25 +290,24 @@ class HttpHandler(BaseHTTPRequestHandler): """ try: # Read the POST body, if it exists, and its MIME type is application/x-www-form-urlencoded - clen: str = self.headers.get('Content-Length', failobj='0') + clen: int = int(str(self.headers.get('Content-Length', failobj='0'))) + if clen > _MAX_POST: + raise ValueError('Request too big') ctype: str = self.headers.get('Content-Type', failobj='application/octet-stream') - post: str = '' - if ctype == 'application/x-www-form-urlencoded': - post = self.rfile.read(int(clen)).decode('utf-8') + post: bytes = self.rfile.read(clen) + path, args = parse_args(self.path, postbody=post, enctype=ctype) # Parse the request and hand it to the handle function - path, args = self._parse_args(self.path, postbody=post) self._handle('POST', path, args) - # Special handling for some errors - except PermissionError as e: + # Special handling for some errors + except PermissionError: self.send_response(403, 'Forbidden') self.end_headers() - print(e) - traceback.print_tb(e.__traceback__) - except ValueError as e: + except ValueError: + self.send_response(400, 'Bad Request') + self.end_headers() + except TypeError: self.send_response(400, 'Bad Request') self.end_headers() - print(e) - traceback.print_tb(e.__traceback__) except BaseException as e: # Generic error handling self.send_response(500, 'Internal Server Error') diff --git a/matemat/webserver/pagelets/__init__.py b/matemat/webserver/pagelets/__init__.py index 9b926d6..71ded5e 100644 --- a/matemat/webserver/pagelets/__init__.py +++ b/matemat/webserver/pagelets/__init__.py @@ -8,3 +8,4 @@ from .main import main_page from .login import login_page from .logout import logout from .touchkey import touchkey_page +from .upload_test import upload_test diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index 876fd71..8fbe831 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, List, Optional, Tuple, Union from matemat.exceptions import AuthenticationError from matemat.webserver import pagelet @@ -8,7 +8,12 @@ from matemat.db import MatematDatabase @pagelet('/login') -def login_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): +def login_page(method: str, + path: str, + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + session_vars: Dict[str, Any], + headers: Dict[str, str])\ + -> Tuple[int, Optional[Union[str, bytes]]]: if 'user' in session_vars: headers['Location'] = '/' return 301, None @@ -38,13 +43,19 @@ def login_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[ ''' return 200, data.format(msg=args['msg'] if 'msg' in args else '') elif method == 'POST': - print(args) + if 'username' not in args or not isinstance(args['username'], str): + return 400, None + if 'password' not in args or not isinstance(args['password'], str): + return 400, None + username: str = args['username'] + password: str = args['password'] with MatematDatabase('test.db') as db: try: - user: User = db.login(args['username'], args['password']) + user: User = db.login(username, password) except AuthenticationError: headers['Location'] = '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' return 301, bytes() session_vars['user'] = user headers['Location'] = '/' return 301, bytes() + return 405, None diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index 86095b0..53a292a 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -1,11 +1,16 @@ -from typing import Any, Dict +from typing import Any, Dict, List, Optional, Tuple, Union from matemat.webserver import pagelet @pagelet('/logout') -def logout(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): +def logout(method: str, + path: str, + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + session_vars: Dict[str, Any], + headers: Dict[str, str])\ + -> Tuple[int, Optional[Union[str, bytes]]]: if 'user' in session_vars: del session_vars['user'] headers['Location'] = '/' diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index 2ead15d..d2dd208 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -1,5 +1,5 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from matemat.webserver import MatematWebserver, pagelet from matemat.primitives import User @@ -7,7 +7,11 @@ from matemat.db import MatematDatabase @pagelet('/') -def main_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str])\ +def main_page(method: str, + path: str, + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + session_vars: Dict[str, Any], + headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: data = ''' diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index fd99fea..2a8202d 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, List, Optional, Tuple, Union from matemat.exceptions import AuthenticationError from matemat.webserver import pagelet @@ -8,7 +8,12 @@ from matemat.db import MatematDatabase @pagelet('/touchkey') -def touchkey_page(method: str, path: str, args: Dict[str, str], session_vars: Dict[str, Any], headers: Dict[str, str]): +def touchkey_page(method: str, + path: str, + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + session_vars: Dict[str, Any], + headers: Dict[str, str])\ + -> Tuple[int, Optional[Union[str, bytes]]]: if 'user' in session_vars: headers['Location'] = '/' return 301, bytes() @@ -37,12 +42,19 @@ def touchkey_page(method: str, path: str, args: Dict[str, str], session_vars: Di ''' return 200, data.format(username=args['username'] if 'username' in args else '') elif method == 'POST': + if 'username' not in args or not isinstance(args['username'], str): + return 400, None + if 'touchkey' not in args or not isinstance(args['touchkey'], str): + return 400, None + username: str = args['username'] + touchkey: str = args['touchkey'] with MatematDatabase('test.db') as db: try: - user: User = db.login(args['username'], touchkey=args['touchkey']) + user: User = db.login(username, touchkey=touchkey) except AuthenticationError: headers['Location'] = f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' return 301, bytes() session_vars['user'] = user headers['Location'] = '/' return 301, None + return 405, None diff --git a/matemat/webserver/pagelets/upload_test.py b/matemat/webserver/pagelets/upload_test.py new file mode 100644 index 0000000..a6f1e85 --- /dev/null +++ b/matemat/webserver/pagelets/upload_test.py @@ -0,0 +1,28 @@ + +from typing import Any, Dict, Union + +from matemat.webserver import pagelet + + +@pagelet('/upload') +def upload_test(method: str, + path: str, + args: Dict[str, Union[str, bytes]], + session_vars: Dict[str, Any], + headers: Dict[str, str]): + if method == 'GET': + return 200, ''' + + + +
    + + + +
    + + + ''' + else: + headers['Content-Type'] = 'text/plain' + return 200, args.items().__str__() diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index de0daf6..b96767e 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, Tuple, Union +from typing import Any, Callable, Dict, List, Tuple, Union import unittest.mock from io import BytesIO @@ -31,8 +31,8 @@ class HttpResponse: 'Content-Length': 0 } self.pagelet: str = None - # The response body. Only UTF-8 strings are supported - self.body: str = '' + # The response body + self.body: bytes = bytes() # Parsing phase, one of 'begin', 'hdr', 'body' or 'done' self.parse_phase = 'begin' # Buffer for uncompleted lines @@ -55,7 +55,7 @@ class HttpResponse: return # If in the body phase, simply decode and append to the body, while the body is not complete yet elif self.parse_phase == 'body': - self.body += fragment.decode('utf-8') + self.body += fragment if len(self.body) >= int(self.headers['Content-Length']): self.__finalize() return @@ -66,24 +66,24 @@ class HttpResponse: if not fragment.endswith(b'\r\n'): # Special treatment for no trailing CR-LF: Add remainder to buffer head, tail = fragment.rsplit(b'\r\n', 1) - data: str = (self.buffer + head).decode('utf-8') + data: bytes = (self.buffer + head) self.buffer = tail else: - data: str = (self.buffer + fragment).decode('utf-8') + data: bytes = (self.buffer + fragment) self.buffer = bytes() # Iterate the lines that are ready to be parsed - for line in data.split('\r\n'): + for line in data.split(b'\r\n'): # The 'begin' phase indicates that the parser is waiting for the HTTP status line if self.parse_phase == 'begin': - if line.startswith('HTTP/'): + if line.startswith(b'HTTP/'): # Parse the statuscode and advance to header parsing - _, statuscode, _ = line.split(' ', 2) + _, statuscode, _ = line.decode('utf-8').split(' ', 2) self.statuscode = int(statuscode) self.parse_phase = 'hdr' elif self.parse_phase == 'hdr': # Parse a header line and add it to the header dict if len(line) > 0: - k, v = line.split(':', 1) + k, v = line.decode('utf-8').split(':', 1) self.headers[k.strip()] = v.strip() else: # Empty line separates header from body @@ -156,12 +156,16 @@ class MockSocket(bytes): def test_pagelet(path: str): - def with_testing_headers(fun: Callable[[str, str, Dict[str, str], Dict[str, Any], Dict[str, str]], + def with_testing_headers(fun: Callable[[str, + str, + Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + Dict[str, Any], + Dict[str, str]], Tuple[int, Union[bytes, str]]]): @pagelet(path) def testing_wrapper(method: str, path: str, - args: Dict[str, str], + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], session_vars: Dict[str, Any], headers: Dict[str, str]): status, body = fun(method, path, args, session_vars, headers) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index ad99247..511c6e3 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -1,14 +1,16 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple,Union from matemat.webserver.httpd import HttpHandler from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet +import codecs + @test_pagelet('/just/testing/post') def post_test_pagelet(method: str, path: str, - args: Dict[str, str], + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], session_vars: Dict[str, Any], headers: Dict[str, str]): """ @@ -16,8 +18,13 @@ def post_test_pagelet(method: str, """ headers['Content-Type'] = 'text/plain' dump: str = '' - for k, v in args.items(): - dump += f'{k}: {v if isinstance(v, str) else ",".join(v)}\n' + for k, (t, v) in args.items(): + if t.startswith('text/'): + if isinstance(v, bytes): + v = v.decode('utf-8') + dump += f'{k}: {",".join(v) if isinstance(v, list) else v}\n' + else: + dump += f'{k}: {codecs.encode(v, "hex").decode("utf-8")}\n' return 200, dump @@ -26,7 +33,7 @@ class TestPost(AbstractHttpdTest): Test cases for the content serving of the web server. """ - def test_post_get_only_args(self): + def test_post_urlenc_get_only_args(self): """ Test a POST request that only contains GET arguments. """ @@ -38,17 +45,17 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['test']) - def test_post_post_only_args(self): + def test_post_urlenc_post_only_args(self): """ Test a POST request that only contains POST arguments (urlencoded). """ @@ -61,17 +68,17 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed self.assertEqual('bar', kv['foo']) self.assertEqual('1', kv['test']) - def test_post_mixed_args(self): + def test_post_urlenc_mixed_args(self): """ Test that mixed POST and GET args are properly parsed, and that POST takes precedence over GET. """ @@ -84,10 +91,10 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed @@ -95,7 +102,7 @@ class TestPost(AbstractHttpdTest): self.assertEqual('1', kv['gettest']) self.assertEqual('2', kv['posttest']) - def test_post_get_array(self): + def test_post_urlenc_get_array(self): """ Test a POST request that contains GET array arguments. """ @@ -107,17 +114,17 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed self.assertListEqual(['bar', 'baz'], kv['foo']) self.assertEqual('1', kv['test']) - def test_post_post_array(self): + def test_post_urlenc_post_array(self): """ Test a POST request that contains POST array arguments. """ @@ -130,17 +137,17 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed self.assertListEqual(['bar', 'baz'], kv['foo']) self.assertEqual('1', kv['test']) - def test_post_mixed_array(self): + def test_post_urlenc_mixed_array(self): """ Test a POST request that contains both GET and POST array arguments. """ @@ -153,13 +160,85 @@ class TestPost(AbstractHttpdTest): packet = self.client_sock.get_response() # Parse response body - lines: List[str] = packet.body.split('\n')[:-1] + lines: List[bytes] = packet.body.split(b'\n')[:-1] kv: Dict[str, str] = dict() for l in lines: - k, v = l.split(':', 1) + k, v = l.decode('utf-8').split(':', 1) kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') # Make sure the arguments were properly parsed self.assertListEqual(['postbar', 'postbaz'], kv['foo']) self.assertListEqual(['1', '42'], kv['gettest']) self.assertListEqual(['1', '2'], kv['posttest']) + + def test_post_no_body(self): + """ + Test a POST request that contains no headers or body. + """ + # Send POST request + self.client_sock.set_request(b'POST /just/testing/post?foo=bar HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + # Make sure a 400 Bad Request is returned + self.assertEqual(400, packet.statuscode) + + def test_post_multipart_post_only(self): + """ + Test a POST request with a miltipart/form-data body. + """ + # Send POST request + formdata = (b'------testboundary\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'------testboundary\r\n' + b'Content-Disposition: form-data; name="bar"; filename="foo.bar"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x80\x0b\x0c\x73\x0e\x0f\r\n' + b'------testboundary--\r\n') + + self.client_sock.set_request(f'POST /just/testing/post HTTP/1.1\r\n' + f'Content-Type: multipart/form-data; boundary=----testboundary\r\n' + f'Content-Length: {len(formdata)}\r\n\r\n'.encode('utf-8') + formdata) + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + lines: List[bytes] = packet.body.split(b'\n')[:-1] + kv: Dict[str, Any] = dict() + for l in lines: + k, v = l.split(b':', 1) + kv[k.decode('utf-8').strip()] = v.strip() + self.assertIn('foo', kv) + self.assertIn('bar', kv) + self.assertEqual(kv['foo'], b'Hello, World!') + self.assertEqual(kv['bar'], b'00010203040506070809800b0c730e0f') + + def test_post_multipart_mixed(self): + """ + Test a POST request with a miltipart/form-data body. + """ + # Send POST request + formdata = (b'------testboundary\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'------testboundary\r\n' + b'Content-Disposition: form-data; name="bar"; filename="foo.bar"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x80\x0b\x0c\x73\x0e\x0f\r\n' + b'------testboundary--\r\n') + + self.client_sock.set_request(f'POST /just/testing/post?getfoo=bar&foo=thisshouldbegone HTTP/1.1\r\n' + f'Content-Type: multipart/form-data; boundary=----testboundary\r\n' + f'Content-Length: {len(formdata)}\r\n\r\n'.encode('utf-8') + formdata) + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + lines: List[bytes] = packet.body.split(b'\n')[:-1] + kv: Dict[str, Any] = dict() + for l in lines: + k, v = l.split(b':', 1) + kv[k.decode('utf-8').strip()] = v.strip() + self.assertIn('foo', kv) + self.assertIn('bar', kv) + self.assertEqual(kv['getfoo'], b'bar') + self.assertEqual(kv['foo'], b'Hello, World!') + self.assertEqual(kv['bar'], b'00010203040506070809800b0c730e0f') diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index f3dc6be..0556764 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, Union import os import os.path @@ -10,7 +10,7 @@ from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_p @test_pagelet('/just/testing/serve_pagelet_ok') def serve_test_pagelet_ok(method: str, path: str, - args: Dict[str, str], + args: Dict[str, Union[bytes, str]], session_vars: Dict[str, Any], headers: Dict[str, str]): headers['Content-Type'] = 'text/plain' @@ -20,7 +20,7 @@ def serve_test_pagelet_ok(method: str, @test_pagelet('/just/testing/serve_pagelet_fail') def serve_test_pagelet_fail(method: str, path: str, - args: Dict[str, str], + args: Dict[str, Union[bytes, str]], session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' @@ -54,7 +54,7 @@ class TestServe(AbstractHttpdTest): self.assertEqual('serve_test_pagelet_ok', packet.pagelet) # Make sure the expected content is served self.assertEqual(200, packet.statuscode) - self.assertEqual('serve test pagelet ok', packet.body) + self.assertEqual(b'serve test pagelet ok', packet.body) def test_serve_pagelet_fail(self): # Call the test pagelet that produces a 500 Internal Server Error result @@ -66,7 +66,7 @@ class TestServe(AbstractHttpdTest): self.assertEqual('serve_test_pagelet_fail', packet.pagelet) # Make sure the expected content is served self.assertEqual(500, packet.statuscode) - self.assertEqual('serve test pagelet fail', packet.body) + self.assertEqual(b'serve test pagelet fail', packet.body) def test_serve_static_ok(self): # Request a static resource @@ -78,7 +78,7 @@ class TestServe(AbstractHttpdTest): self.assertIsNone(packet.pagelet) # Make sure the expected content is served self.assertEqual(200, packet.statuscode) - self.assertEqual('static resource test', packet.body) + self.assertEqual(b'static resource test', packet.body) def test_serve_static_forbidden(self): # Request a static resource with lacking permissions @@ -90,7 +90,7 @@ class TestServe(AbstractHttpdTest): self.assertIsNone(packet.pagelet) # Make sure a 403 header is served self.assertEqual(403, packet.statuscode) - self.assertNotEqual('This should not be readable', packet.body) + self.assertNotEqual(b'This should not be readable', packet.body) def test_serve_not_found(self): # Request a nonexistent resource @@ -116,7 +116,10 @@ class TestServe(AbstractHttpdTest): def test_static_post_not_allowed(self): # Request a resource outside the webroot - self.client_sock.set_request(b'POST /iwanttouploadthis HTTP/1.1\r\n\r\nq=this%20should%20not%20be%20uploaded') + self.client_sock.set_request(b'POST /iwanttopostthis HTTP/1.1\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n' + b'Content-length: 37\r\n\r\n' + b'q=this%20should%20not%20be%20uploaded') HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index b8e21cf..50ade85 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -1,5 +1,5 @@ -from typing import Any, Dict +from typing import Any, Dict, Union from datetime import datetime, timedelta from time import sleep @@ -11,7 +11,7 @@ from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_p @test_pagelet('/just/testing/sessions') def session_test_pagelet(method: str, path: str, - args: Dict[str, str], + args: Dict[str, Union[bytes, str]], session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py new file mode 100644 index 0000000..931f759 --- /dev/null +++ b/matemat/webserver/util.py @@ -0,0 +1,118 @@ + +from typing import Dict, List, Tuple, Optional, Union + +import urllib.parse + + +def _parse_multipart(body: bytes, boundary: str) -> Dict[str, List[Tuple[str, Union[bytes, str]]]]: + """ + Given a HTTP body with form-data in multipart form, and the multipart-boundary, parse the multipart items and + return them as a dictionary. + + :param body: The HTTP multipart/form-data body. + :param boundary: The multipart boundary. + :return: A dictionary of field names as key, and content types and field values as value. + """ + # Generate item header boundary and terminating boundary from general boundary string + _boundary = f'\r\n--{boundary}\r\n'.encode('utf-8') + _end_boundary = f'\r\n--{boundary}--\r\n'.encode('utf-8') + # Split at the end boundary and make sure there comes nothing after it + allparts = body.split(_end_boundary, 1) + if len(allparts) != 2 or allparts[1] != b'': + raise ValueError('Last boundary missing or corrupted') + # Split remaining body into its parts (appending a CRLF for the first boundary to match), and verify at least 1 part + # is there + parts: List[bytes] = (b'\r\n' + allparts[0]).split(_boundary) + if len(parts) < 1 or parts[0] != b'': + raise ValueError('First boundary missing or corrupted') + # Remove the first, empty part + parts = parts[1:] + + # Results go into this dict + args: Dict[str, List[Tuple[str, Union[bytes, str]]]] = dict() + + # Parse each multipart part + for part in parts: + # Parse multipart headers + hdr: Dict[str, str] = dict() + while True: + head, part = part.split(b'\r\n', 1) + # Break on header/body delimiter + if head == b'': + break + # Add header to hdr dict + hk, hv = head.decode('utf-8').split(':') + hdr[hk.strip()] = hv.strip() + # At least Content-Type and Content-Disposition must be present + if 'Content-Type' not in hdr or 'Content-Disposition' not in hdr: + raise ValueError('Missing Content-Type or Content-Disposition header') + # Extract Content-Disposition header value and its arguments + cd, *cdargs = hdr['Content-Disposition'].split(';') + # Content-Disposition MUST be form-data; everything else is rejected + if cd.strip() != 'form-data': + raise ValueError(f'Unknown Content-Disposition: cd') + # Extract the "name" header argument + for cdarg in cdargs: + k, v = cdarg.split('=', 1) + if k.strip() == 'name': + name: str = v.strip() + # Remove quotation marks around the name value + if name.startswith('"') and name.endswith('"'): + name = v[1:-1] + # Add the Content-Type and the content to the header, with the provided name + if name not in args: + args[name] = list() + args[name].append((hdr['Content-Type'].strip(), part)) + + return args + + +def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 'text/plain') \ + -> Tuple[str, Dict[str, Tuple[str, Union[bytes, str, List[str]]]]]: + """ + Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded or + multipart/form-data form, parse the arguments and return them as a dictionary. + + If a key is used both in GET and in POST, the POST value takes precedence, and the GET value is discarded. + + :param request: The request string to parse. + :param postbody: The POST body to parse, defaults to None. + :param enctype: Encoding of the POST body; supported values are application/x-www-form-urlencoded and + multipart/form-data. + :return: A tuple consisting of the base path and a dictionary with the parsed key/value pairs, and the value's + content type. + """ + # Parse the request "URL" (i.e. only the path) + tokens = urllib.parse.urlparse(request) + # Parse the GET arguments + getargs = urllib.parse.parse_qs(tokens.query) + + # TODO: { 'foo': [ ('text/plain', 'bar'), ('application/octet-stream', '\x80') ] } + # TODO: Use a @dataclass once Python 3.7 is out + args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]] = dict() + for k, v in getargs.items(): + args[k] = 'text/plain', v + + if postbody is not None: + if enctype == 'application/x-www-form-urlencoded': + # Parse the POST body + postargs = urllib.parse.parse_qs(postbody.decode('utf-8')) + # Write all POST values into the dict, overriding potential duplicates from GET + for k, v in postargs.items(): + args[k] = 'text/plain', v + elif enctype.startswith('multipart/form-data'): + # Parse the multipart boundary from the Content-Type header + boundary: str = enctype.split('boundary=')[1] + # Parse the multipart body + mpargs = _parse_multipart(postbody, boundary) + for k, v in mpargs.items(): + # TODO: Process all values, not just the first + args[k] = v[0] + else: + raise ValueError(f'Unsupported Content-Type: {enctype}') + # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values + for (k, (ct, v)) in args.items(): + if len(v) == 1: + args[k] = ct, v[0] + # Return the path and the parsed arguments + return tokens.path, args From 5bb1dfad2176d5ae0c8325024b1673eb45e9c944 Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 27 Jun 2018 21:20:36 +0200 Subject: [PATCH 24/46] Fixed a style error --- matemat/webserver/test/test_post.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 511c6e3..0c6e3d2 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -1,5 +1,5 @@ -from typing import Any, Dict, List, Tuple,Union +from typing import Any, Dict, List, Tuple, Union from matemat.webserver.httpd import HttpHandler from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet From 118de8bf95f344ea0c567040334a229d089544f6 Mon Sep 17 00:00:00 2001 From: s3lph Date: Thu, 28 Jun 2018 23:58:01 +0200 Subject: [PATCH 25/46] New request parsing (WIP: Documentation) --- matemat/webserver/__init__.py | 1 + matemat/webserver/httpd.py | 14 +- matemat/webserver/pagelets/__init__.py | 1 - matemat/webserver/pagelets/login.py | 16 +- matemat/webserver/pagelets/logout.py | 4 +- matemat/webserver/pagelets/main.py | 4 +- matemat/webserver/pagelets/touchkey.py | 14 +- matemat/webserver/pagelets/upload_test.py | 28 -- matemat/webserver/requestargs.py | 121 ++++++ matemat/webserver/test/abstract_httpd_test.py | 6 +- matemat/webserver/test/test_parse_request.py | 347 ++++++++++++++++++ matemat/webserver/test/test_post.py | 84 ++--- matemat/webserver/test/test_requestargs.py | 204 ++++++++++ matemat/webserver/test/test_serve.py | 6 +- matemat/webserver/test/test_session.py | 4 +- matemat/webserver/util.py | 64 ++-- 16 files changed, 774 insertions(+), 144 deletions(-) delete mode 100644 matemat/webserver/pagelets/upload_test.py create mode 100644 matemat/webserver/requestargs.py create mode 100644 matemat/webserver/test/test_parse_request.py create mode 100644 matemat/webserver/test/test_requestargs.py diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index f4d86f3..1b4ab06 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -6,4 +6,5 @@ API that can be used by 'pagelets' - single pages of a web service. If a reques server will attempt to serve the request with a static resource in a previously configured webroot directory. """ +from .requestargs import RequestArgument from .httpd import MatematWebserver, HttpHandler, pagelet diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index a4e9cca..79efb98 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import Any, Callable, Dict, Tuple, Union import traceback @@ -13,6 +13,7 @@ from uuid import uuid4 from datetime import datetime, timedelta from matemat import __version__ as matemat_version +from matemat.webserver import RequestArgument from matemat.webserver.util import parse_args @@ -30,7 +31,7 @@ BaseHTTPRequestHandler.log_error = lambda self, fstring='', *args: None # Dictionary to hold registered pagelet paths and their handler functions _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) str, # Request path - Dict[str, Tuple[str, Union[bytes, str, List[str]]]], # args: (name, (type, value)) + Dict[str, RequestArgument], # args: (name, argument) Dict[str, Any], # Session vars Dict[str, str]], # Response headers Tuple[int, Union[bytes, str]]]] = dict() # Returns: (status code, response body) @@ -50,15 +51,14 @@ def pagelet(path: str): (method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]) -> (int, Optional[Union[str, bytes]]) method: The HTTP method (GET, POST) that was used. path: The path that was requested. - args: The arguments that were passed with the request (as GET or POST arguments), each of which may be - either a str or bytes object, or a list of str. + args: The arguments that were passed with the request (as GET or POST arguments). session_vars: The session storage. May be read from and written to. headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. returns: A tuple consisting of the HTTP status code (as an int) and the response body (as str or bytes, @@ -69,7 +69,7 @@ def pagelet(path: str): def http_handler(fun: Callable[[str, str, - Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + Dict[str, RequestArgument], Dict[str, Any], Dict[str, str]], Tuple[int, Union[bytes, str]]]): @@ -181,7 +181,7 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] - def _handle(self, method: str, path: str, args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]]) -> None: + def _handle(self, method: str, path: str, args: Dict[str, RequestArgument]) -> None: """ Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. diff --git a/matemat/webserver/pagelets/__init__.py b/matemat/webserver/pagelets/__init__.py index 71ded5e..9b926d6 100644 --- a/matemat/webserver/pagelets/__init__.py +++ b/matemat/webserver/pagelets/__init__.py @@ -8,4 +8,3 @@ from .main import main_page from .login import login_page from .logout import logout from .touchkey import touchkey_page -from .upload_test import upload_test diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index 8fbe831..f7813b4 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -1,8 +1,8 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union from matemat.exceptions import AuthenticationError -from matemat.webserver import pagelet +from matemat.webserver import pagelet, RequestArgument from matemat.primitives import User from matemat.db import MatematDatabase @@ -10,7 +10,7 @@ from matemat.db import MatematDatabase @pagelet('/login') def login_page(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: @@ -43,15 +43,11 @@ def login_page(method: str, ''' return 200, data.format(msg=args['msg'] if 'msg' in args else '') elif method == 'POST': - if 'username' not in args or not isinstance(args['username'], str): - return 400, None - if 'password' not in args or not isinstance(args['password'], str): - return 400, None - username: str = args['username'] - password: str = args['password'] + username: RequestArgument = args['username'] + password: RequestArgument = args['password'] with MatematDatabase('test.db') as db: try: - user: User = db.login(username, password) + user: User = db.login(username.get_str(), password.get_str()) except AuthenticationError: headers['Location'] = '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' return 301, bytes() diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index 53a292a..b70d7c1 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -1,13 +1,13 @@ from typing import Any, Dict, List, Optional, Tuple, Union -from matemat.webserver import pagelet +from matemat.webserver import pagelet, RequestArgument @pagelet('/logout') def logout(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index d2dd208..2b9ce79 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union -from matemat.webserver import MatematWebserver, pagelet +from matemat.webserver import MatematWebserver, pagelet, RequestArgument from matemat.primitives import User from matemat.db import MatematDatabase @@ -9,7 +9,7 @@ from matemat.db import MatematDatabase @pagelet('/') def main_page(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 2a8202d..22e3df4 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from matemat.exceptions import AuthenticationError -from matemat.webserver import pagelet +from matemat.webserver import pagelet, RequestArgument from matemat.primitives import User from matemat.db import MatematDatabase @@ -10,7 +10,7 @@ from matemat.db import MatematDatabase @pagelet('/touchkey') def touchkey_page(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: @@ -42,15 +42,11 @@ def touchkey_page(method: str, ''' return 200, data.format(username=args['username'] if 'username' in args else '') elif method == 'POST': - if 'username' not in args or not isinstance(args['username'], str): - return 400, None - if 'touchkey' not in args or not isinstance(args['touchkey'], str): - return 400, None - username: str = args['username'] - touchkey: str = args['touchkey'] + username: RequestArgument = args['username'] + touchkey: RequestArgument = args['touchkey'] with MatematDatabase('test.db') as db: try: - user: User = db.login(username, touchkey=touchkey) + user: User = db.login(username.get_str(), touchkey=touchkey.get_str()) except AuthenticationError: headers['Location'] = f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' return 301, bytes() diff --git a/matemat/webserver/pagelets/upload_test.py b/matemat/webserver/pagelets/upload_test.py deleted file mode 100644 index a6f1e85..0000000 --- a/matemat/webserver/pagelets/upload_test.py +++ /dev/null @@ -1,28 +0,0 @@ - -from typing import Any, Dict, Union - -from matemat.webserver import pagelet - - -@pagelet('/upload') -def upload_test(method: str, - path: str, - args: Dict[str, Union[str, bytes]], - session_vars: Dict[str, Any], - headers: Dict[str, str]): - if method == 'GET': - return 200, ''' - - - -
    - - - -
    - - - ''' - else: - headers['Content-Type'] = 'text/plain' - return 200, args.items().__str__() diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py new file mode 100644 index 0000000..a35f759 --- /dev/null +++ b/matemat/webserver/requestargs.py @@ -0,0 +1,121 @@ + +from typing import List, Optional, Tuple, Union + + +class RequestArgument(object): + + def __init__(self, + name: str, + value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = None) -> None: + self.__name: str = name + self.__value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = None + if value is None: + self.__value = [] + else: + if isinstance(value, list): + if len(value) == 1: + self.__value = value[0] + else: + self.__value = value + else: + self.__value = value + + @property + def is_array(self) -> bool: + return isinstance(self.__value, list) + + @property + def is_scalar(self) -> bool: + return not isinstance(self.__value, list) + + @property + def is_view(self) -> bool: + return False + + @property + def name(self) -> str: + return self.__name + + def get_str(self, index: int = None) -> Optional[str]: + if self.is_array: + if index is None: + raise ValueError('index must not be None') + v: Tuple[str, Union[bytes, str]] = self.__value[index] + if isinstance(v[1], str): + return v[1] + elif isinstance(v[1], bytes): + return v[1].decode('utf-8') + else: + if index is not None: + raise ValueError('index must be None') + if isinstance(self.__value[1], str): + return self.__value[1] + elif isinstance(self.__value[1], bytes): + return self.__value[1].decode('utf-8') + + def get_bytes(self, index: int = None) -> Optional[bytes]: + if self.is_array: + if index is None: + raise ValueError('index must not be None') + v: Tuple[str, Union[bytes, str]] = self.__value[index] + if isinstance(v[1], bytes): + return v[1] + elif isinstance(v[1], str): + return v[1].encode('utf-8') + else: + if index is not None: + raise ValueError('index must be None') + if isinstance(self.__value[1], bytes): + return self.__value[1] + elif isinstance(self.__value[1], str): + return self.__value[1].encode('utf-8') + + def get_content_type(self, index: int = None) -> Optional[str]: + if self.is_array: + if index is None: + raise ValueError('index must not be None') + v: Tuple[str, Union[bytes, str]] = self.__value[index] + return v[0] + else: + if index is not None: + raise ValueError('index must be None') + return self.__value[0] + + def append(self, ctype: str, value: Union[str, bytes]): + if self.is_view: + raise TypeError('A RequestArgument view is immutable!') + if len(self) == 0: + self.__value = ctype, value + else: + if self.is_scalar: + self.__value = [self.__value] + self.__value.append((ctype, value)) + + def __len__(self): + return len(self.__value) if self.is_array else 1 + + def __iter__(self): + if self.is_scalar: + yield _View(self.__name, self.__value) + else: + # Typing helper + _value: List[Tuple[str, Union[bytes, str]]] = self.__value + for v in _value: + yield _View(self.__name, v) + + def __getitem__(self, index: Union[int, slice]): + if self.is_scalar: + if index == 0: + return _View(self.__name, self.__value) + raise ValueError('Scalar RequestArgument only indexable with 0') + return _View(self.__name, self.__value[index]) + + +class _View(RequestArgument): + + def __init__(self, name: str, value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]]): + super().__init__(name, value) + + @property + def is_view(self) -> bool: + return True diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index b96767e..daa1126 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -9,7 +9,7 @@ from abc import ABC from datetime import datetime from http.server import HTTPServer -from matemat.webserver.httpd import pagelet +from matemat.webserver import pagelet, RequestArgument class HttpResponse: @@ -158,14 +158,14 @@ def test_pagelet(path: str): def with_testing_headers(fun: Callable[[str, str, - Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + Dict[str, RequestArgument], Dict[str, Any], Dict[str, str]], Tuple[int, Union[bytes, str]]]): @pagelet(path) def testing_wrapper(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]): status, body = fun(method, path, args, session_vars, headers) diff --git a/matemat/webserver/test/test_parse_request.py b/matemat/webserver/test/test_parse_request.py new file mode 100644 index 0000000..a533936 --- /dev/null +++ b/matemat/webserver/test/test_parse_request.py @@ -0,0 +1,347 @@ + +import unittest + +from matemat.webserver.util import parse_args + + +class TestParseRequest(unittest.TestCase): + + def test_parse_get_root(self): + path, args = parse_args('/') + self.assertEqual('/', path) + self.assertEqual(0, len(args)) + + def test_parse_get_no_args(self): + path, args = parse_args('/index.html') + self.assertEqual('/index.html', path) + self.assertEqual(0, len(args)) + + def test_parse_get_root_getargs(self): + path, args = parse_args('/?foo=42&bar=1337&baz=Hello,%20World!') + self.assertEqual('/', path) + self.assertEqual(3, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual('text/plain', args['foo'].get_content_type()) + self.assertEqual('text/plain', args['bar'].get_content_type()) + self.assertEqual('text/plain', args['baz'].get_content_type()) + self.assertEqual('42', args['foo'].get_str()) + self.assertEqual('1337', args['bar'].get_str()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + + def test_parse_get_getargs(self): + path, args = parse_args('/abc/def?foo=42&bar=1337&baz=Hello,%20World!') + self.assertEqual('/abc/def', path) + self.assertEqual(3, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual('text/plain', args['foo'].get_content_type()) + self.assertEqual('text/plain', args['bar'].get_content_type()) + self.assertEqual('text/plain', args['baz'].get_content_type()) + self.assertEqual('42', args['foo'].get_str()) + self.assertEqual('1337', args['bar'].get_str()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + + def test_parse_get_getarray(self): + path, args = parse_args('/abc/def?foo=42&foo=1337&baz=Hello,%20World!') + self.assertEqual('/abc/def', path) + self.assertEqual(2, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_array) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual(2, len(args['foo'])) + self.assertEqual('42', args['foo'].get_str(0)) + self.assertEqual('1337', args['foo'].get_str(1)) + + def test_parse_get_zero_arg(self): + path, args = parse_args('/abc/def?foo=&bar=42') + self.assertEqual(2, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertEqual(1, len(args['foo'])) + self.assertEqual('', args['foo'].get_str()) + self.assertEqual('42', args['bar'].get_str()) + + def test_parse_get_urlencoded_encoding_fail(self): + with self.assertRaises(ValueError): + parse_args('/?foo=42&bar=%80&baz=Hello,%20World!') + + def test_parse_post_urlencoded(self): + path, args = parse_args('/', + postbody=b'foo=42&bar=1337&baz=Hello,%20World!', + enctype='application/x-www-form-urlencoded') + self.assertEqual('/', path) + self.assertEqual(3, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual('text/plain', args['foo'].get_content_type()) + self.assertEqual('text/plain', args['bar'].get_content_type()) + self.assertEqual('text/plain', args['baz'].get_content_type()) + self.assertEqual('42', args['foo'].get_str()) + self.assertEqual('1337', args['bar'].get_str()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + + def test_parse_post_urlencoded_array(self): + path, args = parse_args('/', + postbody=b'foo=42&foo=1337&baz=Hello,%20World!', + enctype='application/x-www-form-urlencoded') + self.assertEqual('/', path) + self.assertEqual(2, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_array) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual(2, len(args['foo'])) + self.assertEqual('42', args['foo'].get_str(0)) + self.assertEqual('1337', args['foo'].get_str(1)) + + def test_parse_post_urlencoded_zero_arg(self): + path, args = parse_args('/abc/def', postbody=b'foo=&bar=42', enctype='application/x-www-form-urlencoded') + self.assertEqual(2, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertEqual(1, len(args['foo'])) + self.assertEqual('', args['foo'].get_str()) + self.assertEqual('42', args['bar'].get_str()) + + def test_parse_post_urlencoded_encoding_fail(self): + with self.assertRaises(ValueError): + parse_args('/', + postbody=b'foo=42&bar=%80&baz=Hello,%20World!', + enctype='application/x-www-form-urlencoded') + + def test_parse_post_multipart_no_args(self): + path, args = parse_args('/', + postbody=b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + self.assertEqual('/', path) + self.assertEqual(0, len(args)) + + def test_parse_post_multipart(self): + path, args = parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + self.assertEqual('/', path) + self.assertEqual(3, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertIn('baz', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual('text/plain', args['foo'].get_content_type()) + self.assertEqual('application/octet-stream', args['bar'].get_content_type()) + self.assertEqual('text/plain', args['baz'].get_content_type()) + self.assertEqual('42', args['foo'].get_str()) + self.assertEqual(b'1337', args['bar'].get_bytes()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + + def test_parse_post_multipart_zero_arg(self): + path, args = parse_args('/abc/def', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + self.assertEqual(2, len(args)) + self.assertIn('foo', args.keys()) + self.assertIn('bar', args.keys()) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertEqual(1, len(args['foo'])) + self.assertEqual('', args['foo'].get_str()) + self.assertEqual('42', args['bar'].get_str()) + + def test_parse_post_multipart_broken_boundaries(self): + with self.assertRaises(ValueError): + # Boundary not defined in Content-Type + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data') + with self.assertRaises(ValueError): + # Corrupted "--" head at first boundary + parse_args('/', + postbody=b'-+testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + with self.assertRaises(ValueError): + # Missing "--" tail at end boundary + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + with self.assertRaises(ValueError): + # Missing Content-Type header in one part + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + with self.assertRaises(ValueError): + # Missing Content-Disposition header in one part + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + with self.assertRaises(ValueError): + # Missing form-data name argument + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + with self.assertRaises(ValueError): + # Unknown Content-Disposition + parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: attachment; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + + def test_get_post_precedence_urlencoded(self): + path, args = parse_args('/foo?foo=thisshouldnotbethere&bar=isurvived', + postbody=b'foo=42&foo=1337&baz=Hello,%20World!', + enctype='application/x-www-form-urlencoded') + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) + self.assertEqual(2, len(args['foo'])) + self.assertEqual(1, len(args['bar'])) + self.assertEqual(1, len(args['baz'])) + self.assertEqual('42', args['foo'].get_str(0)) + self.assertEqual('1337', args['foo'].get_str(1)) + self.assertEqual('isurvived', args['bar'].get_str()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + + def test_get_post_precedence_multipart(self): + path, args = parse_args('/foo?foo=thisshouldnotbethere&bar=isurvived', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"; filename="bar.bin"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) + self.assertEqual(2, len(args['foo'])) + self.assertEqual(1, len(args['bar'])) + self.assertEqual(1, len(args['baz'])) + self.assertEqual('42', args['foo'].get_str(0)) + self.assertEqual('1337', args['foo'].get_str(1)) + self.assertEqual('isurvived', args['bar'].get_str()) + self.assertEqual('Hello, World!', args['baz'].get_str()) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 0c6e3d2..9b2fe22 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Tuple, Union -from matemat.webserver.httpd import HttpHandler +from matemat.webserver import HttpHandler, RequestArgument from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet import codecs @@ -10,7 +10,7 @@ import codecs @test_pagelet('/just/testing/post') def post_test_pagelet(method: str, path: str, - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]): """ @@ -18,13 +18,12 @@ def post_test_pagelet(method: str, """ headers['Content-Type'] = 'text/plain' dump: str = '' - for k, (t, v) in args.items(): - if t.startswith('text/'): - if isinstance(v, bytes): - v = v.decode('utf-8') - dump += f'{k}: {",".join(v) if isinstance(v, list) else v}\n' - else: - dump += f'{k}: {codecs.encode(v, "hex").decode("utf-8")}\n' + for k, ra in args.items(): + for a in ra: + if a.get_content_type().startswith('text/'): + dump += f'{k}: {a.get_str()}\n' + else: + dump += f'{k}: {codecs.encode(a.get_bytes(), "hex").decode("utf-8")}\n' return 200, dump @@ -118,10 +117,14 @@ class TestPost(AbstractHttpdTest): kv: Dict[str, str] = dict() for l in lines: k, v = l.decode('utf-8').split(':', 1) - kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') - + k = k.strip() + v = v.strip() + if k in kv: + kv[k] += f',{v}' + else: + kv[k] = v # Make sure the arguments were properly parsed - self.assertListEqual(['bar', 'baz'], kv['foo']) + self.assertEqual('bar,baz', kv['foo']) self.assertEqual('1', kv['test']) def test_post_urlenc_post_array(self): @@ -141,10 +144,14 @@ class TestPost(AbstractHttpdTest): kv: Dict[str, str] = dict() for l in lines: k, v = l.decode('utf-8').split(':', 1) - kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') - + k = k.strip() + v = v.strip() + if k in kv: + kv[k] += f',{v}' + else: + kv[k] = v # Make sure the arguments were properly parsed - self.assertListEqual(['bar', 'baz'], kv['foo']) + self.assertEqual('bar,baz', kv['foo']) self.assertEqual('1', kv['test']) def test_post_urlenc_mixed_array(self): @@ -164,12 +171,16 @@ class TestPost(AbstractHttpdTest): kv: Dict[str, str] = dict() for l in lines: k, v = l.decode('utf-8').split(':', 1) - kv[k.strip()] = v.strip() if ',' not in v else v.strip().split(',') - + k = k.strip() + v = v.strip() + if k in kv: + kv[k] += f',{v}' + else: + kv[k] = v # Make sure the arguments were properly parsed - self.assertListEqual(['postbar', 'postbaz'], kv['foo']) - self.assertListEqual(['1', '42'], kv['gettest']) - self.assertListEqual(['1', '2'], kv['posttest']) + self.assertEqual('postbar,postbaz', kv['foo']) + self.assertEqual('1,42', kv['gettest']) + self.assertEqual('1,2', kv['posttest']) def test_post_no_body(self): """ @@ -184,7 +195,7 @@ class TestPost(AbstractHttpdTest): def test_post_multipart_post_only(self): """ - Test a POST request with a miltipart/form-data body. + Test a POST request with a miutipart/form-data body. """ # Send POST request formdata = (b'------testboundary\r\n' @@ -211,34 +222,3 @@ class TestPost(AbstractHttpdTest): self.assertIn('bar', kv) self.assertEqual(kv['foo'], b'Hello, World!') self.assertEqual(kv['bar'], b'00010203040506070809800b0c730e0f') - - def test_post_multipart_mixed(self): - """ - Test a POST request with a miltipart/form-data body. - """ - # Send POST request - formdata = (b'------testboundary\r\n' - b'Content-Disposition: form-data; name="foo"\r\n' - b'Content-Type: text/plain\r\n\r\n' - b'Hello, World!\r\n' - b'------testboundary\r\n' - b'Content-Disposition: form-data; name="bar"; filename="foo.bar"\r\n' - b'Content-Type: application/octet-stream\r\n\r\n' - b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x80\x0b\x0c\x73\x0e\x0f\r\n' - b'------testboundary--\r\n') - - self.client_sock.set_request(f'POST /just/testing/post?getfoo=bar&foo=thisshouldbegone HTTP/1.1\r\n' - f'Content-Type: multipart/form-data; boundary=----testboundary\r\n' - f'Content-Length: {len(formdata)}\r\n\r\n'.encode('utf-8') + formdata) - HttpHandler(self.client_sock, ('::1', 45678), self.server) - packet = self.client_sock.get_response() - lines: List[bytes] = packet.body.split(b'\n')[:-1] - kv: Dict[str, Any] = dict() - for l in lines: - k, v = l.split(b':', 1) - kv[k.decode('utf-8').strip()] = v.strip() - self.assertIn('foo', kv) - self.assertIn('bar', kv) - self.assertEqual(kv['getfoo'], b'bar') - self.assertEqual(kv['foo'], b'Hello, World!') - self.assertEqual(kv['bar'], b'00010203040506070809800b0c730e0f') diff --git a/matemat/webserver/test/test_requestargs.py b/matemat/webserver/test/test_requestargs.py new file mode 100644 index 0000000..dcdde14 --- /dev/null +++ b/matemat/webserver/test/test_requestargs.py @@ -0,0 +1,204 @@ + +from typing import List + +import unittest + +from matemat.webserver import RequestArgument +# noinspection PyProtectedMember +from matemat.webserver.requestargs import _View + + +class TestRequestArguments(unittest.TestCase): + + def test_create_default(self): + ra = RequestArgument('foo') + self.assertEqual('foo', ra.name) + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + self.assertTrue(ra.is_array) + self.assertFalse(ra.is_view) + + def test_create_str_scalar(self): + ra = RequestArgument('foo', ('text/plain', 'bar')) + self.assertEqual('foo', ra.name) + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + self.assertFalse(ra.is_array) + self.assertEqual('bar', ra.get_str()) + self.assertEqual(b'bar', ra.get_bytes()) + self.assertEqual('text/plain', ra.get_content_type()) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_str(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_bytes(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_content_type(0)) + self.assertFalse(ra.is_view) + + def test_create_str_scalar_array(self): + ra = RequestArgument('foo', [('text/plain', 'bar')]) + self.assertEqual('foo', ra.name) + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + self.assertFalse(ra.is_array) + self.assertEqual('bar', ra.get_str()) + self.assertEqual(b'bar', ra.get_bytes()) + self.assertEqual('text/plain', ra.get_content_type()) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_str(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_bytes(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_content_type(0)) + self.assertFalse(ra.is_view) + + def test_create_bytes_scalar(self): + ra = RequestArgument('foo', ('application/octet-stream', b'\x00\x80\xff\xfe')) + self.assertEqual('foo', ra.name) + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + self.assertFalse(ra.is_array) + with self.assertRaises(UnicodeDecodeError): + ra.get_str() + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes()) + self.assertEqual('application/octet-stream', ra.get_content_type()) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_str(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_bytes(0)) + with self.assertRaises(ValueError): + self.assertEqual('bar', ra.get_content_type(0)) + self.assertFalse(ra.is_view) + + def test_create_array(self): + ra = RequestArgument('foo', [ + ('text/plain', 'bar'), + ('application/octet-stream', b'\x00\x80\xff\xfe') + ]) + self.assertEqual('foo', ra.name) + self.assertEqual(2, len(ra)) + self.assertFalse(ra.is_scalar) + self.assertTrue(ra.is_array) + with self.assertRaises(ValueError): + ra.get_str() + with self.assertRaises(ValueError): + ra.get_bytes() + with self.assertRaises(ValueError): + ra.get_content_type() + self.assertEqual('bar', ra.get_str(0)) + self.assertEqual(b'bar', ra.get_bytes(0)) + self.assertEqual('text/plain', ra.get_content_type(0)) + with self.assertRaises(UnicodeDecodeError): + ra.get_str(1) + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(1)) + self.assertEqual('application/octet-stream', ra.get_content_type(1)) + self.assertFalse(ra.is_view) + + def test_append_empty_str(self): + ra = RequestArgument('foo') + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + + ra.append('text/plain', 'bar') + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + self.assertEqual('bar', ra.get_str()) + self.assertEqual(b'bar', ra.get_bytes()) + self.assertEqual('text/plain', ra.get_content_type()) + self.assertFalse(ra.is_view) + + def test_append_empty_bytes(self): + ra = RequestArgument('foo') + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + + ra.append('application/octet-stream', b'\x00\x80\xff\xfe') + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + with self.assertRaises(UnicodeDecodeError): + ra.get_str() + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes()) + self.assertEqual('application/octet-stream', ra.get_content_type()) + self.assertFalse(ra.is_view) + + def test_append_multiple(self): + ra = RequestArgument('foo') + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + + ra.append('text/plain', 'bar') + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + + ra.append('application/octet-stream', b'\x00\x80\xff\xfe') + self.assertEqual(2, len(ra)) + self.assertFalse(ra.is_scalar) + + ra.append('text/plain', 'Hello, World!') + self.assertEqual(3, len(ra)) + self.assertFalse(ra.is_scalar) + + def test_iterate_empty(self): + ra = RequestArgument('foo') + self.assertEqual(0, len(ra)) + for _ in ra: + self.fail() + + def test_iterate_scalar(self): + ra = RequestArgument('foo', ('text/plain', 'bar')) + self.assertTrue(ra.is_scalar) + count: int = 0 + for it in ra: + self.assertIsInstance(it, _View) + self.assertEqual('foo', it.name) + self.assertTrue(it.is_view) + self.assertTrue(it.is_scalar) + count += 1 + self.assertEqual(1, count) + + def test_iterate_array(self): + ra = RequestArgument('foo', [('text/plain', 'bar'), ('abc', b'def'), ('xyz', '1337')]) + self.assertFalse(ra.is_scalar) + items: List[str] = list() + for it in ra: + self.assertIsInstance(it, _View) + self.assertTrue(it.is_view) + self.assertTrue(it.is_scalar) + items.append(it.get_content_type()) + self.assertEqual(['text/plain', 'abc', 'xyz'], items) + + def test_iterate_sliced(self): + ra = RequestArgument('foo', [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j'), ('k', 'l')]) + self.assertFalse(ra.is_scalar) + items: List[str] = list() + for it in ra[1:4:2]: + self.assertIsInstance(it, _View) + self.assertTrue(it.is_view) + self.assertTrue(it.is_scalar) + items.append(it.get_content_type()) + self.assertEqual(['c', 'g'], items) + + def test_index_scalar(self): + ra = RequestArgument('foo', ('bar', 'baz')) + it = ra[0] + self.assertIsInstance(it, _View) + self.assertEqual('foo', it.name) + self.assertEqual('bar', it.get_content_type()) + self.assertEqual('baz', it.get_str()) + with self.assertRaises(ValueError): + _ = ra[1] + + def test_index_array(self): + ra = RequestArgument('foo', [('a', 'b'), ('c', 'd')]) + it = ra[1] + self.assertIsInstance(it, _View) + self.assertEqual('foo', it.name) + self.assertEqual('c', it.get_content_type()) + self.assertEqual('d', it.get_str()) + + def test_view_immutable(self): + ra = RequestArgument('foo', ('bar', 'baz')) + it = ra[0] + self.assertIsInstance(it, _View) + with self.assertRaises(TypeError): + it.append('foo', 'bar') diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 0556764..7e159e3 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -3,14 +3,14 @@ from typing import Any, Dict, Union import os import os.path -from matemat.webserver.httpd import HttpHandler +from matemat.webserver import HttpHandler, RequestArgument from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @test_pagelet('/just/testing/serve_pagelet_ok') def serve_test_pagelet_ok(method: str, path: str, - args: Dict[str, Union[bytes, str]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]): headers['Content-Type'] = 'text/plain' @@ -20,7 +20,7 @@ def serve_test_pagelet_ok(method: str, @test_pagelet('/just/testing/serve_pagelet_fail') def serve_test_pagelet_fail(method: str, path: str, - args: Dict[str, Union[bytes, str]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index 50ade85..fe30529 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -4,14 +4,14 @@ from typing import Any, Dict, Union from datetime import datetime, timedelta from time import sleep -from matemat.webserver.httpd import HttpHandler +from matemat.webserver import HttpHandler, RequestArgument from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @test_pagelet('/just/testing/sessions') def session_test_pagelet(method: str, path: str, - args: Dict[str, Union[bytes, str]], + args: Dict[str, RequestArgument], session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py index 931f759..85ef721 100644 --- a/matemat/webserver/util.py +++ b/matemat/webserver/util.py @@ -3,8 +3,10 @@ from typing import Dict, List, Tuple, Optional, Union import urllib.parse +from matemat.webserver import RequestArgument -def _parse_multipart(body: bytes, boundary: str) -> Dict[str, List[Tuple[str, Union[bytes, str]]]]: + +def _parse_multipart(body: bytes, boundary: str) -> List[RequestArgument]: """ Given a HTTP body with form-data in multipart form, and the multipart-boundary, parse the multipart items and return them as a dictionary. @@ -13,6 +15,8 @@ def _parse_multipart(body: bytes, boundary: str) -> Dict[str, List[Tuple[str, Un :param boundary: The multipart boundary. :return: A dictionary of field names as key, and content types and field values as value. """ + # Prepend a CRLF for the first boundary to match + body = b'\r\n' + body # Generate item header boundary and terminating boundary from general boundary string _boundary = f'\r\n--{boundary}\r\n'.encode('utf-8') _end_boundary = f'\r\n--{boundary}--\r\n'.encode('utf-8') @@ -20,16 +24,15 @@ def _parse_multipart(body: bytes, boundary: str) -> Dict[str, List[Tuple[str, Un allparts = body.split(_end_boundary, 1) if len(allparts) != 2 or allparts[1] != b'': raise ValueError('Last boundary missing or corrupted') - # Split remaining body into its parts (appending a CRLF for the first boundary to match), and verify at least 1 part - # is there - parts: List[bytes] = (b'\r\n' + allparts[0]).split(_boundary) + # Split remaining body into its parts, and verify at least 1 part is there + parts: List[bytes] = (allparts[0]).split(_boundary) if len(parts) < 1 or parts[0] != b'': raise ValueError('First boundary missing or corrupted') # Remove the first, empty part parts = parts[1:] # Results go into this dict - args: Dict[str, List[Tuple[str, Union[bytes, str]]]] = dict() + args: Dict[str, RequestArgument] = dict() # Parse each multipart part for part in parts: @@ -50,25 +53,29 @@ def _parse_multipart(body: bytes, boundary: str) -> Dict[str, List[Tuple[str, Un cd, *cdargs = hdr['Content-Disposition'].split(';') # Content-Disposition MUST be form-data; everything else is rejected if cd.strip() != 'form-data': - raise ValueError(f'Unknown Content-Disposition: cd') + raise ValueError(f'Unknown Content-Disposition: {cd}') # Extract the "name" header argument + has_name = False for cdarg in cdargs: k, v = cdarg.split('=', 1) if k.strip() == 'name': + has_name = True name: str = v.strip() # Remove quotation marks around the name value if name.startswith('"') and name.endswith('"'): name = v[1:-1] # Add the Content-Type and the content to the header, with the provided name if name not in args: - args[name] = list() - args[name].append((hdr['Content-Type'].strip(), part)) + args[name] = RequestArgument(name) + args[name].append(hdr['Content-Type'].strip(), part) + if not has_name: + raise ValueError('mutlipart/form-data part without name attribute') - return args + return list(args.values()) def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 'text/plain') \ - -> Tuple[str, Dict[str, Tuple[str, Union[bytes, str, List[str]]]]]: + -> Tuple[str, Dict[str, RequestArgument]]: """ Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded or multipart/form-data form, parse the arguments and return them as a dictionary. @@ -85,34 +92,41 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't # Parse the request "URL" (i.e. only the path) tokens = urllib.parse.urlparse(request) # Parse the GET arguments - getargs = urllib.parse.parse_qs(tokens.query) + if len(tokens.query) == 0: + getargs = dict() + else: + getargs = urllib.parse.parse_qs(tokens.query, strict_parsing=True, keep_blank_values=True, errors='strict') - # TODO: { 'foo': [ ('text/plain', 'bar'), ('application/octet-stream', '\x80') ] } - # TODO: Use a @dataclass once Python 3.7 is out - args: Dict[str, Tuple[str, Union[bytes, str, List[str]]]] = dict() + args: Dict[str, RequestArgument] = dict() for k, v in getargs.items(): - args[k] = 'text/plain', v + args[k] = RequestArgument(k) + for _v in v: + args[k].append('text/plain', _v) if postbody is not None: if enctype == 'application/x-www-form-urlencoded': # Parse the POST body - postargs = urllib.parse.parse_qs(postbody.decode('utf-8')) + pb: str = postbody.decode('utf-8') + if len(pb) == 0: + postargs = dict() + else: + postargs = urllib.parse.parse_qs(pb, strict_parsing=True, keep_blank_values=True, errors='strict') # Write all POST values into the dict, overriding potential duplicates from GET for k, v in postargs.items(): - args[k] = 'text/plain', v + args[k] = RequestArgument(k) + for _v in v: + args[k].append('text/plain', _v) elif enctype.startswith('multipart/form-data'): # Parse the multipart boundary from the Content-Type header - boundary: str = enctype.split('boundary=')[1] + try: + boundary: str = enctype.split('boundary=')[1].strip() + except IndexError: + raise ValueError('Multipart boundary in header not set or corrupted') # Parse the multipart body mpargs = _parse_multipart(postbody, boundary) - for k, v in mpargs.items(): - # TODO: Process all values, not just the first - args[k] = v[0] + for ra in mpargs: + args[ra.name] = ra else: raise ValueError(f'Unsupported Content-Type: {enctype}') - # urllib.parse.parse_qs turns ALL arguments into arrays. This turns arrays of length 1 into scalar values - for (k, (ct, v)) in args.items(): - if len(v) == 1: - args[k] = ct, v[0] # Return the path and the parsed arguments return tokens.path, args From 8898abc77b9b9b4c90635598f358e57e1dfc2232 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 01:12:25 +0200 Subject: [PATCH 26/46] Documentation of the RequestArgument class. --- matemat/webserver/requestargs.py | 164 +++++++++++++++++++++++++++++-- matemat/webserver/util.py | 1 + 2 files changed, 159 insertions(+), 6 deletions(-) diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index a35f759..02df0b2 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -1,121 +1,273 @@ -from typing import List, Optional, Tuple, Union +from typing import Iterator, List, Optional, Tuple, Union class RequestArgument(object): + """ + Container class for HTTP request arguments that simplifies dealing with + - scalar and array arguments: + Automatically converts between single values and arrays where necessary: Arrays with one element can be + accessed as scalars, and scalars can be iterated, yielding themselves as a single item. + - UTF-8 strings and binary data (e.g. file uploads): + All data can be retrieved both as a str (if utf-8 decoding is possible) and a bytes object. + + The objects returned from iteration or indexing are immutable views of (parts of) this object. + + Usage example: + + qsargs = urllib.parse.parse_qs(qs, strict_parsing=True, keep_blank_values=True, errors='strict') + args: Dict[str, RequestArgument] = dict() + for k, vs in qsargs: + args[k] = RequestArgument(k) + for v in vs: + # text/plain usually is a sensible choice for values decoded from urlencoded strings + # IF ALREADY IN STRING FORM (which parse_qs does)! + args[k].append('text/plain', v) + + if 'username' in args and args['username'].is_scalar: + username: str = args['username'].get_str() + else: + raise ValueError() + + """ def __init__(self, name: str, value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = None) -> None: + """ + Create a new RequestArgument with a name and optionally an initial value. + + :param name: The name for this argument, as provided via GET or POST. + :param value: The initial value, if any. Optional, initializes with empty array if omitted. + """ + # Assign name self.__name: str = name + # Initialize value self.__value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = None + # Default to empty array if value is None: self.__value = [] else: if isinstance(value, list): if len(value) == 1: + # An array of length 1 will be reduced to a scalar self.__value = value[0] else: + # Store the array self.__value = value else: + # Scalar value, simply store self.__value = value @property def is_array(self) -> bool: + """ + :return: True, if the value is a (possibly empty) array, False otherwise. + """ return isinstance(self.__value, list) @property def is_scalar(self) -> bool: + """ + :return: True, if the value is a single scalar value, False otherwise. + """ return not isinstance(self.__value, list) @property def is_view(self) -> bool: + """ + :return: True, if this instance is an immutable view, False otherwise. + """ return False @property def name(self) -> str: + """ + :return: The name of this argument. + """ return self.__name - def get_str(self, index: int = None) -> Optional[str]: + def get_str(self, index: int = None) -> str: + """ + Attempts to return a value as a string. If this instance is an scalar, no index must be provided. If this + instance is an array, an index must be provided. + + :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :return: An UTF-8 string representation of the requested value. + :raises UnicodeDecodeError: If the value cannot be decoded into an UTF-8 string. + :raises IndexError: If the index is out of bounds. + :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an + index is provided. + """ if self.is_array: + # instance is an array value if index is None: + # Needs an index for array values raise ValueError('index must not be None') + # Type hint; access array element v: Tuple[str, Union[bytes, str]] = self.__value[index] if isinstance(v[1], str): + # The value already is a string, return return v[1] elif isinstance(v[1], bytes): + # The value is a bytes object, attempt to decode return v[1].decode('utf-8') else: + # instance is a scalar value if index is not None: + # Must not have an index for array values raise ValueError('index must be None') if isinstance(self.__value[1], str): + # The value already is a string, return return self.__value[1] elif isinstance(self.__value[1], bytes): + # The value is a bytes object, attempt to decode return self.__value[1].decode('utf-8') - def get_bytes(self, index: int = None) -> Optional[bytes]: + def get_bytes(self, index: int = None) -> bytes: + """ + Attempts to return a value as a bytes object. If this instance is an scalar, no index must be provided. If + this instance is an array, an index must be provided. + + :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :return: A bytes object representation of the requested value. Strings will be encoded as UTF-8. + :raises IndexError: If the index is out of bounds. + :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an + index is provided. + """ if self.is_array: + # instance is an array value if index is None: + # Needs an index for array values raise ValueError('index must not be None') + # Type hint; access array element v: Tuple[str, Union[bytes, str]] = self.__value[index] if isinstance(v[1], bytes): + # The value already is a bytes object, return return v[1] elif isinstance(v[1], str): + # The value is a string, encode first return v[1].encode('utf-8') else: + # instance is a scalar value if index is not None: + # Must not have an index for array values raise ValueError('index must be None') if isinstance(self.__value[1], bytes): + # The value already is a bytes object, return return self.__value[1] elif isinstance(self.__value[1], str): + # The value is a string, encode first return self.__value[1].encode('utf-8') def get_content_type(self, index: int = None) -> Optional[str]: + """ + Attempts to retrieve a value's Content-Type. If this instance is an scalar, no index must be provided. If this + instance is an array, an index must be provided. + + :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :return: The Content-Type of the requested value, as sent by the client. Not necessarily trustworthy. + :raises IndexError: If the index is out of bounds. + :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an + index is provided. + """ if self.is_array: + # instance is an array value if index is None: + # Needs an index for array values raise ValueError('index must not be None') + # Type hint; access array element v: Tuple[str, Union[bytes, str]] = self.__value[index] + # Return the content type of the requested value return v[0] else: + # instance is a scalar value if index is not None: + # Must not have an index for array values raise ValueError('index must be None') + # Return the content type of the scalar value return self.__value[0] def append(self, ctype: str, value: Union[str, bytes]): + """ + Append a value to this instance. Turns an empty argument into a scalar and a scalar into an array. + + :param ctype: The Content-Type, as provided in the request. + :param value: The scalar value to append, either a string or bytes object. + :raises TypeError: If called on an immutable view. + """ if self.is_view: + # This is an immutable view, raise exception raise TypeError('A RequestArgument view is immutable!') if len(self) == 0: + # Turn an empty argument into a scalar self.__value = ctype, value else: + # First turn the scalar into a one-element array ... if self.is_scalar: self.__value = [self.__value] + # ... then append the new value self.__value.append((ctype, value)) - def __len__(self): + def __len__(self) -> int: + """ + :return: Number of values for this argument. + """ return len(self.__value) if self.is_array else 1 - def __iter__(self): + def __iter__(self) -> Iterator['RequestArgument']: + """ + Iterate the values of this argument. Each value is accessible as if it were a scalar RequestArgument in turn, + although they are immutable. + + :return: An iterator that yields immutable views of the values. + """ if self.is_scalar: + # If this is a scalar, yield an immutable view of the single value yield _View(self.__name, self.__value) else: # Typing helper _value: List[Tuple[str, Union[bytes, str]]] = self.__value for v in _value: + # If this is an array, yield an immutable scalar view for each (ctype, value) element in the array yield _View(self.__name, v) def __getitem__(self, index: Union[int, slice]): + """ + Index the argument with either an int or a slice. The returned values are represented as immutable + RequestArgument views. Scalar arguments may be indexed with int(0). + :param index: The index or slice. + :return: An immutable view of the indexed elements of this argument. + """ if self.is_scalar: + # Scalars may only be indexed with 0 if index == 0: + # Return an immutable view of the single scalar value return _View(self.__name, self.__value) raise ValueError('Scalar RequestArgument only indexable with 0') + # Pass the index or slice through to the array, packing the result in an immutable view return _View(self.__name, self.__value[index]) class _View(RequestArgument): + """ + This class represents an immutable view of a (subset of a) RequestArgument object. Should not be instantiated + directly. + """ - def __init__(self, name: str, value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]]): + def __init__(self, name: str, value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]])\ + -> None: + """ + Create a new immutable view of a (subset of a) RequestArgument. + + :param name: The name for this argument, same as in the original RequestArgument. + :param value: The values to represent in this view, obtained by e.g. indexing or slicing. + """ super().__init__(name, value) @property def is_view(self) -> bool: + """ + :return: True, if this instance is an immutable view, False otherwise. + """ return True diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py index 85ef721..61cf430 100644 --- a/matemat/webserver/util.py +++ b/matemat/webserver/util.py @@ -69,6 +69,7 @@ def _parse_multipart(body: bytes, boundary: str) -> List[RequestArgument]: args[name] = RequestArgument(name) args[name].append(hdr['Content-Type'].strip(), part) if not has_name: + # Content-Disposition header without name attribute raise ValueError('mutlipart/form-data part without name attribute') return list(args.values()) From ab9e470c353d1b36fb55bd72dc0cba62277a22ba Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 01:22:12 +0200 Subject: [PATCH 27/46] Some more type hinting/safety. --- matemat/webserver/requestargs.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index 02df0b2..f69a53d 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -43,7 +43,7 @@ class RequestArgument(object): # Assign name self.__name: str = name # Initialize value - self.__value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = None + self.__value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = [] # Default to empty array if value is None: self.__value = [] @@ -98,6 +98,7 @@ class RequestArgument(object): :raises IndexError: If the index is out of bounds. :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an index is provided. + :raises TypeError: If the requested value is neither a str nor a bytes object. """ if self.is_array: # instance is an array value @@ -123,6 +124,7 @@ class RequestArgument(object): elif isinstance(self.__value[1], bytes): # The value is a bytes object, attempt to decode return self.__value[1].decode('utf-8') + raise TypeError('Value is neither a str nor bytes') def get_bytes(self, index: int = None) -> bytes: """ @@ -134,6 +136,7 @@ class RequestArgument(object): :raises IndexError: If the index is out of bounds. :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an index is provided. + :raises TypeError: If the requested value is neither a str nor a bytes object. """ if self.is_array: # instance is an array value @@ -159,6 +162,7 @@ class RequestArgument(object): elif isinstance(self.__value[1], str): # The value is a string, encode first return self.__value[1].encode('utf-8') + raise TypeError('Value is neither a str nor bytes') def get_content_type(self, index: int = None) -> Optional[str]: """ @@ -177,16 +181,18 @@ class RequestArgument(object): # Needs an index for array values raise ValueError('index must not be None') # Type hint; access array element - v: Tuple[str, Union[bytes, str]] = self.__value[index] + va: Tuple[str, Union[bytes, str]] = self.__value[index] # Return the content type of the requested value - return v[0] + return va[0] else: # instance is a scalar value if index is not None: # Must not have an index for array values raise ValueError('index must be None') + # Type hint + vs: Tuple[str, Union[bytes, str]] = self.__value # Return the content type of the scalar value - return self.__value[0] + return vs[0] def append(self, ctype: str, value: Union[str, bytes]): """ @@ -227,8 +233,8 @@ class RequestArgument(object): yield _View(self.__name, self.__value) else: # Typing helper - _value: List[Tuple[str, Union[bytes, str]]] = self.__value - for v in _value: + vs: List[Tuple[str, Union[bytes, str]]] = self.__value + for v in vs: # If this is an array, yield an immutable scalar view for each (ctype, value) element in the array yield _View(self.__name, v) From 73c7dbe89f28dda245322153ffdc5404736eebe0 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 18:11:26 +0200 Subject: [PATCH 28/46] Documentation of RequestArgument test cases. --- matemat/webserver/requestargs.py | 2 +- matemat/webserver/test/test_requestargs.py | 148 ++++++++++++++++++++- 2 files changed, 145 insertions(+), 5 deletions(-) diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index f69a53d..1a56aeb 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -250,7 +250,7 @@ class RequestArgument(object): if index == 0: # Return an immutable view of the single scalar value return _View(self.__name, self.__value) - raise ValueError('Scalar RequestArgument only indexable with 0') + raise IndexError('Scalar RequestArgument only indexable with 0') # Pass the index or slice through to the array, packing the result in an immutable view return _View(self.__name, self.__value[index]) diff --git a/matemat/webserver/test/test_requestargs.py b/matemat/webserver/test/test_requestargs.py index dcdde14..133ea4a 100644 --- a/matemat/webserver/test/test_requestargs.py +++ b/matemat/webserver/test/test_requestargs.py @@ -9,196 +9,336 @@ from matemat.webserver.requestargs import _View class TestRequestArguments(unittest.TestCase): + """ + Test cases for the RequestArgument class. + """ def test_create_default(self): + """ + Test creation of an empty RequestArgument + """ ra = RequestArgument('foo') + # Name must be set to 1st argument self.assertEqual('foo', ra.name) + # Must be a 0-length array self.assertEqual(0, len(ra)) self.assertFalse(ra.is_scalar) self.assertTrue(ra.is_array) + # Must not be a view self.assertFalse(ra.is_view) def test_create_str_scalar(self): + """ + Test creation of a scalar RequestArgument with string value. + """ ra = RequestArgument('foo', ('text/plain', 'bar')) + # Name must be set to 1st argument self.assertEqual('foo', ra.name) + # Must be a scalar, length 1 self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) self.assertFalse(ra.is_array) + # Scalar value must be representable both as str and bytes self.assertEqual('bar', ra.get_str()) self.assertEqual(b'bar', ra.get_bytes()) + # Content-Type must be set correctly self.assertEqual('text/plain', ra.get_content_type()) + # Using indices must result in an error with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_str(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_bytes(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_content_type(0)) + # Must not be a view self.assertFalse(ra.is_view) def test_create_str_scalar_array(self): + """ + Test creation of a scalar RequestArgument with string value, passing an array instead of a single tuple. + """ ra = RequestArgument('foo', [('text/plain', 'bar')]) + # Name must be set to 1st argument self.assertEqual('foo', ra.name) + # Must be a scalar, length 1 self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) self.assertFalse(ra.is_array) + # Scalar value must be representable both as str and bytes self.assertEqual('bar', ra.get_str()) self.assertEqual(b'bar', ra.get_bytes()) + # Content-Type must be set correctly self.assertEqual('text/plain', ra.get_content_type()) + # Using indices must result in an error with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_str(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_bytes(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_content_type(0)) + # Must not be a view self.assertFalse(ra.is_view) def test_create_bytes_scalar(self): + """ + Test creation of a scalar RequestArgument with bytes value. + """ ra = RequestArgument('foo', ('application/octet-stream', b'\x00\x80\xff\xfe')) + # Name must be set to 1st argument self.assertEqual('foo', ra.name) + # Must be a scalar, length 1 self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) self.assertFalse(ra.is_array) + # Conversion to UTF-8 string must fail; bytes representation must work with self.assertRaises(UnicodeDecodeError): ra.get_str() self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes()) + # Content-Type must be set correctly self.assertEqual('application/octet-stream', ra.get_content_type()) + # Using indices must result in an error with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_str(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_bytes(0)) with self.assertRaises(ValueError): self.assertEqual('bar', ra.get_content_type(0)) + # Must not be a view self.assertFalse(ra.is_view) def test_create_array(self): + """ + Test creation of an array RequestArgument with mixed str and bytes initial value. + """ ra = RequestArgument('foo', [ ('text/plain', 'bar'), ('application/octet-stream', b'\x00\x80\xff\xfe') ]) + # Name must be set to 1st argument self.assertEqual('foo', ra.name) + # Must be an array, length 2 self.assertEqual(2, len(ra)) self.assertFalse(ra.is_scalar) self.assertTrue(ra.is_array) + # Retrieving values without an index must fail with self.assertRaises(ValueError): ra.get_str() with self.assertRaises(ValueError): ra.get_bytes() with self.assertRaises(ValueError): ra.get_content_type() + # The first value must be representable both as str and bytes, and have ctype text/plain self.assertEqual('bar', ra.get_str(0)) self.assertEqual(b'bar', ra.get_bytes(0)) self.assertEqual('text/plain', ra.get_content_type(0)) + # Conversion of the second value to UTF-8 string must fail; bytes representation must work with self.assertRaises(UnicodeDecodeError): ra.get_str(1) self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(1)) + # The second value's ctype must be correct self.assertEqual('application/octet-stream', ra.get_content_type(1)) + # Must not be a view self.assertFalse(ra.is_view) def test_append_empty_str(self): + """ + Test appending a str value to an empty RequestArgument. + """ + # Initialize the empty RequestArgument ra = RequestArgument('foo') self.assertEqual(0, len(ra)) self.assertFalse(ra.is_scalar) + # Append a string value ra.append('text/plain', 'bar') + # New length must be 1, empty array must be converted to scalar self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) + # Retrieval of the new value must work both in str and bytes representation self.assertEqual('bar', ra.get_str()) self.assertEqual(b'bar', ra.get_bytes()) + # Content type of the new value must be correct self.assertEqual('text/plain', ra.get_content_type()) + # Must not be a view self.assertFalse(ra.is_view) def test_append_empty_bytes(self): + """ + Test appending a bytes value to an empty RequestArgument. + """ + # Initialize the empty RequestArgument ra = RequestArgument('foo') self.assertEqual(0, len(ra)) self.assertFalse(ra.is_scalar) + # Append a bytes value ra.append('application/octet-stream', b'\x00\x80\xff\xfe') + # New length must be 1, empty array must be converted to scalar self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) + # Conversion of the new value to UTF-8 string must fail; bytes representation must work with self.assertRaises(UnicodeDecodeError): ra.get_str() self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes()) + # Content type of the new value must be correct self.assertEqual('application/octet-stream', ra.get_content_type()) + # Must not be a view self.assertFalse(ra.is_view) def test_append_multiple(self): + """ + Test appending multiple values to an empty RequestArgument. + """ + # Initialize the empty RequestArgument ra = RequestArgument('foo') self.assertEqual(0, len(ra)) self.assertFalse(ra.is_scalar) + # Append a first value ra.append('text/plain', 'bar') + # New length must be 1, empty array must be converted to scalar self.assertEqual(1, len(ra)) self.assertTrue(ra.is_scalar) + self.assertEqual(b'bar', ra.get_bytes()) + # Append a second value ra.append('application/octet-stream', b'\x00\x80\xff\xfe') + # New length must be 2, scalar must be converted to array self.assertEqual(2, len(ra)) self.assertFalse(ra.is_scalar) + self.assertEqual(b'bar', ra.get_bytes(0)) + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(1)) + # Append a third value ra.append('text/plain', 'Hello, World!') + # New length must be 3, array must remain array self.assertEqual(3, len(ra)) self.assertFalse(ra.is_scalar) + self.assertEqual(b'bar', ra.get_bytes(0)) + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(1)) + self.assertEqual(b'Hello, World!', ra.get_bytes(2)) def test_iterate_empty(self): + """ + Test iterating an empty RequestArgument. + """ ra = RequestArgument('foo') self.assertEqual(0, len(ra)) + # No value must be yielded from iterating an empty instance for _ in ra: self.fail() def test_iterate_scalar(self): + """ + Test iterating a scalar RequestArgument. + """ ra = RequestArgument('foo', ('text/plain', 'bar')) self.assertTrue(ra.is_scalar) + # Counter for the number of iterations count: int = 0 for it in ra: + # Make sure the yielded value is a scalar view and has the same name as the original instance self.assertIsInstance(it, _View) - self.assertEqual('foo', it.name) self.assertTrue(it.is_view) + self.assertEqual('foo', it.name) self.assertTrue(it.is_scalar) count += 1 + # Only one value must be yielded from iterating a scalar instance self.assertEqual(1, count) def test_iterate_array(self): + """ + Test iterating an array RequestArgument. + """ ra = RequestArgument('foo', [('text/plain', 'bar'), ('abc', b'def'), ('xyz', '1337')]) self.assertFalse(ra.is_scalar) + # Container to put the iterated ctypes into items: List[str] = list() for it in ra: + # Make sure the yielded values are scalar views and have the same name as the original instance self.assertIsInstance(it, _View) self.assertTrue(it.is_view) self.assertTrue(it.is_scalar) + # Collect the value's ctype items.append(it.get_content_type()) + # Compare collected ctypes with expected result self.assertEqual(['text/plain', 'abc', 'xyz'], items) - def test_iterate_sliced(self): + def test_slice(self): + """ + Test slicing an array RequestArgument. + """ ra = RequestArgument('foo', [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j'), ('k', 'l')]) - self.assertFalse(ra.is_scalar) + # Create the sliced view + sliced = ra[1:4:2] + # Make sure the sliced value is a view + self.assertIsInstance(sliced, _View) + self.assertTrue(sliced.is_view) + # Make sure the slice has the same name + self.assertEqual('foo', sliced.name) + # Make sure the slice has the expected shape (array of the 2nd and 4th scalar in the original) + self.assertTrue(sliced.is_array) + self.assertEqual(2, len(sliced)) + self.assertEqual('d', sliced.get_str(0)) + self.assertEqual('h', sliced.get_str(1)) + + def test_iterate_sliced(self): + """ + Test iterating a sliced array RequestArgument. + """ + ra = RequestArgument('foo', [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j'), ('k', 'l')]) + # Container to put the iterated ctypes into items: List[str] = list() + # Iterate the sliced view for it in ra[1:4:2]: + # Make sure the yielded values are scalar views and have the same name as the original instance self.assertIsInstance(it, _View) self.assertTrue(it.is_view) + self.assertEqual('foo', it.name) self.assertTrue(it.is_scalar) items.append(it.get_content_type()) + # Make sure the expected values are collected (array of the 2nd and 4th scalar in the original) self.assertEqual(['c', 'g'], items) def test_index_scalar(self): + """ + Test indexing of a scalar RequestArgument. + """ ra = RequestArgument('foo', ('bar', 'baz')) + # Index the scalar RequestArgument instance, obtaining an immutable view it = ra[0] + # Make sure the value is a scalar view with the same properties as the original instance self.assertIsInstance(it, _View) + self.assertTrue(it.is_scalar) self.assertEqual('foo', it.name) self.assertEqual('bar', it.get_content_type()) self.assertEqual('baz', it.get_str()) - with self.assertRaises(ValueError): + # Make sure other indices don't work + with self.assertRaises(IndexError): _ = ra[1] def test_index_array(self): + """ + Test indexing of an array RequestArgument. + """ ra = RequestArgument('foo', [('a', 'b'), ('c', 'd')]) + # Index the array RequestArgument instance, obtaining an immutable view it = ra[1] + # Make sure the value is a scalar view with the same properties as the value in the original instance self.assertIsInstance(it, _View) self.assertEqual('foo', it.name) self.assertEqual('c', it.get_content_type()) self.assertEqual('d', it.get_str()) def test_view_immutable(self): + """ + Test immutability of views. + """ ra = RequestArgument('foo', ('bar', 'baz')) + # Index the scalar RequestArgument instance, obtaining an immutable view it = ra[0] + # Make sure the returned value is a view self.assertIsInstance(it, _View) + # Make sure the returned value is immutable with self.assertRaises(TypeError): it.append('foo', 'bar') From 21a927046d43b37c883973c10efc54b43d70d646 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 18:29:51 +0200 Subject: [PATCH 29/46] Reworked RequestArgument API to somewhat more lax concerning 0-indices, potentially leading to safer code. --- matemat/webserver/requestargs.py | 177 +++++++-------------- matemat/webserver/test/test_requestargs.py | 66 ++++---- 2 files changed, 94 insertions(+), 149 deletions(-) diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index 1a56aeb..dcad518 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -43,35 +43,31 @@ class RequestArgument(object): # Assign name self.__name: str = name # Initialize value - self.__value: Union[Tuple[str, Union[bytes, str]], List[Tuple[str, Union[bytes, str]]]] = [] + self.__value: List[Tuple[str, Union[bytes, str]]] = [] # Default to empty array if value is None: self.__value = [] else: if isinstance(value, list): - if len(value) == 1: - # An array of length 1 will be reduced to a scalar - self.__value = value[0] - else: - # Store the array - self.__value = value - else: - # Scalar value, simply store + # Store the array self.__value = value + else: + # Turn scalar into an array before storing + self.__value = [value] @property def is_array(self) -> bool: """ :return: True, if the value is a (possibly empty) array, False otherwise. """ - return isinstance(self.__value, list) + return len(self.__value) != 1 @property def is_scalar(self) -> bool: """ :return: True, if the value is a single scalar value, False otherwise. """ - return not isinstance(self.__value, list) + return len(self.__value) == 1 @property def is_view(self) -> bool: @@ -87,112 +83,70 @@ class RequestArgument(object): """ return self.__name - def get_str(self, index: int = None) -> str: + def get_str(self, index: int = 0) -> str: """ - Attempts to return a value as a string. If this instance is an scalar, no index must be provided. If this - instance is an array, an index must be provided. + Attempts to return a value as a string. The index defaults to 0. - :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :param index: The index of the value to retrieve. Default: 0. :return: An UTF-8 string representation of the requested value. :raises UnicodeDecodeError: If the value cannot be decoded into an UTF-8 string. :raises IndexError: If the index is out of bounds. - :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an - index is provided. + :raises ValueError: If the index is not an int. :raises TypeError: If the requested value is neither a str nor a bytes object. """ - if self.is_array: - # instance is an array value - if index is None: - # Needs an index for array values - raise ValueError('index must not be None') - # Type hint; access array element - v: Tuple[str, Union[bytes, str]] = self.__value[index] - if isinstance(v[1], str): - # The value already is a string, return - return v[1] - elif isinstance(v[1], bytes): - # The value is a bytes object, attempt to decode - return v[1].decode('utf-8') - else: - # instance is a scalar value - if index is not None: - # Must not have an index for array values - raise ValueError('index must be None') - if isinstance(self.__value[1], str): - # The value already is a string, return - return self.__value[1] - elif isinstance(self.__value[1], bytes): - # The value is a bytes object, attempt to decode - return self.__value[1].decode('utf-8') + if not isinstance(index, int): + # Index must be an int + raise ValueError('index must not be None') + # Type hint; access array element + v: Tuple[str, Union[bytes, str]] = self.__value[index] + if isinstance(v[1], str): + # The value already is a string, return + return v[1] + elif isinstance(v[1], bytes): + # The value is a bytes object, attempt to decode + return v[1].decode('utf-8') raise TypeError('Value is neither a str nor bytes') - def get_bytes(self, index: int = None) -> bytes: + def get_bytes(self, index: int = 0) -> bytes: """ - Attempts to return a value as a bytes object. If this instance is an scalar, no index must be provided. If - this instance is an array, an index must be provided. + Attempts to return a value as a bytes object. The index defaults to 0. - :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :param index: The index of the value to retrieve. Default: 0. :return: A bytes object representation of the requested value. Strings will be encoded as UTF-8. :raises IndexError: If the index is out of bounds. - :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an - index is provided. + :raises ValueError: If the index is not an int. :raises TypeError: If the requested value is neither a str nor a bytes object. """ - if self.is_array: - # instance is an array value - if index is None: - # Needs an index for array values - raise ValueError('index must not be None') - # Type hint; access array element - v: Tuple[str, Union[bytes, str]] = self.__value[index] - if isinstance(v[1], bytes): - # The value already is a bytes object, return - return v[1] - elif isinstance(v[1], str): - # The value is a string, encode first - return v[1].encode('utf-8') - else: - # instance is a scalar value - if index is not None: - # Must not have an index for array values - raise ValueError('index must be None') - if isinstance(self.__value[1], bytes): - # The value already is a bytes object, return - return self.__value[1] - elif isinstance(self.__value[1], str): - # The value is a string, encode first - return self.__value[1].encode('utf-8') + if not isinstance(index, int): + # Index must be a int + raise ValueError('index must not be None') + # Type hint; access array element + v: Tuple[str, Union[bytes, str]] = self.__value[index] + if isinstance(v[1], bytes): + # The value already is a bytes object, return + return v[1] + elif isinstance(v[1], str): + # The value is a string, encode first + return v[1].encode('utf-8') raise TypeError('Value is neither a str nor bytes') - def get_content_type(self, index: int = None) -> Optional[str]: + def get_content_type(self, index: int = 0) -> str: """ - Attempts to retrieve a value's Content-Type. If this instance is an scalar, no index must be provided. If this - instance is an array, an index must be provided. + Attempts to retrieve a value's Content-Type. The index defaults to 0. - :param index: For array values: The index of the value to retrieve. For scalar values: None (default). + :param index: The index of the value to retrieve. Default: 0. :return: The Content-Type of the requested value, as sent by the client. Not necessarily trustworthy. :raises IndexError: If the index is out of bounds. - :raises ValueError: If this is an array value, and no index is provided, or if this is a scalar value and an - index is provided. + :raises ValueError: If the index is not an int. """ - if self.is_array: - # instance is an array value - if index is None: - # Needs an index for array values - raise ValueError('index must not be None') - # Type hint; access array element - va: Tuple[str, Union[bytes, str]] = self.__value[index] - # Return the content type of the requested value - return va[0] - else: - # instance is a scalar value - if index is not None: - # Must not have an index for array values - raise ValueError('index must be None') - # Type hint - vs: Tuple[str, Union[bytes, str]] = self.__value - # Return the content type of the scalar value - return vs[0] + # instance is an array value + if not isinstance(index, int): + # Needs an index for array values + raise ValueError('index must not be None') + # Type hint; access array element + va: Tuple[str, Union[bytes, str]] = self.__value[index] + # Return the content type of the requested value + return va[0] def append(self, ctype: str, value: Union[str, bytes]): """ @@ -205,21 +159,13 @@ class RequestArgument(object): if self.is_view: # This is an immutable view, raise exception raise TypeError('A RequestArgument view is immutable!') - if len(self) == 0: - # Turn an empty argument into a scalar - self.__value = ctype, value - else: - # First turn the scalar into a one-element array ... - if self.is_scalar: - self.__value = [self.__value] - # ... then append the new value - self.__value.append((ctype, value)) + self.__value.append((ctype, value)) def __len__(self) -> int: """ :return: Number of values for this argument. """ - return len(self.__value) if self.is_array else 1 + return len(self.__value) def __iter__(self) -> Iterator['RequestArgument']: """ @@ -228,29 +174,18 @@ class RequestArgument(object): :return: An iterator that yields immutable views of the values. """ - if self.is_scalar: - # If this is a scalar, yield an immutable view of the single value - yield _View(self.__name, self.__value) - else: - # Typing helper - vs: List[Tuple[str, Union[bytes, str]]] = self.__value - for v in vs: - # If this is an array, yield an immutable scalar view for each (ctype, value) element in the array - yield _View(self.__name, v) + for v in self.__value: + # Yield an immutable scalar view for each (ctype, value) element in the array + yield _View(self.__name, v) def __getitem__(self, index: Union[int, slice]): """ Index the argument with either an int or a slice. The returned values are represented as immutable - RequestArgument views. Scalar arguments may be indexed with int(0). + RequestArgument views. + :param index: The index or slice. :return: An immutable view of the indexed elements of this argument. """ - if self.is_scalar: - # Scalars may only be indexed with 0 - if index == 0: - # Return an immutable view of the single scalar value - return _View(self.__name, self.__value) - raise IndexError('Scalar RequestArgument only indexable with 0') # Pass the index or slice through to the array, packing the result in an immutable view return _View(self.__name, self.__value[index]) diff --git a/matemat/webserver/test/test_requestargs.py b/matemat/webserver/test/test_requestargs.py index 133ea4a..3383863 100644 --- a/matemat/webserver/test/test_requestargs.py +++ b/matemat/webserver/test/test_requestargs.py @@ -43,13 +43,17 @@ class TestRequestArguments(unittest.TestCase): self.assertEqual(b'bar', ra.get_bytes()) # Content-Type must be set correctly self.assertEqual('text/plain', ra.get_content_type()) - # Using indices must result in an error - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_str(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_bytes(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_content_type(0)) + # Using 0 indices must yield the same results + self.assertEqual('bar', ra.get_str(0)) + self.assertEqual(b'bar', ra.get_bytes(0)) + self.assertEqual('text/plain', ra.get_content_type(0)) + # Using other indices must result in an error + with self.assertRaises(IndexError): + ra.get_str(1) + with self.assertRaises(IndexError): + ra.get_bytes(1) + with self.assertRaises(IndexError): + ra.get_content_type(1) # Must not be a view self.assertFalse(ra.is_view) @@ -69,13 +73,17 @@ class TestRequestArguments(unittest.TestCase): self.assertEqual(b'bar', ra.get_bytes()) # Content-Type must be set correctly self.assertEqual('text/plain', ra.get_content_type()) - # Using indices must result in an error - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_str(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_bytes(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_content_type(0)) + # Using 0 indices must yield the same results + self.assertEqual('bar', ra.get_str(0)) + self.assertEqual(b'bar', ra.get_bytes(0)) + self.assertEqual('text/plain', ra.get_content_type(0)) + # Using other indices must result in an error + with self.assertRaises(IndexError): + ra.get_str(1) + with self.assertRaises(IndexError): + ra.get_bytes(1) + with self.assertRaises(IndexError): + ra.get_content_type(1) # Must not be a view self.assertFalse(ra.is_view) @@ -96,13 +104,18 @@ class TestRequestArguments(unittest.TestCase): self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes()) # Content-Type must be set correctly self.assertEqual('application/octet-stream', ra.get_content_type()) - # Using indices must result in an error - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_str(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_bytes(0)) - with self.assertRaises(ValueError): - self.assertEqual('bar', ra.get_content_type(0)) + # Using 0 indices must yield the same results + with self.assertRaises(UnicodeDecodeError): + ra.get_str(0) + self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(0)) + self.assertEqual('application/octet-stream', ra.get_content_type(0)) + # Using other indices must result in an error + with self.assertRaises(IndexError): + ra.get_str(1) + with self.assertRaises(IndexError): + ra.get_bytes(1) + with self.assertRaises(IndexError): + ra.get_content_type(1) # Must not be a view self.assertFalse(ra.is_view) @@ -120,13 +133,10 @@ class TestRequestArguments(unittest.TestCase): self.assertEqual(2, len(ra)) self.assertFalse(ra.is_scalar) self.assertTrue(ra.is_array) - # Retrieving values without an index must fail - with self.assertRaises(ValueError): - ra.get_str() - with self.assertRaises(ValueError): - ra.get_bytes() - with self.assertRaises(ValueError): - ra.get_content_type() + # Retrieving values without an index must yield the first element + self.assertEqual('bar', ra.get_str()) + self.assertEqual(b'bar', ra.get_bytes()) + self.assertEqual('text/plain', ra.get_content_type()) # The first value must be representable both as str and bytes, and have ctype text/plain self.assertEqual('bar', ra.get_str(0)) self.assertEqual(b'bar', ra.get_bytes(0)) From 2f927cec41d282a9b6e1485b80b8f5a015fde42b Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 21:56:22 +0200 Subject: [PATCH 30/46] Implemented a container for RequestArgument instances, with some more unit tests. --- matemat/webserver/__init__.py | 2 +- matemat/webserver/requestargs.py | 140 ++++++++++++++-- matemat/webserver/test/test_requestargs.py | 179 ++++++++++++++++++++- 3 files changed, 303 insertions(+), 18 deletions(-) diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index 1b4ab06..c52368e 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -6,5 +6,5 @@ API that can be used by 'pagelets' - single pages of a web service. If a reques server will attempt to serve the request with a static resource in a previously configured webroot directory. """ -from .requestargs import RequestArgument +from .requestargs import RequestArgument, RequestArguments from .httpd import MatematWebserver, HttpHandler, pagelet diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index dcad518..2150b31 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -1,5 +1,79 @@ -from typing import Iterator, List, Optional, Tuple, Union +from typing import Dict, Iterator, List, Tuple, Union + + +class RequestArguments(object): + """ + Container for HTTP Request arguments. + + Usage: + + # Create empty instance + ra = RequestArguments() + # Add an entry for the key 'foo' with the value 'bar' and Content-Type 'text/plain' + ra['foo'].append('text/plain', 'bar') + # Retrieve the value for the key 'foo', as a string... + foo = str(ra.foo) + # ... or as raw bytes + foo = bytes(ra.foo) + """ + + def __init__(self) -> None: + """ + Create an empty container instance. + """ + self.__container: Dict[str, RequestArgument] = dict() + + def __getitem__(self, key: str) -> 'RequestArgument': + """ + Retrieve the argument for the given name, creating it on the fly, if it doesn't exist. + + :param key: Name of the argument to retrieve. + :return: A RequestArgument instance. + :raises TypeError: If key is not a string. + """ + if not isinstance(key, str): + raise TypeError('key must be a str') + # Create empty argument, if it doesn't exist + if key not in self.__container: + self.__container[key] = RequestArgument(key) + # Return the argument for the name + return self.__container[key] + + def __getattr__(self, key: str) -> 'RequestArgument': + """ + Syntactic sugar for accessing values with a name that can be used in Python attributes. The value will be + returned as an immutable view. + + :param key: Name of the argument to retrieve. + :return: An immutable view of the RequestArgument instance. + """ + return _View.of(self.__container[key]) + + def __iter__(self) -> Iterator['RequestArguments']: + """ + Returns an iterator over the values in this instance. Values are represented as immutable views. + + :return: An iterator that yields immutable views of the values. + """ + for ra in self.__container.values(): + # Yield an immutable scalar view for each value + yield _View.of(ra) + + def __contains__(self, key: str) -> bool: + """ + Checks whether an argument with a given name exists in the RequestArguments instance. + + :param key: The name to check whether it exists. + :return: True, if present, False otherwise. + """ + return key in self.__container + + def __len__(self) -> int: + """ + :return: The number of arguments in this instance. + """ + return len(self.__container) class RequestArgument(object): @@ -16,18 +90,16 @@ class RequestArgument(object): Usage example: qsargs = urllib.parse.parse_qs(qs, strict_parsing=True, keep_blank_values=True, errors='strict') - args: Dict[str, RequestArgument] = dict() + args: RequestArguments for k, vs in qsargs: - args[k] = RequestArgument(k) + args[k].clear() for v in vs: # text/plain usually is a sensible choice for values decoded from urlencoded strings # IF ALREADY IN STRING FORM (which parse_qs does)! args[k].append('text/plain', v) - if 'username' in args and args['username'].is_scalar: - username: str = args['username'].get_str() - else: - raise ValueError() + if 'username' in args and args.username.is_scalar: + username = str(args.username) """ @@ -91,12 +163,12 @@ class RequestArgument(object): :return: An UTF-8 string representation of the requested value. :raises UnicodeDecodeError: If the value cannot be decoded into an UTF-8 string. :raises IndexError: If the index is out of bounds. - :raises ValueError: If the index is not an int. + :raises TypeError: If the index is not an int. :raises TypeError: If the requested value is neither a str nor a bytes object. """ if not isinstance(index, int): # Index must be an int - raise ValueError('index must not be None') + raise TypeError('index must be an int') # Type hint; access array element v: Tuple[str, Union[bytes, str]] = self.__value[index] if isinstance(v[1], str): @@ -107,6 +179,14 @@ class RequestArgument(object): return v[1].decode('utf-8') raise TypeError('Value is neither a str nor bytes') + def __str__(self) -> str: + """ + Attempts to return the first value as a string. + :return: An UTF-8 string representation of the first value. + :raises UnicodeDecodeError: If the value cannot be decoded into an UTF-8 string. + """ + return self.get_str() + def get_bytes(self, index: int = 0) -> bytes: """ Attempts to return a value as a bytes object. The index defaults to 0. @@ -114,12 +194,12 @@ class RequestArgument(object): :param index: The index of the value to retrieve. Default: 0. :return: A bytes object representation of the requested value. Strings will be encoded as UTF-8. :raises IndexError: If the index is out of bounds. - :raises ValueError: If the index is not an int. + :raises TypeError: If the index is not an int. :raises TypeError: If the requested value is neither a str nor a bytes object. """ if not isinstance(index, int): # Index must be a int - raise ValueError('index must not be None') + raise TypeError('index must be an int') # Type hint; access array element v: Tuple[str, Union[bytes, str]] = self.__value[index] if isinstance(v[1], bytes): @@ -130,6 +210,13 @@ class RequestArgument(object): return v[1].encode('utf-8') raise TypeError('Value is neither a str nor bytes') + def __bytes__(self) -> bytes: + """ + Attempts to return the first value as a bytes object. + :return: A bytes string representation of the first value. + """ + return self.get_bytes() + def get_content_type(self, index: int = 0) -> str: """ Attempts to retrieve a value's Content-Type. The index defaults to 0. @@ -137,18 +224,20 @@ class RequestArgument(object): :param index: The index of the value to retrieve. Default: 0. :return: The Content-Type of the requested value, as sent by the client. Not necessarily trustworthy. :raises IndexError: If the index is out of bounds. - :raises ValueError: If the index is not an int. + :raises TypeError: If the index is not an int. """ # instance is an array value if not isinstance(index, int): # Needs an index for array values - raise ValueError('index must not be None') + raise TypeError('index must be an int') # Type hint; access array element va: Tuple[str, Union[bytes, str]] = self.__value[index] # Return the content type of the requested value + if not isinstance(va[0], str): + raise TypeError('Content-Type is not a str') return va[0] - def append(self, ctype: str, value: Union[str, bytes]): + def append(self, ctype: str, value: Union[str, bytes]) -> None: """ Append a value to this instance. Turns an empty argument into a scalar and a scalar into an array. @@ -161,6 +250,17 @@ class RequestArgument(object): raise TypeError('A RequestArgument view is immutable!') self.__value.append((ctype, value)) + def clear(self) -> None: + """ + Remove all values from this instance. + + :raises TypeError: If called on an immutable view. + """ + if self.is_view: + # This is an immutable view, raise exception + raise TypeError('A RequestArgument view is immutable!') + self.__value.clear() + def __len__(self) -> int: """ :return: Number of values for this argument. @@ -178,7 +278,7 @@ class RequestArgument(object): # Yield an immutable scalar view for each (ctype, value) element in the array yield _View(self.__name, v) - def __getitem__(self, index: Union[int, slice]): + def __getitem__(self, index: Union[int, slice]) -> 'RequestArgument': """ Index the argument with either an int or a slice. The returned values are represented as immutable RequestArgument views. @@ -206,6 +306,16 @@ class _View(RequestArgument): """ super().__init__(name, value) + @staticmethod + def of(argument: 'RequestArgument') ->'RequestArgument': + """ + Create an immutable, unsliced view of an RequestArgument instance. + + :param argument: The RequestArgument instance to create a view of. + :return: An immutable view of the provided RequestArgument instance. + """ + return argument[:] + @property def is_view(self) -> bool: """ diff --git a/matemat/webserver/test/test_requestargs.py b/matemat/webserver/test/test_requestargs.py index 3383863..3e093a2 100644 --- a/matemat/webserver/test/test_requestargs.py +++ b/matemat/webserver/test/test_requestargs.py @@ -1,9 +1,10 @@ -from typing import List +from typing import Dict, List, Set, Tuple import unittest +import urllib.parse -from matemat.webserver import RequestArgument +from matemat.webserver import RequestArgument, RequestArguments # noinspection PyProtectedMember from matemat.webserver.requestargs import _View @@ -228,6 +229,56 @@ class TestRequestArguments(unittest.TestCase): self.assertEqual(b'\x00\x80\xff\xfe', ra.get_bytes(1)) self.assertEqual(b'Hello, World!', ra.get_bytes(2)) + def test_clear_empty(self): + """ + Test clearing an empty RequestArgument. + """ + # Initialize the empty RequestArgument + ra = RequestArgument('foo') + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + ra.clear() + # Clearing an empty RequestArgument shouldn't have any effect + self.assertEqual('foo', ra.name) + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + + def test_clear_scalar(self): + """ + Test clearing a scalar RequestArgument. + """ + # Initialize the scalar RequestArgument + ra = RequestArgument('foo', ('text/plain', 'bar')) + self.assertEqual(1, len(ra)) + self.assertTrue(ra.is_scalar) + ra.clear() + # Clearing a scalar RequestArgument should reduce its size to 0 + self.assertEqual('foo', ra.name) + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + with self.assertRaises(IndexError): + ra.get_str() + + def test_clear_array(self): + """ + Test clearing an array RequestArgument. + """ + # Initialize the array RequestArgument + ra = RequestArgument('foo', [ + ('text/plain', 'bar'), + ('application/octet-stream', b'\x00\x80\xff\xfe'), + ('text/plain', 'baz'), + ]) + self.assertEqual(3, len(ra)) + self.assertFalse(ra.is_scalar) + ra.clear() + # Clearing an array RequestArgument should reduce its size to 0 + self.assertEqual('foo', ra.name) + self.assertEqual(0, len(ra)) + self.assertFalse(ra.is_scalar) + with self.assertRaises(IndexError): + ra.get_str() + def test_iterate_empty(self): """ Test iterating an empty RequestArgument. @@ -352,3 +403,127 @@ class TestRequestArguments(unittest.TestCase): # Make sure the returned value is immutable with self.assertRaises(TypeError): it.append('foo', 'bar') + with self.assertRaises(TypeError): + it.clear() + + def test_str_shorthand(self): + """ + Test the shorthand for get_str(0). + """ + ra = RequestArgument('foo', ('bar', 'baz')) + self.assertEqual('baz', str(ra)) + + def test_bytes_shorthand(self): + """ + Test the shorthand for get_bytes(0). + """ + ra = RequestArgument('foo', ('bar', b'\x00\x80\xff\xfe')) + self.assertEqual(b'\x00\x80\xff\xfe', bytes(ra)) + + # noinspection PyTypeChecker + def test_insert_garbage(self): + """ + Test proper handling with non-int indices and non-str/non-bytes data + :return: + """ + ra = RequestArgument('foo', 42) + with self.assertRaises(TypeError): + str(ra) + ra = RequestArgument('foo', (None, 42)) + with self.assertRaises(TypeError): + str(ra) + with self.assertRaises(TypeError): + bytes(ra) + with self.assertRaises(TypeError): + ra.get_content_type() + with self.assertRaises(TypeError): + ra.get_str('foo') + with self.assertRaises(TypeError): + ra.get_bytes('foo') + with self.assertRaises(TypeError): + ra.get_content_type('foo') + + def test_requestarguments_index(self): + """ + Make sure indexing a RequestArguments instance creates a new entry on the fly. + """ + ra = RequestArguments() + self.assertEqual(0, len(ra)) + self.assertFalse('foo' in ra) + # Create new entry + _ = ra['foo'] + self.assertEqual(1, len(ra)) + self.assertTrue('foo' in ra) + # Already exists, no new entry created + _ = ra['foo'] + self.assertEqual(1, len(ra)) + # Entry must be empty and mutable, and have the correct name + self.assertFalse(ra['foo'].is_view) + self.assertEqual(0, len(ra['foo'])) + self.assertEqual('foo', ra['foo'].name) + # Key must be a string + with self.assertRaises(TypeError): + # noinspection PyTypeChecker + _ = ra[42] + + def test_requestarguments_attr(self): + """ + Test attribute access syntactic sugar. + """ + ra = RequestArguments() + # Attribute should not exist yet + with self.assertRaises(KeyError): + _ = ra.foo + # Create entry + _ = ra['foo'] + # Creating entry should have created the attribute + self.assertEqual('foo', ra.foo.name) + # Attribute access should yield an immutable view + self.assertTrue(ra.foo.is_view) + + def test_requestarguments_iterate(self): + """ + Test iterating a RequestArguments instance. + """ + # Create an instance with some values + ra = RequestArguments() + ra['foo'].append('a', 'b') + ra['bar'].append('c', 'd') + ra['foo'].append('e', 'f') + # Container for test values (name, value) + items: Set[Tuple[str, str]] = set() + # Iterate RequestArguments instance, adding the name and value of each to the set + for a in ra: + items.add((a.name, str(a))) + # Compare result with expected value + self.assertEqual(2, len(items)) + self.assertIn(('foo', 'b'), items) + self.assertIn(('bar', 'd'), items) + + def test_requestarguments_full_use_case(self): + """ + Simulate a minimal RequestArguments use case. + """ + # Create empty RequestArguments instance + ra = RequestArguments() + # Parse GET request + getargs: Dict[str, List[str]] = urllib.parse.parse_qs('foo=42&bar=1337&foo=43&baz=Hello,%20World!') + # Insert GET arguments into RequestArguments + for k, vs in getargs.items(): + for v in vs: + ra[k].append('text/plain', v) + # Parse POST request + postargs: Dict[str, List[str]] = urllib.parse.parse_qs('foo=postfoo&postbar=42&foo=postfoo') + # Insert POST arguments into RequestArguments + for k, vs in postargs.items(): + # In this implementation, POST args replace GET args + ra[k].clear() + for v in vs: + ra[k].append('text/plain', v) + + # Someplace else: Use the RequestArguments instance. + self.assertEqual('1337', ra.bar.get_str()) + self.assertEqual('Hello, World!', ra.baz.get_str()) + self.assertEqual('42', ra.postbar.get_str()) + for a in ra.foo: + self.assertEqual('postfoo', a.get_str()) From 0fb60d1828695ec52d5008b6415441138058c604 Mon Sep 17 00:00:00 2001 From: s3lph Date: Fri, 29 Jun 2018 22:13:39 +0200 Subject: [PATCH 31/46] Using the new RequestArguments API throughout the project. --- matemat/webserver/httpd.py | 10 ++--- matemat/webserver/pagelets/login.py | 10 ++--- matemat/webserver/pagelets/logout.py | 4 +- matemat/webserver/pagelets/main.py | 6 +-- matemat/webserver/pagelets/touchkey.py | 12 +++-- matemat/webserver/test/abstract_httpd_test.py | 8 ++-- matemat/webserver/test/test_parse_request.py | 44 +++++++++---------- matemat/webserver/test/test_post.py | 12 ++--- matemat/webserver/test/test_serve.py | 8 ++-- matemat/webserver/test/test_session.py | 6 +-- matemat/webserver/util.py | 28 ++++++------ 11 files changed, 73 insertions(+), 75 deletions(-) diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 79efb98..c59e3fc 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -13,7 +13,7 @@ from uuid import uuid4 from datetime import datetime, timedelta from matemat import __version__ as matemat_version -from matemat.webserver import RequestArgument +from matemat.webserver import RequestArguments from matemat.webserver.util import parse_args @@ -31,7 +31,7 @@ BaseHTTPRequestHandler.log_error = lambda self, fstring='', *args: None # Dictionary to hold registered pagelet paths and their handler functions _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) str, # Request path - Dict[str, RequestArgument], # args: (name, argument) + RequestArguments, # HTTP Request arguments Dict[str, Any], # Session vars Dict[str, str]], # Response headers Tuple[int, Union[bytes, str]]]] = dict() # Returns: (status code, response body) @@ -51,7 +51,7 @@ def pagelet(path: str): (method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]) -> (int, Optional[Union[str, bytes]]) @@ -69,7 +69,7 @@ def pagelet(path: str): def http_handler(fun: Callable[[str, str, - Dict[str, RequestArgument], + RequestArguments, Dict[str, Any], Dict[str, str]], Tuple[int, Union[bytes, str]]]): @@ -181,7 +181,7 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] - def _handle(self, method: str, path: str, args: Dict[str, RequestArgument]) -> None: + def _handle(self, method: str, path: str, args: RequestArguments) -> None: """ Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index f7813b4..7d0cc2d 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Optional, Tuple, Union from matemat.exceptions import AuthenticationError -from matemat.webserver import pagelet, RequestArgument +from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User from matemat.db import MatematDatabase @@ -10,7 +10,7 @@ from matemat.db import MatematDatabase @pagelet('/login') def login_page(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: @@ -41,13 +41,11 @@ def login_page(method: str, ''' - return 200, data.format(msg=args['msg'] if 'msg' in args else '') + return 200, data.format(msg=str(args.msg) if 'msg' in args else '') elif method == 'POST': - username: RequestArgument = args['username'] - password: RequestArgument = args['password'] with MatematDatabase('test.db') as db: try: - user: User = db.login(username.get_str(), password.get_str()) + user: User = db.login(str(args.username), str(args.password)) except AuthenticationError: headers['Location'] = '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' return 301, bytes() diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index b70d7c1..beb86a3 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -1,13 +1,13 @@ from typing import Any, Dict, List, Optional, Tuple, Union -from matemat.webserver import pagelet, RequestArgument +from matemat.webserver import pagelet, RequestArguments @pagelet('/logout') def logout(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index 2b9ce79..e22c872 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union -from matemat.webserver import MatematWebserver, pagelet, RequestArgument +from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User from matemat.db import MatematDatabase @@ -9,7 +9,7 @@ from matemat.db import MatematDatabase @pagelet('/') def main_page(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 22e3df4..4de8009 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -1,8 +1,8 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union from matemat.exceptions import AuthenticationError -from matemat.webserver import pagelet, RequestArgument +from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User from matemat.db import MatematDatabase @@ -10,7 +10,7 @@ from matemat.db import MatematDatabase @pagelet('/touchkey') def touchkey_page(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ -> Tuple[int, Optional[Union[str, bytes]]]: @@ -40,13 +40,11 @@ def touchkey_page(method: str, ''' - return 200, data.format(username=args['username'] if 'username' in args else '') + return 200, data.format(username=str(args.username) if 'username' in args else '') elif method == 'POST': - username: RequestArgument = args['username'] - touchkey: RequestArgument = args['touchkey'] with MatematDatabase('test.db') as db: try: - user: User = db.login(username.get_str(), touchkey=touchkey.get_str()) + user: User = db.login(str(args.username), touchkey=str(args.touchkey)) except AuthenticationError: headers['Location'] = f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' return 301, bytes() diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index daa1126..103979b 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import Any, Callable, Dict, Tuple, Union import unittest.mock from io import BytesIO @@ -9,7 +9,7 @@ from abc import ABC from datetime import datetime from http.server import HTTPServer -from matemat.webserver import pagelet, RequestArgument +from matemat.webserver import pagelet, RequestArguments class HttpResponse: @@ -158,14 +158,14 @@ def test_pagelet(path: str): def with_testing_headers(fun: Callable[[str, str, - Dict[str, RequestArgument], + RequestArguments, Dict[str, Any], Dict[str, str]], Tuple[int, Union[bytes, str]]]): @pagelet(path) def testing_wrapper(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): status, body = fun(method, path, args, session_vars, headers) diff --git a/matemat/webserver/test/test_parse_request.py b/matemat/webserver/test/test_parse_request.py index a533936..0a94065 100644 --- a/matemat/webserver/test/test_parse_request.py +++ b/matemat/webserver/test/test_parse_request.py @@ -20,9 +20,9 @@ class TestParseRequest(unittest.TestCase): path, args = parse_args('/?foo=42&bar=1337&baz=Hello,%20World!') self.assertEqual('/', path) self.assertEqual(3, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertTrue(args['baz'].is_scalar) @@ -37,9 +37,9 @@ class TestParseRequest(unittest.TestCase): path, args = parse_args('/abc/def?foo=42&bar=1337&baz=Hello,%20World!') self.assertEqual('/abc/def', path) self.assertEqual(3, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertTrue(args['baz'].is_scalar) @@ -54,8 +54,8 @@ class TestParseRequest(unittest.TestCase): path, args = parse_args('/abc/def?foo=42&foo=1337&baz=Hello,%20World!') self.assertEqual('/abc/def', path) self.assertEqual(2, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_array) self.assertTrue(args['baz'].is_scalar) self.assertEqual(2, len(args['foo'])) @@ -65,8 +65,8 @@ class TestParseRequest(unittest.TestCase): def test_parse_get_zero_arg(self): path, args = parse_args('/abc/def?foo=&bar=42') self.assertEqual(2, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertEqual(1, len(args['foo'])) @@ -83,9 +83,9 @@ class TestParseRequest(unittest.TestCase): enctype='application/x-www-form-urlencoded') self.assertEqual('/', path) self.assertEqual(3, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertTrue(args['baz'].is_scalar) @@ -102,8 +102,8 @@ class TestParseRequest(unittest.TestCase): enctype='application/x-www-form-urlencoded') self.assertEqual('/', path) self.assertEqual(2, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_array) self.assertTrue(args['baz'].is_scalar) self.assertEqual(2, len(args['foo'])) @@ -113,8 +113,8 @@ class TestParseRequest(unittest.TestCase): def test_parse_post_urlencoded_zero_arg(self): path, args = parse_args('/abc/def', postbody=b'foo=&bar=42', enctype='application/x-www-form-urlencoded') self.assertEqual(2, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertEqual(1, len(args['foo'])) @@ -152,9 +152,9 @@ class TestParseRequest(unittest.TestCase): enctype='multipart/form-data; boundary=testBoundary1337') self.assertEqual('/', path) self.assertEqual(3, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) - self.assertIn('baz', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertTrue(args['baz'].is_scalar) @@ -177,8 +177,8 @@ class TestParseRequest(unittest.TestCase): b'--testBoundary1337--\r\n', enctype='multipart/form-data; boundary=testBoundary1337') self.assertEqual(2, len(args)) - self.assertIn('foo', args.keys()) - self.assertIn('bar', args.keys()) + self.assertIn('foo', args) + self.assertIn('bar', args) self.assertTrue(args['foo'].is_scalar) self.assertTrue(args['bar'].is_scalar) self.assertEqual(1, len(args['foo'])) diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 9b2fe22..0bc5d16 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List -from matemat.webserver import HttpHandler, RequestArgument +from matemat.webserver import HttpHandler, RequestArguments from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet import codecs @@ -10,7 +10,7 @@ import codecs @test_pagelet('/just/testing/post') def post_test_pagelet(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): """ @@ -18,12 +18,12 @@ def post_test_pagelet(method: str, """ headers['Content-Type'] = 'text/plain' dump: str = '' - for k, ra in args.items(): + for ra in args: for a in ra: if a.get_content_type().startswith('text/'): - dump += f'{k}: {a.get_str()}\n' + dump += f'{a.name}: {a.get_str()}\n' else: - dump += f'{k}: {codecs.encode(a.get_bytes(), "hex").decode("utf-8")}\n' + dump += f'{a.name}: {codecs.encode(a.get_bytes(), "hex").decode("utf-8")}\n' return 200, dump diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 7e159e3..722870b 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -1,16 +1,16 @@ -from typing import Any, Dict, Union +from typing import Any, Dict import os import os.path -from matemat.webserver import HttpHandler, RequestArgument +from matemat.webserver import HttpHandler, RequestArguments from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @test_pagelet('/just/testing/serve_pagelet_ok') def serve_test_pagelet_ok(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): headers['Content-Type'] = 'text/plain' @@ -20,7 +20,7 @@ def serve_test_pagelet_ok(method: str, @test_pagelet('/just/testing/serve_pagelet_fail') def serve_test_pagelet_fail(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index fe30529..5cf408e 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -1,17 +1,17 @@ -from typing import Any, Dict, Union +from typing import Any, Dict from datetime import datetime, timedelta from time import sleep -from matemat.webserver import HttpHandler, RequestArgument +from matemat.webserver import HttpHandler, RequestArguments from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @test_pagelet('/just/testing/sessions') def session_test_pagelet(method: str, path: str, - args: Dict[str, RequestArgument], + args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): session_vars['test'] = 'hello, world!' diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py index 61cf430..2bc2244 100644 --- a/matemat/webserver/util.py +++ b/matemat/webserver/util.py @@ -1,9 +1,9 @@ -from typing import Dict, List, Tuple, Optional, Union +from typing import Dict, List, Tuple, Optional import urllib.parse -from matemat.webserver import RequestArgument +from matemat.webserver import RequestArguments, RequestArgument def _parse_multipart(body: bytes, boundary: str) -> List[RequestArgument]: @@ -76,7 +76,7 @@ def _parse_multipart(body: bytes, boundary: str) -> List[RequestArgument]: def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 'text/plain') \ - -> Tuple[str, Dict[str, RequestArgument]]: + -> Tuple[str, RequestArguments]: """ Given a HTTP request path, and optionally a HTTP POST body in application/x-www-form-urlencoded or multipart/form-data form, parse the arguments and return them as a dictionary. @@ -98,11 +98,11 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't else: getargs = urllib.parse.parse_qs(tokens.query, strict_parsing=True, keep_blank_values=True, errors='strict') - args: Dict[str, RequestArgument] = dict() - for k, v in getargs.items(): - args[k] = RequestArgument(k) - for _v in v: - args[k].append('text/plain', _v) + args = RequestArguments() + for k, vs in getargs.items(): + args[k].clear() + for v in vs: + args[k].append('text/plain', v) if postbody is not None: if enctype == 'application/x-www-form-urlencoded': @@ -113,10 +113,10 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't else: postargs = urllib.parse.parse_qs(pb, strict_parsing=True, keep_blank_values=True, errors='strict') # Write all POST values into the dict, overriding potential duplicates from GET - for k, v in postargs.items(): - args[k] = RequestArgument(k) - for _v in v: - args[k].append('text/plain', _v) + for k, vs in postargs.items(): + args[k].clear() + for v in vs: + args[k].append('text/plain', v) elif enctype.startswith('multipart/form-data'): # Parse the multipart boundary from the Content-Type header try: @@ -126,7 +126,9 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't # Parse the multipart body mpargs = _parse_multipart(postbody, boundary) for ra in mpargs: - args[ra.name] = ra + args[ra.name].clear() + for a in ra: + args[ra.name].append(a.get_content_type(), bytes(a)) else: raise ValueError(f'Unsupported Content-Type: {enctype}') # Return the path and the parsed arguments From a89f2cd15de792fc4d547f466ebd5e6ba212e1bb Mon Sep 17 00:00:00 2001 From: s3lph Date: Wed, 4 Jul 2018 22:05:47 +0200 Subject: [PATCH 32/46] Updated documentation in wiki, and a minor doc fix in the code --- doc | 2 +- matemat/webserver/httpd.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc b/doc index 8f85927..14b8380 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 8f859277827574f80e392d818c25dcf24afaf5b0 +Subproject commit 14b8380090858c3bed5c3b2ee7cf1408aaa133df diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index c59e3fc..b703f72 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, Tuple, Union +from typing import Any, Callable, Dict, Optional, Tuple, Union import traceback @@ -72,7 +72,7 @@ def pagelet(path: str): RequestArguments, Dict[str, Any], Dict[str, str]], - Tuple[int, Union[bytes, str]]]): + Tuple[int, Optional[Union[bytes, str]]]]): # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun # Don't change the function itself at all From f2e48fe339683dfc5f52ab72db0ca14e936e829e Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 15:20:54 +0200 Subject: [PATCH 33/46] Removed bcrypt dependency from README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f162e95..a6d078e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ This project intends to provide a well-tested and maintainable alternative to - Python 3 (>=3.6) - Python dependencies: - apsw - - bcrypt ## Usage From b453721821407fa9f6559b2f74c3d4b01d5a1742 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 15:23:24 +0200 Subject: [PATCH 34/46] README: Added link to old matemat webapp --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a6d078e..d1eec08 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ It provides a touch-input-friendly user interface (as most input happens through soda machine's touch screen). This project intends to provide a well-tested and maintainable alternative to -[TODO][todo] (discontinued). +[ckruse/matemat][oldapp] (last commit 2013-07-09). ## Further Documentation @@ -36,6 +36,7 @@ python -m matemat [MIT License][mit-license] +[oldapp]: https://github.com/ckruse/matemat [mit-license]: https://gitlab.com/s3lph/matemat/blob/master/LICENSE [master]: https://gitlab.com/s3lph/matemat/commits/master [wiki]: https://gitlab.com/s3lph/matemat/wikis/home \ No newline at end of file From 00bcae9874d5f0bbc7b883bc2e360bce0a2ee123 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 15:29:32 +0200 Subject: [PATCH 35/46] Fixed behavior for missing Content-Type in multipart parts --- matemat/webserver/test/test_parse_request.py | 45 +++++++++++++------- matemat/webserver/util.py | 7 ++- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/matemat/webserver/test/test_parse_request.py b/matemat/webserver/test/test_parse_request.py index 0a94065..acb225c 100644 --- a/matemat/webserver/test/test_parse_request.py +++ b/matemat/webserver/test/test_parse_request.py @@ -185,6 +185,35 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('', args['foo'].get_str()) self.assertEqual('42', args['bar'].get_str()) + def test_parse_post_multipart_no_contenttype(self): + """ + Test that the Content-Type is set to 'application/octet-stream' if it is absent from the multipart header. + """ + path, args = parse_args('/', + postbody=b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="foo"\r\n' + b'Content-Type: text/plain\r\n\r\n' + b'42\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' + b'Content-Type: application/octet-stream\r\n\r\n' + b'1337\r\n' + b'--testBoundary1337\r\n' + b'Content-Disposition: form-data; name="baz"\r\n\r\n' + b'Hello, World!\r\n' + b'--testBoundary1337--\r\n', + enctype='multipart/form-data; boundary=testBoundary1337') + self.assertEqual('/', path) + self.assertEqual(3, len(args)) + self.assertIn('foo', args) + self.assertIn('bar', args) + self.assertIn('baz', args) + self.assertTrue(args['foo'].is_scalar) + self.assertTrue(args['bar'].is_scalar) + self.assertTrue(args['baz'].is_scalar) + self.assertEqual('application/octet-stream', args['baz'].get_content_type()) + self.assertEqual('Hello, World!', args['baz'].get_str()) + def test_parse_post_multipart_broken_boundaries(self): with self.assertRaises(ValueError): # Boundary not defined in Content-Type @@ -237,22 +266,6 @@ class TestParseRequest(unittest.TestCase): b'Hello, World!\r\n' b'--testBoundary1337\r\n', enctype='multipart/form-data; boundary=testBoundary1337') - with self.assertRaises(ValueError): - # Missing Content-Type header in one part - parse_args('/', - postbody=b'--testBoundary1337\r\n' - b'Content-Disposition: form-data; name="foo"\r\n' - b'Content-Type: text/plain\r\n\r\n' - b'42\r\n' - b'--testBoundary1337\r\n' - b'Content-Disposition: form-data; name="bar"; filename="bar.bin"\r\n' - b'Content-Type: application/octet-stream\r\n\r\n' - b'1337\r\n' - b'--testBoundary1337\r\n' - b'Content-Disposition: form-data; name="baz"\r\n\r\n' - b'Hello, World!\r\n' - b'--testBoundary1337--\r\n', - enctype='multipart/form-data; boundary=testBoundary1337') with self.assertRaises(ValueError): # Missing Content-Disposition header in one part parse_args('/', diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py index 2bc2244..6e19d7b 100644 --- a/matemat/webserver/util.py +++ b/matemat/webserver/util.py @@ -46,8 +46,11 @@ def _parse_multipart(body: bytes, boundary: str) -> List[RequestArgument]: # Add header to hdr dict hk, hv = head.decode('utf-8').split(':') hdr[hk.strip()] = hv.strip() - # At least Content-Type and Content-Disposition must be present - if 'Content-Type' not in hdr or 'Content-Disposition' not in hdr: + # No content type set - set broadest possible type + if 'Content-Type' not in hdr: + hdr['Content-Type'] = 'application/octet-stream' + # At least Content-Disposition must be present + if 'Content-Disposition' not in hdr: raise ValueError('Missing Content-Type or Content-Disposition header') # Extract Content-Disposition header value and its arguments cd, *cdargs = hdr['Content-Disposition'].split(';') From 528f7322ac1731d4437d2158a07b1187f1c24351 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 15:37:57 +0200 Subject: [PATCH 36/46] Docstrings for the request parser tests. --- matemat/webserver/test/test_parse_request.py | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/matemat/webserver/test/test_parse_request.py b/matemat/webserver/test/test_parse_request.py index acb225c..d144ac1 100644 --- a/matemat/webserver/test/test_parse_request.py +++ b/matemat/webserver/test/test_parse_request.py @@ -7,16 +7,25 @@ from matemat.webserver.util import parse_args class TestParseRequest(unittest.TestCase): def test_parse_get_root(self): + """ + Test that the simple root path is parsed correctly ('/' path, no args). + """ path, args = parse_args('/') self.assertEqual('/', path) self.assertEqual(0, len(args)) def test_parse_get_no_args(self): + """ + Test that a GET request without arguments is parsed correctly. + """ path, args = parse_args('/index.html') self.assertEqual('/index.html', path) self.assertEqual(0, len(args)) def test_parse_get_root_getargs(self): + """ + Test that a GET request for '/' with scalar arguments is parsed correctly. + """ path, args = parse_args('/?foo=42&bar=1337&baz=Hello,%20World!') self.assertEqual('/', path) self.assertEqual(3, len(args)) @@ -34,6 +43,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_parse_get_getargs(self): + """ + Test that a GET request for an arbitrary path with scalar arguments is parsed correctly. + """ path, args = parse_args('/abc/def?foo=42&bar=1337&baz=Hello,%20World!') self.assertEqual('/abc/def', path) self.assertEqual(3, len(args)) @@ -51,6 +63,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_parse_get_getarray(self): + """ + Test that a GET request with mixed scalar and array arguments is parsed correctly. + """ path, args = parse_args('/abc/def?foo=42&foo=1337&baz=Hello,%20World!') self.assertEqual('/abc/def', path) self.assertEqual(2, len(args)) @@ -63,6 +78,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('1337', args['foo'].get_str(1)) def test_parse_get_zero_arg(self): + """ + Test that a GET request with an empty argument is parsed correctly. + """ path, args = parse_args('/abc/def?foo=&bar=42') self.assertEqual(2, len(args)) self.assertIn('foo', args) @@ -74,10 +92,16 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('42', args['bar'].get_str()) def test_parse_get_urlencoded_encoding_fail(self): + """ + Test that a GET request with non-decodable escape sequences fails. + """ with self.assertRaises(ValueError): parse_args('/?foo=42&bar=%80&baz=Hello,%20World!') def test_parse_post_urlencoded(self): + """ + Test that a urlencoded POST request with scalar arguments is parsed correctly. + """ path, args = parse_args('/', postbody=b'foo=42&bar=1337&baz=Hello,%20World!', enctype='application/x-www-form-urlencoded') @@ -97,6 +121,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_parse_post_urlencoded_array(self): + """ + Test that a urlencoded POST request with mixed scalar and array arguments is parsed correctly. + """ path, args = parse_args('/', postbody=b'foo=42&foo=1337&baz=Hello,%20World!', enctype='application/x-www-form-urlencoded') @@ -111,6 +138,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('1337', args['foo'].get_str(1)) def test_parse_post_urlencoded_zero_arg(self): + """ + Test that a urlencoded POST request with an empty argument is parsed correctly. + """ path, args = parse_args('/abc/def', postbody=b'foo=&bar=42', enctype='application/x-www-form-urlencoded') self.assertEqual(2, len(args)) self.assertIn('foo', args) @@ -122,12 +152,18 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('42', args['bar'].get_str()) def test_parse_post_urlencoded_encoding_fail(self): + """ + Test that a urlencoded POST request with a non-decodable escape sequence fails. + """ with self.assertRaises(ValueError): parse_args('/', postbody=b'foo=42&bar=%80&baz=Hello,%20World!', enctype='application/x-www-form-urlencoded') def test_parse_post_multipart_no_args(self): + """ + Test that a multipart POST request with no arguments is parsed correctly. + """ path, args = parse_args('/', postbody=b'--testBoundary1337--\r\n', enctype='multipart/form-data; boundary=testBoundary1337') @@ -135,6 +171,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual(0, len(args)) def test_parse_post_multipart(self): + """ + Test that a multipart POST request with scalar arguments is parsed correctly. + """ path, args = parse_args('/', postbody=b'--testBoundary1337\r\n' b'Content-Disposition: form-data; name="foo"\r\n' @@ -166,6 +205,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_parse_post_multipart_zero_arg(self): + """ + Test that a multipart POST request with an empty argument is parsed correctly. + """ path, args = parse_args('/abc/def', postbody=b'--testBoundary1337\r\n' b'Content-Disposition: form-data; name="foo"\r\n' @@ -215,6 +257,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_parse_post_multipart_broken_boundaries(self): + """ + Test that multiple cases with broken multipart boundaries fail. + """ with self.assertRaises(ValueError): # Boundary not defined in Content-Type parse_args('/', @@ -318,6 +363,9 @@ class TestParseRequest(unittest.TestCase): enctype='multipart/form-data; boundary=testBoundary1337') def test_get_post_precedence_urlencoded(self): + """ + Test the precedence of urlencoded POST arguments over GET arguments. + """ path, args = parse_args('/foo?foo=thisshouldnotbethere&bar=isurvived', postbody=b'foo=42&foo=1337&baz=Hello,%20World!', enctype='application/x-www-form-urlencoded') @@ -333,6 +381,9 @@ class TestParseRequest(unittest.TestCase): self.assertEqual('Hello, World!', args['baz'].get_str()) def test_get_post_precedence_multipart(self): + """ + Test the precedence of multipart POST arguments over GET arguments. + """ path, args = parse_args('/foo?foo=thisshouldnotbethere&bar=isurvived', postbody=b'--testBoundary1337\r\n' b'Content-Disposition: form-data; name="foo"\r\n' From f1ff14d29cc124f2ca4e698d1ac28ff79eea97d2 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 18:58:23 +0200 Subject: [PATCH 37/46] More user-friendly return value in the pagelet API. --- matemat/exceptions/AuthenticatonError.py | 2 +- matemat/exceptions/HttpException.py | 20 ++++ matemat/exceptions/__init__.py | 1 + matemat/webserver/httpd.py | 106 ++++++++++++------ matemat/webserver/pagelets/login.py | 19 ++-- matemat/webserver/pagelets/logout.py | 5 +- matemat/webserver/pagelets/main.py | 39 +++---- matemat/webserver/pagelets/touchkey.py | 19 ++-- matemat/webserver/test/abstract_httpd_test.py | 8 +- matemat/webserver/test/test_post.py | 2 +- matemat/webserver/test/test_serve.py | 10 +- matemat/webserver/test/test_session.py | 2 +- 12 files changed, 141 insertions(+), 92 deletions(-) create mode 100644 matemat/exceptions/HttpException.py diff --git a/matemat/exceptions/AuthenticatonError.py b/matemat/exceptions/AuthenticatonError.py index 0d23c2e..d332fdc 100644 --- a/matemat/exceptions/AuthenticatonError.py +++ b/matemat/exceptions/AuthenticatonError.py @@ -2,7 +2,7 @@ from typing import Optional -class AuthenticationError(BaseException): +class AuthenticationError(Exception): def __init__(self, msg: Optional[str] = None) -> None: super().__init__() diff --git a/matemat/exceptions/HttpException.py b/matemat/exceptions/HttpException.py new file mode 100644 index 0000000..4f792ce --- /dev/null +++ b/matemat/exceptions/HttpException.py @@ -0,0 +1,20 @@ + +class HttpException(Exception): + + def __init__(self, status: int = 500, title: str = 'An error occurred', message: str = None) -> None: + super().__init__() + self.__status: int = status + self.__title: str = title + self.__message: str = message + + @property + def status(self) -> int: + return self.__status + + @property + def title(self) -> str: + return self.__title + + @property + def message(self) -> str: + return self.__message diff --git a/matemat/exceptions/__init__.py b/matemat/exceptions/__init__.py index ebae437..9b10b5f 100644 --- a/matemat/exceptions/__init__.py +++ b/matemat/exceptions/__init__.py @@ -4,3 +4,4 @@ This package provides custom exception classes used in the Matemat codebase. from .AuthenticatonError import AuthenticationError from .DatabaseConsistencyError import DatabaseConsistencyError +from .HttpException import HttpException diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index b703f72..a161d42 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import Any, Callable, Dict, Tuple, Union import traceback @@ -13,10 +13,10 @@ from uuid import uuid4 from datetime import datetime, timedelta from matemat import __version__ as matemat_version +from matemat.exceptions import HttpException from matemat.webserver import RequestArguments from matemat.webserver.util import parse_args - # # Python internal class hacks # @@ -27,15 +27,16 @@ TCPServer.address_family = socket.AF_INET6 BaseHTTPRequestHandler.log_request = lambda self, code='-', size='-': None BaseHTTPRequestHandler.log_error = lambda self, fstring='', *args: None - # Dictionary to hold registered pagelet paths and their handler functions _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) str, # Request path RequestArguments, # HTTP Request arguments Dict[str, Any], # Session vars Dict[str, str]], # Response headers - Tuple[int, Union[bytes, str]]]] = dict() # Returns: (status code, response body) - + Union[ # Return type: either a response body, or a redirect + bytes, str, # Response body: will assign HTTP/1.0 200 OK + Tuple[int, str] # Redirect: First element must be 301, second the redirect path + ]]] = dict() # Inactivity timeout for client sessions _SESSION_TIMEOUT: int = 3600 @@ -54,15 +55,17 @@ def pagelet(path: str): args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]) - -> (int, Optional[Union[str, bytes]]) + -> Union[bytes, str, Tuple[int, str]] method: The HTTP method (GET, POST) that was used. path: The path that was requested. args: The arguments that were passed with the request (as GET or POST arguments). session_vars: The session storage. May be read from and written to. headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. - returns: A tuple consisting of the HTTP status code (as an int) and the response body (as str or bytes, - may be None) + returns: One of the following: + - A HTTP Response body as str or bytes + - A HTTP redirect: A tuple of 301 (an int) and the path to redirect to (a str) + raises: HttpException: If a non-200 HTTP status code should be returned :param path: The path to register the function for. """ @@ -72,11 +75,15 @@ def pagelet(path: str): RequestArguments, Dict[str, Any], Dict[str, str]], - Tuple[int, Optional[Union[bytes, str]]]]): + Union[ + bytes, str, + Tuple[int, str] + ]]): # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun # Don't change the function itself at all return fun + # Return the inner function (Python requires a "real" function annotation to not have any arguments except # the function itself) return http_handler @@ -167,7 +174,7 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id not in self.server.session_vars: self.server.session_vars[session_id] = (now + timedelta(seconds=_SESSION_TIMEOUT)), dict() else: - self.server.session_vars[session_id] =\ + self.server.session_vars[session_id] = \ (now + timedelta(seconds=_SESSION_TIMEOUT), self.server.session_vars[session_id][1]) # Return the session ID and timeout return session_id, self.server.session_vars[session_id][0] @@ -181,6 +188,45 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] + @staticmethod + def _parse_pagelet_result(pagelet_res: Union[bytes, str, Tuple[int, str]], headers: Dict[str, str]) \ + -> Tuple[int, bytes]: + """ + Process the return value of a pagelet function call. + + :param pagelet_res: The pagelet return value. + :param headers: The dict of HTTP response headers, needed for setting the redirect header. + :return: The HTTP Response status code (an int) and body (a bytes). + :raises TypeError: If the pagelet result was not in the expected form. + """ + # The HTTP Response Status Code, defaults to 200 OK + hsc: int = 200 + # The HTTP Response body, defaults to None + data: Union[bytes, str] = None + if isinstance(pagelet_res, tuple): + # If the return type is a tuple, the first element must be 301 (the HTTP Redirect status code) + head, tail = pagelet_res + if head == 301: + # Set the HTTP Response Status Code, and the redirect header + hsc = 301 + headers['Location'] = tail + else: + raise TypeError(f'Return value of pagelet not understood: {pagelet_res}') + elif isinstance(pagelet_res, str) or isinstance(pagelet_res, bytes): + # Return value is a response body + data = pagelet_res + else: + # Return value is not a response body or a redirect + raise TypeError(f'Return value of pagelet not understood: {pagelet_res}') + # The pagelet may return None as data as a shorthand for an empty response + if data is None: + data = bytes() + # If the pagelet returns a Python str, convert it to an UTF-8 encoded bytes object + if isinstance(data, str): + data = data.encode('utf-8') + # Return the resulting status code and body + return hsc, data + def _handle(self, method: str, path: str, args: RequestArguments) -> None: """ Handle a HTTP request by either dispatching it to the appropriate pagelet or by serving a static resource. @@ -209,13 +255,9 @@ class HttpHandler(BaseHTTPRequestHandler): 'Cache-Control': 'no-cache' } # Call the pagelet function - hsc, data = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) - # The pagelet may return None as data as a shorthand for an empty response - if data is None: - data = bytes() - # If the pagelet returns a Python str, convert it to an UTF-8 encoded bytes object - if isinstance(data, str): - data = data.encode('utf-8') + pagelet_res = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) + # Parse the pagelet's return value, vielding a HTTP status code and a response body + hsc, data = HttpHandler._parse_pagelet_result(pagelet_res, headers) # Send the HTTP status code self.send_response(hsc) # Format the session cookie timeout string and send the session cookie header @@ -272,16 +314,18 @@ class HttpHandler(BaseHTTPRequestHandler): path, args = parse_args(self.path) self._handle('GET', path, args) # Special handling for some errors + except HttpException as e: + self.send_error(e.status, e.title, e.message) except PermissionError: - self.send_response(403, 'Forbidden') - self.end_headers() + self.send_error(403, 'Forbidden') except ValueError: - self.send_response(400, 'Bad Request') - self.end_headers() - except BaseException: + self.send_error(400, 'Bad Request') + except BaseException as e: # Generic error handling - self.send_response(500, 'Internal Server Error') - self.end_headers() + self.send_error(500, 'Internal Server Error') + print(e) + + traceback.print_tb(e.__traceback__) # noinspection PyPep8Naming def do_POST(self) -> None: @@ -299,19 +343,15 @@ class HttpHandler(BaseHTTPRequestHandler): # Parse the request and hand it to the handle function self._handle('POST', path, args) # Special handling for some errors + except HttpException as e: + self.send_error(e.status, e.title, e.message) except PermissionError: - self.send_response(403, 'Forbidden') - self.end_headers() + self.send_error(403, 'Forbidden') except ValueError: - self.send_response(400, 'Bad Request') - self.end_headers() - except TypeError: - self.send_response(400, 'Bad Request') - self.end_headers() + self.send_error(400, 'Bad Request') except BaseException as e: # Generic error handling - self.send_response(500, 'Internal Server Error') - self.end_headers() + self.send_error(500, 'Internal Server Error') print(e) traceback.print_tb(e.__traceback__) diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index 7d0cc2d..150b174 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Tuple, Union -from matemat.exceptions import AuthenticationError +from matemat.exceptions import AuthenticationError, HttpException from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User from matemat.db import MatematDatabase @@ -13,10 +13,9 @@ def login_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Tuple[int, Optional[Union[str, bytes]]]: + -> Union[bytes, str, Tuple[int, str]]: if 'user' in session_vars: - headers['Location'] = '/' - return 301, None + return 301, '/' if method == 'GET': data = ''' @@ -41,15 +40,13 @@ def login_page(method: str, ''' - return 200, data.format(msg=str(args.msg) if 'msg' in args else '') + return data.format(msg=str(args.msg) if 'msg' in args else '') elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), str(args.password)) except AuthenticationError: - headers['Location'] = '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' - return 301, bytes() + return 301, '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' session_vars['user'] = user - headers['Location'] = '/' - return 301, bytes() - return 405, None + return 301, '/' + raise HttpException(405) diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index beb86a3..05bcfe2 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -10,8 +10,7 @@ def logout(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Tuple[int, Optional[Union[str, bytes]]]: + -> Union[bytes, str, Tuple[int, str]]: if 'user' in session_vars: del session_vars['user'] - headers['Location'] = '/' - return 301, None + return 301, '/' diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index e22c872..db4c49a 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -12,7 +12,7 @@ def main_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Tuple[int, Optional[Union[str, bytes]]]: + -> Union[bytes, str, Tuple[int, str]]: data = ''' @@ -34,24 +34,19 @@ def main_page(method: str, ''' - try: - with MatematDatabase('test.db') as db: - if 'user' in session_vars: - user: User = session_vars['user'] - products = db.list_products() - plist = '\n'.join([f'
  • {p.name} ' + - f'{p.price_member//100 if user.is_member else p.price_non_member//100}' + - f'.{p.price_member%100 if user.is_member else p.price_non_member%100}' - for p in products]) - uname = f'{user.name} (Logout)' - data = data.format(user=uname, list=plist) - else: - users = db.list_users() - ulist = '\n'.join([f'
  • {u.name}' for u in users]) - ulist = ulist + '
  • Password login' - data = data.format(user='', list=ulist) - return 200, data - except BaseException as e: - import traceback - traceback.print_tb(e.__traceback__) - return 500, None + with MatematDatabase('test.db') as db: + if 'user' in session_vars: + user: User = session_vars['user'] + products = db.list_products() + plist = '\n'.join([f'
  • {p.name} ' + + f'{p.price_member//100 if user.is_member else p.price_non_member//100}' + + f'.{p.price_member%100 if user.is_member else p.price_non_member%100}' + for p in products]) + uname = f'{user.name} (Logout)' + data = data.format(user=uname, list=plist) + else: + users = db.list_users() + ulist = '\n'.join([f'
  • {u.name}' for u in users]) + ulist = ulist + '
  • Password login' + data = data.format(user='', list=ulist) + return data diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 4de8009..280610e 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Tuple, Union -from matemat.exceptions import AuthenticationError +from matemat.exceptions import AuthenticationError, HttpException from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User from matemat.db import MatematDatabase @@ -13,10 +13,9 @@ def touchkey_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Tuple[int, Optional[Union[str, bytes]]]: + -> Union[bytes, str, Tuple[int, str]]: if 'user' in session_vars: - headers['Location'] = '/' - return 301, bytes() + return 301, '/' if method == 'GET': data = ''' @@ -40,15 +39,13 @@ def touchkey_page(method: str, ''' - return 200, data.format(username=str(args.username) if 'username' in args else '') + return data.format(username=str(args.username) if 'username' in args else '') elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), touchkey=str(args.touchkey)) except AuthenticationError: - headers['Location'] = f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' - return 301, bytes() + return 301, f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' session_vars['user'] = user - headers['Location'] = '/' - return 301, None - return 405, None + return 301, '/' + raise HttpException(405) diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index 103979b..ff4c0c6 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -16,6 +16,8 @@ class HttpResponse: """ A really basic HTTP response container and parser class, just good enough for unit testing a HTTP server, if even. + DO NOT USE THIS OUTSIDE UNIT TESTING! + Usage: response = HttpResponse() while response.parse_phase != 'done' @@ -161,16 +163,16 @@ def test_pagelet(path: str): RequestArguments, Dict[str, Any], Dict[str, str]], - Tuple[int, Union[bytes, str]]]): + Union[bytes, str, Tuple[int, str]]]): @pagelet(path) def testing_wrapper(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str]): - status, body = fun(method, path, args, session_vars, headers) headers['X-Test-Pagelet'] = fun.__name__ - return status, body + result = fun(method, path, args, session_vars, headers) + return result return testing_wrapper return with_testing_headers diff --git a/matemat/webserver/test/test_post.py b/matemat/webserver/test/test_post.py index 0bc5d16..e105354 100644 --- a/matemat/webserver/test/test_post.py +++ b/matemat/webserver/test/test_post.py @@ -24,7 +24,7 @@ def post_test_pagelet(method: str, dump += f'{a.name}: {a.get_str()}\n' else: dump += f'{a.name}: {codecs.encode(a.get_bytes(), "hex").decode("utf-8")}\n' - return 200, dump + return dump class TestPost(AbstractHttpdTest): diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 722870b..b507df7 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -3,6 +3,7 @@ from typing import Any, Dict import os import os.path +from matemat.exceptions import HttpException from matemat.webserver import HttpHandler, RequestArguments from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet @@ -14,7 +15,7 @@ def serve_test_pagelet_ok(method: str, session_vars: Dict[str, Any], headers: Dict[str, str]): headers['Content-Type'] = 'text/plain' - return 200, 'serve test pagelet ok' + return 'serve test pagelet ok' @test_pagelet('/just/testing/serve_pagelet_fail') @@ -25,7 +26,7 @@ def serve_test_pagelet_fail(method: str, headers: Dict[str, str]): session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' - return 500, 'serve test pagelet fail' + raise HttpException() class TestServe(AbstractHttpdTest): @@ -62,11 +63,8 @@ class TestServe(AbstractHttpdTest): HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() - # Make sure the correct pagelet was called - self.assertEqual('serve_test_pagelet_fail', packet.pagelet) - # Make sure the expected content is served + # Make sure an error is raised self.assertEqual(500, packet.statuscode) - self.assertEqual(b'serve test pagelet fail', packet.body) def test_serve_static_ok(self): # Request a static resource diff --git a/matemat/webserver/test/test_session.py b/matemat/webserver/test/test_session.py index 5cf408e..2e64ceb 100644 --- a/matemat/webserver/test/test_session.py +++ b/matemat/webserver/test/test_session.py @@ -16,7 +16,7 @@ def session_test_pagelet(method: str, headers: Dict[str, str]): session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' - return 200, 'session test' + return 'session test' class TestSession(AbstractHttpdTest): From a53144797020584fc9d2d1e5ac3bdfea79cd4031 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 19:10:41 +0200 Subject: [PATCH 38/46] Added the changed pagelet API to the external documentation. --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 14b8380..51e9404 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 14b8380090858c3bed5c3b2ee7cf1408aaa133df +Subproject commit 51e940460ddbaebb7f2ffc48d00d9ef19cf8d33f From 14f339e63052e8d05c53741b7493327aef648542 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sat, 7 Jul 2018 19:24:00 +0200 Subject: [PATCH 39/46] Added unit test for redirection testing. --- matemat/webserver/httpd.py | 1 - matemat/webserver/test/test_serve.py | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index a161d42..3892ac8 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -324,7 +324,6 @@ class HttpHandler(BaseHTTPRequestHandler): # Generic error handling self.send_error(500, 'Internal Server Error') print(e) - traceback.print_tb(e.__traceback__) # noinspection PyPep8Naming diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index b507df7..5bcdd4d 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -18,6 +18,15 @@ def serve_test_pagelet_ok(method: str, return 'serve test pagelet ok' +@test_pagelet('/just/testing/serve_pagelet_redirect') +def serve_test_pagelet_redirect(method: str, + path: str, + args: RequestArguments, + session_vars: Dict[str, Any], + headers: Dict[str, str]): + return 301, '/foo/bar' + + @test_pagelet('/just/testing/serve_pagelet_fail') def serve_test_pagelet_fail(method: str, path: str, @@ -66,6 +75,18 @@ class TestServe(AbstractHttpdTest): # Make sure an error is raised self.assertEqual(500, packet.statuscode) + def test_serve_pagelet_redirect(self): + # Call the test pagelet that redirects to another path + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_redirect HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure the correct redirect is issued + self.assertEqual(301, packet.statuscode) + self.assertEqual('/foo/bar', packet.headers['Location']) + # Make sure the response body is empty + self.assertEqual(0, len(packet.body)) + def test_serve_static_ok(self): # Request a static resource self.client_sock.set_request(b'GET /static_resource.txt HTTP/1.1\r\n\r\n') From 079d9909c0ce233a3375e20869c795b7812afc29 Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 8 Jul 2018 15:10:22 +0200 Subject: [PATCH 40/46] Some typing fixes that make mypy a litte happier --- matemat/webserver/httpd.py | 28 +++++++++++++++++++++------- matemat/webserver/requestargs.py | 2 +- matemat/webserver/util.py | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index b703f72..3215ad2 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -1,5 +1,5 @@ -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import Any, Callable, Dict, Optional, Tuple, Type, Union import traceback @@ -34,7 +34,7 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) RequestArguments, # HTTP Request arguments Dict[str, Any], # Session vars Dict[str, str]], # Response headers - Tuple[int, Union[bytes, str]]]] = dict() # Returns: (status code, response body) + Tuple[int, Optional[Union[bytes, str]]]]] = dict() # Returns: (status code, response body) # Inactivity timeout for client sessions @@ -82,6 +82,23 @@ def pagelet(path: str): return http_handler +class MatematHTTPServer(HTTPServer): + """ + A http.server.HTTPServer subclass that acts as a container for data that must be persistent between requests. + """ + + def __init__(self, + server_address: Any, + handler: Type[BaseHTTPRequestHandler], + webroot: str, + bind_and_activate: bool = True) -> None: + super().__init__(server_address, handler, bind_and_activate) + # Resolve webroot directory + self.webroot = os.path.abspath(webroot) + # Set up session vars dict + self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + + class MatematWebserver(object): """ Then main webserver class, internally uses Python's http.server. @@ -113,11 +130,7 @@ class MatematWebserver(object): # Rewrite IPv4 address to IPv6-mapped form listen = f'::ffff:{listen}' # Create the http server - self._httpd = HTTPServer((listen, port), HttpHandler) - # Set up session vars dict - self._httpd.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() - # Resolve webroot directory - self._httpd.webroot = os.path.abspath(webroot) + self._httpd = MatematHTTPServer((listen, port), HttpHandler, webroot) def start(self) -> None: """ @@ -136,6 +149,7 @@ class HttpHandler(BaseHTTPRequestHandler): def __init__(self, request: bytes, client_address: Tuple[str, int], server: HTTPServer) -> None: super().__init__(request, client_address, server) + self.server: MatematHTTPServer @property def server_version(self) -> str: diff --git a/matemat/webserver/requestargs.py b/matemat/webserver/requestargs.py index 2150b31..373db90 100644 --- a/matemat/webserver/requestargs.py +++ b/matemat/webserver/requestargs.py @@ -50,7 +50,7 @@ class RequestArguments(object): """ return _View.of(self.__container[key]) - def __iter__(self) -> Iterator['RequestArguments']: + def __iter__(self) -> Iterator['RequestArgument']: """ Returns an iterator over the values in this instance. Values are represented as immutable views. diff --git a/matemat/webserver/util.py b/matemat/webserver/util.py index 6e19d7b..c95d303 100644 --- a/matemat/webserver/util.py +++ b/matemat/webserver/util.py @@ -97,7 +97,7 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't tokens = urllib.parse.urlparse(request) # Parse the GET arguments if len(tokens.query) == 0: - getargs = dict() + getargs: Dict[str, List[str]] = dict() else: getargs = urllib.parse.parse_qs(tokens.query, strict_parsing=True, keep_blank_values=True, errors='strict') @@ -112,7 +112,7 @@ def parse_args(request: str, postbody: Optional[bytes] = None, enctype: str = 't # Parse the POST body pb: str = postbody.decode('utf-8') if len(pb) == 0: - postargs = dict() + postargs: Dict[str, List[str]] = dict() else: postargs = urllib.parse.parse_qs(pb, strict_parsing=True, keep_blank_values=True, errors='strict') # Write all POST values into the dict, overriding potential duplicates from GET From a0d1520ecfb93cc81711b4cb46b4f203d3398e6e Mon Sep 17 00:00:00 2001 From: s3lph Date: Sun, 8 Jul 2018 15:13:58 +0200 Subject: [PATCH 41/46] Fixed a line-to-long style error --- matemat/webserver/httpd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 3215ad2..2bae715 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -34,7 +34,8 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) RequestArguments, # HTTP Request arguments Dict[str, Any], # Session vars Dict[str, str]], # Response headers - Tuple[int, Optional[Union[bytes, str]]]]] = dict() # Returns: (status code, response body) + # Returns: (status code, response body) + Tuple[int, Optional[Union[bytes, str]]]]] = dict() # Inactivity timeout for client sessions From 4d2d2d30c1a7943d25dcd204a8a38bca870126fe Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 9 Jul 2018 00:11:40 +0200 Subject: [PATCH 42/46] Added Jinja2 dependency --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index d1eec08..b27f921 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ This project intends to provide a well-tested and maintainable alternative to - Python 3 (>=3.6) - Python dependencies: - apsw + - jinja2 ## Usage diff --git a/requirements.txt b/requirements.txt index b8ab4ea..7663204 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ apsw +jinja2 From e3c65776b55835b2831c5b2305825e3fc830b796 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 9 Jul 2018 01:09:53 +0200 Subject: [PATCH 43/46] A first, semi-sane integration of Jinja2 templates --- matemat/webserver/httpd.py | 55 +++++++++++++++++++------- matemat/webserver/pagelets/login.py | 29 ++------------ matemat/webserver/pagelets/logout.py | 2 +- matemat/webserver/pagelets/main.py | 35 ++-------------- matemat/webserver/pagelets/touchkey.py | 31 +++------------ templates/login.html | 20 ++++++++++ templates/main.html | 33 ++++++++++++++++ templates/touchkey.html | 20 ++++++++++ 8 files changed, 127 insertions(+), 98 deletions(-) create mode 100644 templates/login.html create mode 100644 templates/main.html create mode 100644 templates/touchkey.html diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index 1f99aa1..c82ba10 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -12,6 +12,8 @@ from http.cookies import SimpleCookie from uuid import uuid4 from datetime import datetime, timedelta +import jinja2 + from matemat import __version__ as matemat_version from matemat.exceptions import HttpException from matemat.webserver import RequestArguments @@ -35,7 +37,8 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) Dict[str, str]], # Response headers Union[ # Return type: either a response body, or a redirect bytes, str, # Response body: will assign HTTP/1.0 200 OK - Tuple[int, str] # Redirect: First element must be 301, second the redirect path + Tuple[int, str], # Redirect: First element must be 301, second the redirect path + Tuple[str, Dict[str, Any]] # Jinja template name and kwargs ]]] = dict() # Inactivity timeout for client sessions @@ -65,6 +68,8 @@ def pagelet(path: str): returns: One of the following: - A HTTP Response body as str or bytes - A HTTP redirect: A tuple of 301 (an int) and the path to redirect to (a str) + - A Jinja template call: A tuple of the template name (a string) and the template rendering + arguments (a kwargs dict) raises: HttpException: If a non-200 HTTP status code should be returned :param path: The path to register the function for. @@ -77,7 +82,8 @@ def pagelet(path: str): Dict[str, str]], Union[ bytes, str, - Tuple[int, str] + Tuple[int, str], + Tuple[str, Dict[str, Any]] ]]): # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun @@ -97,13 +103,18 @@ class MatematHTTPServer(HTTPServer): def __init__(self, server_address: Any, handler: Type[BaseHTTPRequestHandler], - webroot: str, + staticroot: str, + templateroot: str, bind_and_activate: bool = True) -> None: super().__init__(server_address, handler, bind_and_activate) # Resolve webroot directory - self.webroot = os.path.abspath(webroot) + self.webroot = os.path.abspath(staticroot) # Set up session vars dict self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() + # Set up the Jinja2 environment + self.jinja_env: jinja2.Environment = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.abspath(templateroot)) + ) class MatematWebserver(object): @@ -121,13 +132,18 @@ class MatematWebserver(object): server.start() """ - def __init__(self, listen: str = '::', port: int = 80, webroot: str = './webroot') -> None: + def __init__(self, + listen: str = '::', + port: int = 80, + staticroot: str = './static', + templateroot: str = './templates') -> None: """ Instantiate a MatematWebserver. - :param listen: The IPv4 or IPv6 address to listen on - :param port: The TCP port to listen on - :param webroot: Path to the webroot directory + :param listen: The IPv4 or IPv6 address to listen on. + :param port: The TCP port to listen on. + :param staticroot: Path to the static webroot directory. + :param templateroot: Path to the Jinja2 templates root directory. """ if len(listen) == 0: # Empty string should be interpreted as all addresses @@ -137,7 +153,7 @@ class MatematWebserver(object): # Rewrite IPv4 address to IPv6-mapped form listen = f'::ffff:{listen}' # Create the http server - self._httpd = MatematHTTPServer((listen, port), HttpHandler, webroot) + self._httpd = MatematHTTPServer((listen, port), HttpHandler, staticroot, templateroot) def start(self) -> None: """ @@ -202,8 +218,12 @@ class HttpHandler(BaseHTTPRequestHandler): if session_id in self.server.session_vars: del self.server.session_vars[session_id] - @staticmethod - def _parse_pagelet_result(pagelet_res: Union[bytes, str, Tuple[int, str]], headers: Dict[str, str]) \ + def _parse_pagelet_result(self, + pagelet_res: Union[bytes, # Response body as bytes + str, # Response body as str + Tuple[int, str], # Redirect + Tuple[str, Dict[str, Any]]], # Jinja template name, kwargs dict + headers: Dict[str, str]) \ -> Tuple[int, bytes]: """ Process the return value of a pagelet function call. @@ -218,12 +238,19 @@ class HttpHandler(BaseHTTPRequestHandler): # The HTTP Response body, defaults to None data: Union[bytes, str] = None if isinstance(pagelet_res, tuple): - # If the return type is a tuple, the first element must be 301 (the HTTP Redirect status code) + # If the return type is a tuple, it has to be either a redirect, in which case the first element must be + # int(301), or it is a template call, in which casse the first element must be the template name and the + # second element must be the kwargs dict to the template's render function head, tail = pagelet_res - if head == 301: + if head == 301 and isinstance(tail, str): # Set the HTTP Response Status Code, and the redirect header hsc = 301 headers['Location'] = tail + elif isinstance(head, str) and isinstance(tail, dict): + # Load the Jinja2 template and render it with the provided arguments + template = self.server.jinja_env.get_template(head) + tail['matemat_version'] = self.server_version + data = template.render(**tail) else: raise TypeError(f'Return value of pagelet not understood: {pagelet_res}') elif isinstance(pagelet_res, str) or isinstance(pagelet_res, bytes): @@ -271,7 +298,7 @@ class HttpHandler(BaseHTTPRequestHandler): # Call the pagelet function pagelet_res = _PAGELET_PATHS[path](method, path, args, self.session_vars, headers) # Parse the pagelet's return value, vielding a HTTP status code and a response body - hsc, data = HttpHandler._parse_pagelet_result(pagelet_res, headers) + hsc, data = self._parse_pagelet_result(pagelet_res, headers) # Send the HTTP status code self.send_response(hsc) # Format the session cookie timeout string and send the session cookie header diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index 150b174..d22fcd7 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -13,40 +13,17 @@ def login_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str]]: + -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: if 'user' in session_vars: return 301, '/' if method == 'GET': - data = ''' - - - - Matemat - - - -

    Matemat

    - {msg} -
    - Username:
    - Password:
    - -
    - - - ''' - return data.format(msg=str(args.msg) if 'msg' in args else '') + return 'login.html', {} elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), str(args.password)) except AuthenticationError: - return 301, '/login?msg=Username%20or%20password%20wrong.%20Please%20try%20again.' + return 301, '/login' session_vars['user'] = user return 301, '/' raise HttpException(405) diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index 05bcfe2..766558a 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -10,7 +10,7 @@ def logout(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str]]: + -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: if 'user' in session_vars: del session_vars['user'] return 301, '/' diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index db4c49a..24e6f0b 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -12,41 +12,12 @@ def main_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str]]: - data = ''' - - - - Matemat - - - -

    Matemat

    - {user} -
      - {list} -
    - - - ''' + -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: with MatematDatabase('test.db') as db: if 'user' in session_vars: user: User = session_vars['user'] products = db.list_products() - plist = '\n'.join([f'
  • {p.name} ' + - f'{p.price_member//100 if user.is_member else p.price_non_member//100}' + - f'.{p.price_member%100 if user.is_member else p.price_non_member%100}' - for p in products]) - uname = f'{user.name} (Logout)' - data = data.format(user=uname, list=plist) + return 'main.html', {'user': user, 'list': products} else: users = db.list_users() - ulist = '\n'.join([f'
  • {u.name}' for u in users]) - ulist = ulist + '
  • Password login' - data = data.format(user='', list=ulist) - return data + return 'main.html', {'list': users} diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 280610e..27c943e 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -1,6 +1,8 @@ from typing import Any, Dict, Tuple, Union +import urllib.parse + from matemat.exceptions import AuthenticationError, HttpException from matemat.webserver import pagelet, RequestArguments from matemat.primitives import User @@ -13,39 +15,18 @@ def touchkey_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str]]: + -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: if 'user' in session_vars: return 301, '/' if method == 'GET': - data = ''' - - - - Matemat - - - -

    Matemat

    -
    -
    - Touchkey:
    - -
    - - - ''' - return data.format(username=str(args.username) if 'username' in args else '') + return 'touchkey.html', {'username': str(args.username)} if 'username' in args else {} elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), touchkey=str(args.touchkey)) except AuthenticationError: - return 301, f'/touchkey?username={args["username"]}&msg=Please%20try%20again.' + quoted = urllib.parse.quote_plus(bytes(args.username)) + return 301, f'/touchkey?username={quoted}' session_vars['user'] = user return 301, '/' raise HttpException(405) diff --git a/templates/login.html b/templates/login.html new file mode 100644 index 0000000..bdb7168 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,20 @@ + + + + Matemat + + + +

    Matemat

    +
    + Username:
    + Password:
    + +
    + + diff --git a/templates/main.html b/templates/main.html new file mode 100644 index 0000000..d9a6d45 --- /dev/null +++ b/templates/main.html @@ -0,0 +1,33 @@ + + + + Matemat + + + +

    Matemat

    + {{ user|default("") }} +
      + {% if user is defined %} + {% for l in list %} +
    • {{ l.name }} + {% if user.is_member %} + {{ l.price_member }} + {% else %} + {{ l.price_non_member }} + {% endif %} + {% endfor %} + {% else %} + {% for l in list %} +
    • {{ l.name }} + {% endfor %} +
    • Password login + {% endif %} +
    + + diff --git a/templates/touchkey.html b/templates/touchkey.html new file mode 100644 index 0000000..ab12308 --- /dev/null +++ b/templates/touchkey.html @@ -0,0 +1,20 @@ + + + + Matemat + + + +

    Matemat

    +
    +
    + Touchkey:
    + +
    + + From 1b00c80133b2cf53835824390db6b2f462ca182d Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 9 Jul 2018 20:50:02 +0200 Subject: [PATCH 44/46] Implemented a more explicit Pagelet return API using class instances to describe the action to take. --- matemat/webserver/__init__.py | 1 + matemat/webserver/httpd.py | 68 ++++++++-------- matemat/webserver/pagelets/login.py | 14 ++-- matemat/webserver/pagelets/logout.py | 8 +- matemat/webserver/pagelets/main.py | 10 +-- matemat/webserver/pagelets/touchkey.py | 12 +-- matemat/webserver/responses.py | 63 +++++++++++++++ matemat/webserver/test/abstract_httpd_test.py | 6 ++ matemat/webserver/test/test_serve.py | 79 +++++++++++++++---- 9 files changed, 186 insertions(+), 75 deletions(-) create mode 100644 matemat/webserver/responses.py diff --git a/matemat/webserver/__init__.py b/matemat/webserver/__init__.py index c52368e..6059687 100644 --- a/matemat/webserver/__init__.py +++ b/matemat/webserver/__init__.py @@ -7,4 +7,5 @@ server will attempt to serve the request with a static resource in a previously """ from .requestargs import RequestArgument, RequestArguments +from .responses import PageletResponse, RedirectResponse, TemplateResponse from .httpd import MatematWebserver, HttpHandler, pagelet diff --git a/matemat/webserver/httpd.py b/matemat/webserver/httpd.py index c82ba10..41866de 100644 --- a/matemat/webserver/httpd.py +++ b/matemat/webserver/httpd.py @@ -16,7 +16,7 @@ import jinja2 from matemat import __version__ as matemat_version from matemat.exceptions import HttpException -from matemat.webserver import RequestArguments +from matemat.webserver import RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.webserver.util import parse_args # @@ -37,8 +37,7 @@ _PAGELET_PATHS: Dict[str, Callable[[str, # HTTP method (GET, POST, ...) Dict[str, str]], # Response headers Union[ # Return type: either a response body, or a redirect bytes, str, # Response body: will assign HTTP/1.0 200 OK - Tuple[int, str], # Redirect: First element must be 301, second the redirect path - Tuple[str, Dict[str, Any]] # Jinja template name and kwargs + PageletResponse, # A generic response ]]] = dict() # Inactivity timeout for client sessions @@ -67,9 +66,8 @@ def pagelet(path: str): headers: The dictionary of HTTP response headers. Add headers you wish to send with the response. returns: One of the following: - A HTTP Response body as str or bytes - - A HTTP redirect: A tuple of 301 (an int) and the path to redirect to (a str) - - A Jinja template call: A tuple of the template name (a string) and the template rendering - arguments (a kwargs dict) + - A PageletResponse class instance: An instance of (a subclass of) + matemat.webserver.PageletResponse, e.g. encapsulating a redirect or a Jinja2 template. raises: HttpException: If a non-200 HTTP status code should be returned :param path: The path to register the function for. @@ -82,8 +80,7 @@ def pagelet(path: str): Dict[str, str]], Union[ bytes, str, - Tuple[int, str], - Tuple[str, Dict[str, Any]] + PageletResponse ]]): # Add the function to the dict of pagelets _PAGELET_PATHS[path] = fun @@ -221,8 +218,7 @@ class HttpHandler(BaseHTTPRequestHandler): def _parse_pagelet_result(self, pagelet_res: Union[bytes, # Response body as bytes str, # Response body as str - Tuple[int, str], # Redirect - Tuple[str, Dict[str, Any]]], # Jinja template name, kwargs dict + PageletResponse], # Encapsulated or unresolved response body headers: Dict[str, str]) \ -> Tuple[int, bytes]: """ @@ -235,36 +231,34 @@ class HttpHandler(BaseHTTPRequestHandler): """ # The HTTP Response Status Code, defaults to 200 OK hsc: int = 200 - # The HTTP Response body, defaults to None - data: Union[bytes, str] = None - if isinstance(pagelet_res, tuple): - # If the return type is a tuple, it has to be either a redirect, in which case the first element must be - # int(301), or it is a template call, in which casse the first element must be the template name and the - # second element must be the kwargs dict to the template's render function - head, tail = pagelet_res - if head == 301 and isinstance(tail, str): - # Set the HTTP Response Status Code, and the redirect header - hsc = 301 - headers['Location'] = tail - elif isinstance(head, str) and isinstance(tail, dict): - # Load the Jinja2 template and render it with the provided arguments - template = self.server.jinja_env.get_template(head) - tail['matemat_version'] = self.server_version - data = template.render(**tail) - else: - raise TypeError(f'Return value of pagelet not understood: {pagelet_res}') - elif isinstance(pagelet_res, str) or isinstance(pagelet_res, bytes): - # Return value is a response body + # The HTTP Response body, defaults to empty + data: bytes = bytes() + + # If the response is a bytes object, it is used without further modification + if isinstance(pagelet_res, bytes): data = pagelet_res + + # If the response is a str object, it is encoded into a bytes object + elif isinstance(pagelet_res, str): + data = pagelet_res.encode('utf-8') + + # If the response is a PageletResponse object, the status code is extracted. Generation of the body depends + # on the subtype + elif isinstance(pagelet_res, PageletResponse): + hsc = pagelet_res.status + + # If the object is a RedirectResponse instance, no body is needed + if isinstance(pagelet_res, RedirectResponse): + headers['Location'] = pagelet_res.location + # If the object is a TemplateRespinse instance, pass the Jinja2 environment instance for rendering + elif isinstance(pagelet_res, TemplateResponse): + # noinspection PyProtectedMember + data = pagelet_res._render(self.server.jinja_env) + # else: Empty body + else: - # Return value is not a response body or a redirect raise TypeError(f'Return value of pagelet not understood: {pagelet_res}') - # The pagelet may return None as data as a shorthand for an empty response - if data is None: - data = bytes() - # If the pagelet returns a Python str, convert it to an UTF-8 encoded bytes object - if isinstance(data, str): - data = data.encode('utf-8') + # Return the resulting status code and body return hsc, data diff --git a/matemat/webserver/pagelets/login.py b/matemat/webserver/pagelets/login.py index d22fcd7..6ef9a9c 100644 --- a/matemat/webserver/pagelets/login.py +++ b/matemat/webserver/pagelets/login.py @@ -1,8 +1,8 @@ -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, Union from matemat.exceptions import AuthenticationError, HttpException -from matemat.webserver import pagelet, RequestArguments +from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.primitives import User from matemat.db import MatematDatabase @@ -13,17 +13,17 @@ def login_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: + -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: - return 301, '/' + return RedirectResponse('/') if method == 'GET': - return 'login.html', {} + return TemplateResponse('login.html') elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), str(args.password)) except AuthenticationError: - return 301, '/login' + return RedirectResponse('/login') session_vars['user'] = user - return 301, '/' + return RedirectResponse('/') raise HttpException(405) diff --git a/matemat/webserver/pagelets/logout.py b/matemat/webserver/pagelets/logout.py index 766558a..cbd7cad 100644 --- a/matemat/webserver/pagelets/logout.py +++ b/matemat/webserver/pagelets/logout.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Union -from matemat.webserver import pagelet, RequestArguments +from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse @pagelet('/logout') @@ -10,7 +10,7 @@ def logout(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: + -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: del session_vars['user'] - return 301, '/' + return RedirectResponse('/') diff --git a/matemat/webserver/pagelets/main.py b/matemat/webserver/pagelets/main.py index 24e6f0b..be0bd60 100644 --- a/matemat/webserver/pagelets/main.py +++ b/matemat/webserver/pagelets/main.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Union -from matemat.webserver import pagelet, RequestArguments +from matemat.webserver import pagelet, RequestArguments, PageletResponse, TemplateResponse from matemat.primitives import User from matemat.db import MatematDatabase @@ -12,12 +12,12 @@ def main_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: + -> Union[bytes, str, PageletResponse]: with MatematDatabase('test.db') as db: if 'user' in session_vars: user: User = session_vars['user'] products = db.list_products() - return 'main.html', {'user': user, 'list': products} + return TemplateResponse('main.html', user=user, list=products) else: users = db.list_users() - return 'main.html', {'list': users} + return TemplateResponse('main.html', list=users) diff --git a/matemat/webserver/pagelets/touchkey.py b/matemat/webserver/pagelets/touchkey.py index 27c943e..2b81c89 100644 --- a/matemat/webserver/pagelets/touchkey.py +++ b/matemat/webserver/pagelets/touchkey.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Tuple, Union import urllib.parse from matemat.exceptions import AuthenticationError, HttpException -from matemat.webserver import pagelet, RequestArguments +from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.primitives import User from matemat.db import MatematDatabase @@ -15,18 +15,18 @@ def touchkey_page(method: str, args: RequestArguments, session_vars: Dict[str, Any], headers: Dict[str, str])\ - -> Union[bytes, str, Tuple[int, str], Tuple[str, Dict[str, Any]]]: + -> Union[bytes, str, PageletResponse]: if 'user' in session_vars: - return 301, '/' + return RedirectResponse('/') if method == 'GET': - return 'touchkey.html', {'username': str(args.username)} if 'username' in args else {} + return TemplateResponse('touchkey.html', username=str(args.username) if 'username' in args else None) elif method == 'POST': with MatematDatabase('test.db') as db: try: user: User = db.login(str(args.username), touchkey=str(args.touchkey)) except AuthenticationError: quoted = urllib.parse.quote_plus(bytes(args.username)) - return 301, f'/touchkey?username={quoted}' + return RedirectResponse(f'/touchkey?username={quoted}') session_vars['user'] = user - return 301, '/' + return RedirectResponse('/') raise HttpException(405) diff --git a/matemat/webserver/responses.py b/matemat/webserver/responses.py new file mode 100644 index 0000000..f1e7f90 --- /dev/null +++ b/matemat/webserver/responses.py @@ -0,0 +1,63 @@ + +from jinja2 import Environment, Template + + +class PageletResponse: + """ + Base class for pagelet return values that require more action than simply sending plain data. + + An instance of this base class will result in an empty 200 OK response. + """ + + def __init__(self, status: int = 200): + """ + Create an empty response. + + :param status: The HTTP status code, defaults to 200 (OK). + """ + self.status: int = status + + +class RedirectResponse(PageletResponse): + """ + A pagelet response that causes the server to redirect to another location, using a 301 Permanently Moved (uncached) + response status, and a Location header. + """ + + def __init__(self, location: str): + """ + Create a redirection response with the given redirection location. + + :param location: The location to redirect to. + """ + super().__init__(status=301) + self.location: str = location + + +class TemplateResponse(PageletResponse): + """ + A pagelet response that causes the server to load a Jinja2 template and render it with the provided arguments, then + sending the result as response body, with a 200 OK response status. + """ + + def __init__(self, name: str, **kwargs): + """ + Create a template response with the given template name and arguments. + + :param name: Name of the template to load. + :param kwargs: Arguments for rendering the template, will be passed to jinja2.Template.render as is. + """ + super().__init__() + self.name: str = name + self.kwargs = kwargs + + def _render(self, jinja_env: Environment) -> bytes: + """ + Load and render the template using the Jinja2 environment managed by the web server instance. This method + should not be called by a pagelet. + + :param jinja_env: The Jinja2 environment. + :return: An UTF-8 encoded bytes object containing the template rendering result. + """ + template: Template = jinja_env.get_template(self.name) + return template.render(**self.kwargs).encode('utf-8') diff --git a/matemat/webserver/test/abstract_httpd_test.py b/matemat/webserver/test/abstract_httpd_test.py index ff4c0c6..b121dfd 100644 --- a/matemat/webserver/test/abstract_httpd_test.py +++ b/matemat/webserver/test/abstract_httpd_test.py @@ -9,6 +9,8 @@ from abc import ABC from datetime import datetime from http.server import HTTPServer +import jinja2 + from matemat.webserver import pagelet, RequestArguments @@ -107,6 +109,10 @@ class MockServer: self.session_vars: Dict[str, Tuple[datetime, Dict[str, Any]]] = dict() # Webroot for statically served content self.webroot: str = webroot + # Jinja environment with a single, static template + self.jinja_env = jinja2.Environment( + loader=jinja2.DictLoader({'test.txt': 'Hello, {{ what }}!'}) + ) class MockSocket(bytes): diff --git a/matemat/webserver/test/test_serve.py b/matemat/webserver/test/test_serve.py index 5bcdd4d..7b0b61d 100644 --- a/matemat/webserver/test/test_serve.py +++ b/matemat/webserver/test/test_serve.py @@ -1,21 +1,31 @@ -from typing import Any, Dict +from typing import Any, Dict, Union import os import os.path from matemat.exceptions import HttpException -from matemat.webserver import HttpHandler, RequestArguments +from matemat.webserver import HttpHandler, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.webserver.test.abstract_httpd_test import AbstractHttpdTest, test_pagelet -@test_pagelet('/just/testing/serve_pagelet_ok') -def serve_test_pagelet_ok(method: str, - path: str, - args: RequestArguments, - session_vars: Dict[str, Any], - headers: Dict[str, str]): +@test_pagelet('/just/testing/serve_pagelet_str') +def serve_test_pagelet_str(method: str, + path: str, + args: RequestArguments, + session_vars: Dict[str, Any], + headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: headers['Content-Type'] = 'text/plain' - return 'serve test pagelet ok' + return 'serve test pagelet str' + + +@test_pagelet('/just/testing/serve_pagelet_bytes') +def serve_test_pagelet_bytes(method: str, + path: str, + args: RequestArguments, + session_vars: Dict[str, Any], + headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers['Content-Type'] = 'application/octet-stream' + return b'serve\x80test\xffpagelet\xfebytes' @test_pagelet('/just/testing/serve_pagelet_redirect') @@ -23,16 +33,27 @@ def serve_test_pagelet_redirect(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]): - return 301, '/foo/bar' + headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + return RedirectResponse('/foo/bar') +@test_pagelet('/just/testing/serve_pagelet_template') +def serve_test_pagelet_template(method: str, + path: str, + args: RequestArguments, + session_vars: Dict[str, Any], + headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: + headers['Content-Type'] = 'text/plain' + return TemplateResponse('test.txt', what='World') + + +# noinspection PyTypeChecker @test_pagelet('/just/testing/serve_pagelet_fail') def serve_test_pagelet_fail(method: str, path: str, args: RequestArguments, session_vars: Dict[str, Any], - headers: Dict[str, str]): + headers: Dict[str, str]) -> Union[bytes, str, PageletResponse]: session_vars['test'] = 'hello, world!' headers['Content-Type'] = 'text/plain' raise HttpException() @@ -54,17 +75,29 @@ class TestServe(AbstractHttpdTest): f.write('This should not be readable') os.chmod(forbidden, 0) - def test_serve_pagelet_ok(self): + def test_serve_pagelet_str(self): # Call the test pagelet that produces a 200 OK result - self.client_sock.set_request(b'GET /just/testing/serve_pagelet_ok HTTP/1.1\r\n\r\n') + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_str HTTP/1.1\r\n\r\n') HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() # Make sure the correct pagelet was called - self.assertEqual('serve_test_pagelet_ok', packet.pagelet) + self.assertEqual('serve_test_pagelet_str', packet.pagelet) # Make sure the expected content is served self.assertEqual(200, packet.statuscode) - self.assertEqual(b'serve test pagelet ok', packet.body) + self.assertEqual(b'serve test pagelet str', packet.body) + + def test_serve_pagelet_bytes(self): + # Call the test pagelet that produces a 200 OK result + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_bytes HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure the correct pagelet was called + self.assertEqual('serve_test_pagelet_bytes', packet.pagelet) + # Make sure the expected content is served + self.assertEqual(200, packet.statuscode) + self.assertEqual(b'serve\x80test\xffpagelet\xfebytes', packet.body) def test_serve_pagelet_fail(self): # Call the test pagelet that produces a 500 Internal Server Error result @@ -81,12 +114,26 @@ class TestServe(AbstractHttpdTest): HttpHandler(self.client_sock, ('::1', 45678), self.server) packet = self.client_sock.get_response() + # Make sure the correct pagelet was called + self.assertEqual('serve_test_pagelet_redirect', packet.pagelet) # Make sure the correct redirect is issued self.assertEqual(301, packet.statuscode) self.assertEqual('/foo/bar', packet.headers['Location']) # Make sure the response body is empty self.assertEqual(0, len(packet.body)) + def test_serve_pagelet_template(self): + # Call the test pagelet that redirects to another path + self.client_sock.set_request(b'GET /just/testing/serve_pagelet_template HTTP/1.1\r\n\r\n') + HttpHandler(self.client_sock, ('::1', 45678), self.server) + packet = self.client_sock.get_response() + + # Make sure the correct pagelet was called + self.assertEqual('serve_test_pagelet_template', packet.pagelet) + self.assertEqual(200, packet.statuscode) + # Make sure the response body was rendered correctly by the templating engine + self.assertEqual(b'Hello, World!', packet.body) + def test_serve_static_ok(self): # Request a static resource self.client_sock.set_request(b'GET /static_resource.txt HTTP/1.1\r\n\r\n') From 0481b5bf9853163c055db9893103e2c518968bf5 Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 9 Jul 2018 22:46:35 +0200 Subject: [PATCH 45/46] Always add Matemat version number to Jinja template. --- matemat/webserver/responses.py | 4 +++- templates/base.html | 17 +++++++++++++++++ templates/login.html | 19 ++++--------------- templates/main.html | 19 ++++--------------- templates/touchkey.html | 19 ++++--------------- 5 files changed, 32 insertions(+), 46 deletions(-) create mode 100644 templates/base.html diff --git a/matemat/webserver/responses.py b/matemat/webserver/responses.py index f1e7f90..6f0308f 100644 --- a/matemat/webserver/responses.py +++ b/matemat/webserver/responses.py @@ -1,6 +1,8 @@ from jinja2 import Environment, Template +from matemat import __version__ + class PageletResponse: """ @@ -60,4 +62,4 @@ class TemplateResponse(PageletResponse): :return: An UTF-8 encoded bytes object containing the template rendering result. """ template: Template = jinja_env.get_template(self.name) - return template.render(**self.kwargs).encode('utf-8') + return template.render(**self.kwargs, __version__=__version__).encode('utf-8') diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..1ddd113 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,17 @@ + + + + Matemat + + + +

    Matemat {{__version__}}

    + {% block main %} + {% endblock %} + + diff --git a/templates/login.html b/templates/login.html index bdb7168..13c5dca 100644 --- a/templates/login.html +++ b/templates/login.html @@ -1,20 +1,9 @@ - - - - Matemat - - - -

    Matemat

    +{% extends "base.html" %} + +{% block main %}
    Username:
    Password:
    - - +{% endblock%} diff --git a/templates/main.html b/templates/main.html index d9a6d45..6aa5887 100644 --- a/templates/main.html +++ b/templates/main.html @@ -1,16 +1,6 @@ - - - - Matemat - - - -

    Matemat

    +{% extends "base.html" %} + +{% block main %} {{ user|default("") }}
      {% if user is defined %} @@ -29,5 +19,4 @@
    • Password login {% endif %}
    - - +{% endblock %} diff --git a/templates/touchkey.html b/templates/touchkey.html index ab12308..bb0a5b9 100644 --- a/templates/touchkey.html +++ b/templates/touchkey.html @@ -1,20 +1,9 @@ - - - - Matemat - - - -

    Matemat

    +{% extends "base.html" %} + +{% block main %}

    Touchkey:
    - - +{% endblock %} From 63a1ac291f6aaf2f1826cba21ceec69c627a798b Mon Sep 17 00:00:00 2001 From: s3lph Date: Mon, 9 Jul 2018 23:48:37 +0200 Subject: [PATCH 46/46] Updated pagelet documentation to adopt the PageletResponse return value API. --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 51e9404..9634785 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 51e940460ddbaebb7f2ffc48d00d9ef19cf8d33f +Subproject commit 9634785e5621b324f72598b3cee83bbc45c25d7b