Merge branch '18-libmagic-mime-type-detection' into staging-unstable

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 ---
-image: s3lph/matemat-ci:20180711-02
+image: s3lph/matemat-ci:20180720-01
 
 stages:
 - test

README.md

@@ -18,6 +18,7 @@ This project intends to provide a well-tested and maintainable alternative to
 
 - Python 3 (>=3.6)
 - Python dependencies:
+  - file-magic
   - jinja2
   
 ## Usage

matemat/webserver/httpd.py

@@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, Tuple, Type, Union
 import logging
 import os
 import socket
-import mimetypes
+import magic
 from socketserver import TCPServer
 from http.server import HTTPServer, BaseHTTPRequestHandler
 from http.cookies import SimpleCookie
@@ -308,7 +308,6 @@ class HttpHandler(BaseHTTPRequestHandler):
         if path in _PAGELET_PATHS:
             # Prepare some headers.  Those can still be overwritten by the pagelet
             headers: Dict[str, str] = {
-                'Content-Type': 'text/html; charset=utf-8',
                 'Cache-Control': 'no-cache'
             }
             # Call the pagelet function
@@ -328,6 +327,16 @@ class HttpHandler(BaseHTTPRequestHandler):
                              f'matemat_session_id={session_id}; expires={expires}')
             # Compute the body length and add the appropriate header
             headers['Content-Length'] = str(len(data))
+            # If the pagelet did not set its own Content-Type header, use libmagic to guess an appropriate one
+            if 'Content-Type' not in headers:
+                try:
+                    filemagic: magic.FileMagic = magic.detect_from_content(data)
+                    mimetype: str = filemagic.mime_type
+                    charset: str = filemagic.encoding
+                except ValueError:
+                    mimetype = 'application/octet-stream'
+                    charset = 'binary'
+                headers['Content-Type'] = f'{mimetype}; charset={charset}'
             # Send all headers set by the pagelet
             for name, value in headers.items():
                 self.send_header(name, value)
@@ -365,13 +374,16 @@ class HttpHandler(BaseHTTPRequestHandler):
                     data = f.read()
                 # File read successfully, send 'OK' header
                 self.send_response(200)
-                # TODO: Guess the MIME type.  Unfortunately this call solely relies on the file extension, not ideal?
+                # Guess the MIME type and encoding using libmagic
-                mimetype, _ = mimetypes.guess_type(filepath)
+                try:
-                # Fall back to octet-stream type, if unknown
+                    filemagic: magic.FileMagic = magic.detect_from_filename(filepath)
-                if mimetype is None:
+                    mimetype: str = filemagic.mime_type
+                    charset: str = filemagic.encoding
+                except ValueError:
                     mimetype = 'application/octet-stream'
+                    charset = 'binary'
                 # Send content type and length header
-                self.send_header('Content-Type', mimetype)
+                self.send_header('Content-Type', f'{mimetype}; charset={charset}')
                 self.send_header('Content-Length', str(len(data)))
                 self.send_header('Last-Modified', fileage.strftime('%a, %d %b %Y %H:%M:%S GMT'))
                 self.send_header('Cache-Control', 'max-age=1')

matemat/webserver/pagelets/admin.py

@@ -2,6 +2,7 @@
 from typing import Any, Dict, Union
 
 import os
+import magic
 
 from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
 from matemat.db import MatematDatabase
@@ -76,6 +77,10 @@ def handle_change(args: RequestArguments, user: User, db: MatematDatabase, confi
             if 'avatar' not in args:
                 return
             avatar = bytes(args.avatar)
+            filemagic: magic.FileMagic = magic.detect_from_content(avatar)
+            if filemagic.mime_type != 'image/png':
+                # TODO: Optionally convert to png
+                return
             abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/')
             os.makedirs(abspath, exist_ok=True)
             with open(os.path.join(abspath, f'{user.id}.png'), 'wb') as f:
@@ -110,10 +115,15 @@ def handle_admin_change(args: RequestArguments, db: MatematDatabase, config: Dic
             newproduct = db.create_product(name, price_member, price_non_member)
             if 'image' in args:
                 image = bytes(args.image)
-                abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/')
+                filemagic: magic.FileMagic = magic.detect_from_content(image)
-                os.makedirs(abspath, exist_ok=True)
+                if filemagic.mime_type != 'image/png':
-                with open(os.path.join(abspath, f'{newproduct.id}.png'), 'wb') as f:
+                    # TODO: Optionally convert to png
-                    f.write(image)
+                    return
+                if len(image) > 0:
+                    abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/')
+                    os.makedirs(abspath, exist_ok=True)
+                    with open(os.path.join(abspath, f'{newproduct.id}.png'), 'wb') as f:
+                        f.write(image)
 
         elif change == 'restock':
             if 'productid' not in args or 'amount' not in args:

matemat/webserver/pagelets/modproduct.py

@@ -2,6 +2,7 @@
 from typing import Any, Dict, Union
 
 import os
+import magic
 
 from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
 from matemat.db import MatematDatabase
@@ -69,6 +70,10 @@ def handle_change(args: RequestArguments, product: Product, db: MatematDatabase,
             pass
         if 'image' in args:
             image = bytes(args.image)
+            filemagic: magic.FileMagic = magic.detect_from_content(image)
+            if filemagic.mime_type != 'image/png':
+                # TODO: Optionally convert to png
+                return
             if len(image) > 0:
                 abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/')
                 os.makedirs(abspath, exist_ok=True)

matemat/webserver/pagelets/moduser.py

@@ -2,6 +2,7 @@
 from typing import Any, Dict, Union
 
 import os
+import magic
 
 from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
 from matemat.db import MatematDatabase
@@ -74,6 +75,10 @@ def handle_change(args: RequestArguments, user: User, db: MatematDatabase, confi
             db.change_password(user, '', password, verify_password=False)
         if 'avatar' in args:
             avatar = bytes(args.avatar)
+            filemagic: magic.FileMagic = magic.detect_from_content(avatar)
+            if filemagic.mime_type != 'image/png':
+                # TODO: Optionally convert to png
+                return
             if len(avatar) > 0:
                 abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/')
                 os.makedirs(abspath, exist_ok=True)

requirements.txt

@@ -1 +1,2 @@
+file-magic
 jinja2

testing/Dockerfile

@@ -5,7 +5,7 @@ RUN useradd -d /home/matemat -m matemat
 RUN mkdir -p /var/matemat/db && chown matemat:matemat -R /var/matemat/db
 RUN mkdir -p /var/matemat/upload && chown matemat:matemat -R /var/matemat/upload
 RUN apt-get update -qy
-RUN apt-get install -y --no-install-recommends sudo openssh-client git docker.io python3-dev python3-pip python3-coverage python3-setuptools build-essential
+RUN apt-get install -y --no-install-recommends file sudo openssh-client git docker.io python3-dev python3-pip python3-coverage python3-setuptools build-essential
 RUN pip3 install wheel pycodestyle mypy
 
 WORKDIR /home/matemat