Merge branch '18-libmagic-mime-type-detection' into staging-unstable

This commit is contained in:
s3lph 2018-07-20 10:59:39 +02:00
commit b6b07fdac5
8 changed files with 47 additions and 13 deletions

View file

@ -1,5 +1,5 @@
--- ---
image: s3lph/matemat-ci:20180711-02 image: s3lph/matemat-ci:20180720-01
stages: stages:
- test - test

View file

@ -18,6 +18,7 @@ This project intends to provide a well-tested and maintainable alternative to
- Python 3 (>=3.6) - Python 3 (>=3.6)
- Python dependencies: - Python dependencies:
- file-magic
- jinja2 - jinja2
## Usage ## Usage

View file

@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, Tuple, Type, Union
import logging import logging
import os import os
import socket import socket
import mimetypes import magic
from socketserver import TCPServer from socketserver import TCPServer
from http.server import HTTPServer, BaseHTTPRequestHandler from http.server import HTTPServer, BaseHTTPRequestHandler
from http.cookies import SimpleCookie from http.cookies import SimpleCookie
@ -308,7 +308,6 @@ class HttpHandler(BaseHTTPRequestHandler):
if path in _PAGELET_PATHS: if path in _PAGELET_PATHS:
# Prepare some headers. Those can still be overwritten by the pagelet # Prepare some headers. Those can still be overwritten by the pagelet
headers: Dict[str, str] = { headers: Dict[str, str] = {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'no-cache' 'Cache-Control': 'no-cache'
} }
# Call the pagelet function # Call the pagelet function
@ -328,6 +327,16 @@ class HttpHandler(BaseHTTPRequestHandler):
f'matemat_session_id={session_id}; expires={expires}') f'matemat_session_id={session_id}; expires={expires}')
# Compute the body length and add the appropriate header # Compute the body length and add the appropriate header
headers['Content-Length'] = str(len(data)) headers['Content-Length'] = str(len(data))
# If the pagelet did not set its own Content-Type header, use libmagic to guess an appropriate one
if 'Content-Type' not in headers:
try:
filemagic: magic.FileMagic = magic.detect_from_content(data)
mimetype: str = filemagic.mime_type
charset: str = filemagic.encoding
except ValueError:
mimetype = 'application/octet-stream'
charset = 'binary'
headers['Content-Type'] = f'{mimetype}; charset={charset}'
# Send all headers set by the pagelet # Send all headers set by the pagelet
for name, value in headers.items(): for name, value in headers.items():
self.send_header(name, value) self.send_header(name, value)
@ -365,13 +374,16 @@ class HttpHandler(BaseHTTPRequestHandler):
data = f.read() data = f.read()
# File read successfully, send 'OK' header # File read successfully, send 'OK' header
self.send_response(200) self.send_response(200)
# TODO: Guess the MIME type. Unfortunately this call solely relies on the file extension, not ideal? # Guess the MIME type and encoding using libmagic
mimetype, _ = mimetypes.guess_type(filepath) try:
# Fall back to octet-stream type, if unknown filemagic: magic.FileMagic = magic.detect_from_filename(filepath)
if mimetype is None: mimetype: str = filemagic.mime_type
charset: str = filemagic.encoding
except ValueError:
mimetype = 'application/octet-stream' mimetype = 'application/octet-stream'
charset = 'binary'
# Send content type and length header # Send content type and length header
self.send_header('Content-Type', mimetype) self.send_header('Content-Type', f'{mimetype}; charset={charset}')
self.send_header('Content-Length', str(len(data))) self.send_header('Content-Length', str(len(data)))
self.send_header('Last-Modified', fileage.strftime('%a, %d %b %Y %H:%M:%S GMT')) self.send_header('Last-Modified', fileage.strftime('%a, %d %b %Y %H:%M:%S GMT'))
self.send_header('Cache-Control', 'max-age=1') self.send_header('Cache-Control', 'max-age=1')

View file

@ -2,6 +2,7 @@
from typing import Any, Dict, Union from typing import Any, Dict, Union
import os import os
import magic
from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
from matemat.db import MatematDatabase from matemat.db import MatematDatabase
@ -76,6 +77,10 @@ def handle_change(args: RequestArguments, user: User, db: MatematDatabase, confi
if 'avatar' not in args: if 'avatar' not in args:
return return
avatar = bytes(args.avatar) avatar = bytes(args.avatar)
filemagic: magic.FileMagic = magic.detect_from_content(avatar)
if filemagic.mime_type != 'image/png':
# TODO: Optionally convert to png
return
abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/') abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/')
os.makedirs(abspath, exist_ok=True) os.makedirs(abspath, exist_ok=True)
with open(os.path.join(abspath, f'{user.id}.png'), 'wb') as f: with open(os.path.join(abspath, f'{user.id}.png'), 'wb') as f:
@ -110,10 +115,15 @@ def handle_admin_change(args: RequestArguments, db: MatematDatabase, config: Dic
newproduct = db.create_product(name, price_member, price_non_member) newproduct = db.create_product(name, price_member, price_non_member)
if 'image' in args: if 'image' in args:
image = bytes(args.image) image = bytes(args.image)
abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/') filemagic: magic.FileMagic = magic.detect_from_content(image)
os.makedirs(abspath, exist_ok=True) if filemagic.mime_type != 'image/png':
with open(os.path.join(abspath, f'{newproduct.id}.png'), 'wb') as f: # TODO: Optionally convert to png
f.write(image) return
if len(image) > 0:
abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/')
os.makedirs(abspath, exist_ok=True)
with open(os.path.join(abspath, f'{newproduct.id}.png'), 'wb') as f:
f.write(image)
elif change == 'restock': elif change == 'restock':
if 'productid' not in args or 'amount' not in args: if 'productid' not in args or 'amount' not in args:

View file

@ -2,6 +2,7 @@
from typing import Any, Dict, Union from typing import Any, Dict, Union
import os import os
import magic
from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
from matemat.db import MatematDatabase from matemat.db import MatematDatabase
@ -69,6 +70,10 @@ def handle_change(args: RequestArguments, product: Product, db: MatematDatabase,
pass pass
if 'image' in args: if 'image' in args:
image = bytes(args.image) image = bytes(args.image)
filemagic: magic.FileMagic = magic.detect_from_content(image)
if filemagic.mime_type != 'image/png':
# TODO: Optionally convert to png
return
if len(image) > 0: if len(image) > 0:
abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/') abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/products/')
os.makedirs(abspath, exist_ok=True) os.makedirs(abspath, exist_ok=True)

View file

@ -2,6 +2,7 @@
from typing import Any, Dict, Union from typing import Any, Dict, Union
import os import os
import magic
from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse from matemat.webserver import pagelet, RequestArguments, PageletResponse, RedirectResponse, TemplateResponse
from matemat.db import MatematDatabase from matemat.db import MatematDatabase
@ -74,6 +75,10 @@ def handle_change(args: RequestArguments, user: User, db: MatematDatabase, confi
db.change_password(user, '', password, verify_password=False) db.change_password(user, '', password, verify_password=False)
if 'avatar' in args: if 'avatar' in args:
avatar = bytes(args.avatar) avatar = bytes(args.avatar)
filemagic: magic.FileMagic = magic.detect_from_content(avatar)
if filemagic.mime_type != 'image/png':
# TODO: Optionally convert to png
return
if len(avatar) > 0: if len(avatar) > 0:
abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/') abspath: str = os.path.join(os.path.abspath(config['UploadDir']), 'thumbnails/users/')
os.makedirs(abspath, exist_ok=True) os.makedirs(abspath, exist_ok=True)

View file

@ -1 +1,2 @@
file-magic
jinja2 jinja2

View file

@ -5,7 +5,7 @@ RUN useradd -d /home/matemat -m matemat
RUN mkdir -p /var/matemat/db && chown matemat:matemat -R /var/matemat/db RUN mkdir -p /var/matemat/db && chown matemat:matemat -R /var/matemat/db
RUN mkdir -p /var/matemat/upload && chown matemat:matemat -R /var/matemat/upload RUN mkdir -p /var/matemat/upload && chown matemat:matemat -R /var/matemat/upload
RUN apt-get update -qy RUN apt-get update -qy
RUN apt-get install -y --no-install-recommends sudo openssh-client git docker.io python3-dev python3-pip python3-coverage python3-setuptools build-essential RUN apt-get install -y --no-install-recommends file sudo openssh-client git docker.io python3-dev python3-pip python3-coverage python3-setuptools build-essential
RUN pip3 install wheel pycodestyle mypy RUN pip3 install wheel pycodestyle mypy
WORKDIR /home/matemat WORKDIR /home/matemat