Create imagemap, download ALL chaostreffs, variate distance in label placement
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
s3lph 2022-10-12 02:43:47 +02:00
parent 4c7b1b5387
commit a10e679a13
Signed by: s3lph
GPG key ID: 8AC98A811E5BEFF5
4 changed files with 192 additions and 48 deletions

3
.gitignore vendored
View file

@ -1,3 +1,2 @@
cache/
map.png
map.svg
out/

View file

@ -11,6 +11,7 @@ import sys
import argparse
import shutil
import random
import math
from lxml import etree
import pyproj
@ -37,9 +38,25 @@ class CachePaths:
self.chaostreff_info = os.path.join(path, 'chaostreff-info.json')
ERFA_URL = 'https://doku.ccc.de/Spezial:Semantische_Suche/format%3Djson/limit%3D50/link%3Dall/headers%3Dshow/searchlabel%3DJSON/class%3Dsortable-20wikitable-20smwtable/sort%3D/order%3Dasc/offset%3D0/-5B-5BKategorie:Erfa-2DKreise-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity/-3FChaostreff-2DPhysical-2DAddress/-3FChaostreff-2DPhysical-2DHousenumber/-3FChaostreff-2DPhysical-2DPostcode/-3FChaostreff-2DPhysical-2DCity/-3FChaostreff-2DCountry/-3FPublic-2DWeb/-3FChaostreff-2DLongname/mainlabel%3D/prettyprint%3Dtrue/unescape%3Dtrue'
class OutputPaths:
CHAOSTREFF_URL = 'https://doku.ccc.de/Spezial:Semantische_Suche/format%3Djson/limit%3D50/link%3Dall/headers%3Dshow/searchlabel%3DJSON/class%3Dsortable-20wikitable-20smwtable/sort%3D/order%3Dasc/offset%3D0/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity/-3FChaostreff-2DPhysical-2DAddress/-3FChaostreff-2DPhysical-2DHousenumber/-3FChaostreff-2DPhysical-2DPostcode/-3FChaostreff-2DPhysical-2DCity/-3FChaostreff-2DCountry/-3FPublic-2DWeb/-3FChaostreff-2DLongname/mainlabel%3D/prettyprint%3Dtrue/unescape%3Dtrue'
def __init__(self, path: str):
if os.path.exists(path) and not os.path.isdir(path):
raise AttributeError(f'Path exists but is not a directory: {path}')
os.makedirs(path, exist_ok=True)
self.path = path
self.svg_path = os.path.join(path, 'map.svg')
self.png_path = os.path.join(path, 'map.png')
self.html_path = os.path.join(path, 'erfamap.html')
self.imagemap_path = os.path.join(path, 'imagemap.html')
def rel(self, path: str):
return os.path.relpath(path, start=self.path)
ERFA_URL = 'https://doku.ccc.de/index.php?title=Spezial:Semantische_Suche&x=-5B-5BKategorie%3AErfa-2DKreise-5D-5D-20-5B-5BChaostreff-2DActive%3A%3Awahr-5D-5D%2F-3FChaostreff-2DCity%2F-3FChaostreff-2DPhysical-2DAddress%2F-3FChaostreff-2DPhysical-2DHousenumber%2F-3FChaostreff-2DPhysical-2DPostcode%2F-3FChaostreff-2DPhysical-2DCity%2F-3FChaostreff-2DCountry%2F-3FPublic-2DWeb%2F-3FChaostreff-2DLongname%2F-3FChaostreff-2DNickname%2F-3FChaostreff-2DRealname&format=json&limit=200&link=all&headers=show&searchlabel=JSON&class=sortable+wikitable+smwtable&sort=&order=asc&offset=0&mainlabel=&prettyprint=true&unescape=true'
CHAOSTREFF_URL = 'https://doku.ccc.de/index.php?title=Spezial:Semantische_Suche&x=-5B-5BKategorie%3AChaostreffs-5D-5D-20-5B-5BChaostreff-2DActive%3A%3Awahr-5D-5D%2F-3FChaostreff-2DCity%2F-3FChaostreff-2DPhysical-2DAddress%2F-3FChaostreff-2DPhysical-2DHousenumber%2F-3FChaostreff-2DPhysical-2DPostcode%2F-3FChaostreff-2DPhysical-2DCity%2F-3FChaostreff-2DCountry%2F-3FPublic-2DWeb%2F-3FChaostreff-2DLongname%2F-3FChaostreff-2DNickname%2F-3FChaostreff-2DRealname&format=json&limit=200&link=all&headers=show&searchlabel=JSON&class=sortable+wikitable+smwtable&sort=&order=asc&offset=0&mainlabel=&prettyprint=true&unescape=true'
def sparql_query(query):
@ -167,6 +184,7 @@ def address_lookup(name, erfa):
city = erfa['Chaostreff-City'][0]
country = erfa['Chaostreff-Country'][0]
# Try the most accurate address first, try increasingly inaccurate addresses on failure.
formats = [
# Muttenz, Schweiz
f'{city}, {country}'
@ -210,6 +228,13 @@ def fetch_erfas(target, url):
erfas[city]['web'] = erfa['printouts']['Public-Web'][0]
if len(erfa['printouts']['Chaostreff-Longname']) > 0:
erfas[city]['name'] = erfa['printouts']['Chaostreff-Longname'][0]
elif len(erfa['printouts']['Chaostreff-Nickname']) > 0:
erfas[city]['name'] = erfa['printouts']['Chaostreff-Nickname'][0]
elif len(erfa['printouts']['Chaostreff-Realname']) > 0:
erfas[city]['name'] = erfa['printouts']['Chaostreff-Realname'][0]
else:
erfas[city]['name'] = name
with open(target, 'w') as f:
json.dump(erfas, f)
@ -257,6 +282,8 @@ class BoundingBox:
self.meta = meta
self.base_weight = base_weight
self.finished = False
self._weight = 0
self._optimal = True
def __contains__(self, other):
if isinstance(other, BoundingBox):
@ -323,14 +350,15 @@ class BoundingBox:
def distance2(self, other):
c1 = self.center
c2 = other.center
return (c1[0] - c2[0])**2 + (c1[1] - c2[1])**2
return math.sqrt((c1[0] - c2[0])**2 + (c1[1] - c2[1])**2)
def with_margin(self, margin):
return BoundingBox(self.left - margin, self.top - margin,
self.right + margin, self.bottom + margin,
width=None, height=None, meta=self.meta, base_weight=self.base_weight)
def chebyshev_distance(self, other):
def chebyshev_distance(self, other):
# https://en.wikipedia.org/wiki/Chebyshev_distance
if other in self:
return 0
if isinstance(other, BoundingBox):
@ -358,22 +386,27 @@ class BoundingBox:
def compute_weight(self, other, erfas, chaostreffs, pdist, ns):
w = 0
self._optimal = True
swm = self.with_margin(pdist)
swe = self.with_margin(ns.dotsize_erfa)
swc = self.with_margin(ns.dotsize_treff)
swme = swm.with_margin(ns.dotsize_erfa)
swmc = swm.with_margin(ns.dotsize_treff)
# I hope these weights are somewhat reasonably balanced...
# Basically the weights correspond to geometrical distances,
# except for an actual collision, which gets a huge extra weight.
for o in other:
if o.meta['city'] == self.meta['city']:
continue
if o in self:
if o.finished:
w += 1000
self._optimal = False
else:
w += 50
self._optimal = False
else:
w += max(pdist*2 - swc.chebyshev_distance(o), 0)
w += max(pdist*2 - swm.chebyshev_distance(o), 0)
continue
if o in swm:
if o.finished:
@ -385,6 +418,7 @@ class BoundingBox:
continue
if location in swe:
w += 1000
self._optimal = False
else:
w += max(pdist*2 - swe.chebyshev_distance(location), 0)
for city, location in chaostreffs.items():
@ -392,6 +426,7 @@ class BoundingBox:
continue
if location in swc:
w += 1000
self._optimal = False
else:
w += max(pdist*2 - swc.chebyshev_distance(location), 0)
self._weight = w
@ -402,19 +437,23 @@ class BoundingBox:
@property
def is_optimal(self):
return self._weight == 0
# Candidate is considered optimal if it doesn't collide with anything else
return self._optimal
def __str__(self):
return f'(({int(self.left)}, {int(self.top)}, {int(self.right)}, {int(self.bottom)}), weight={self.weight})'
def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
# Load the font and measure its various different heights
font = ImageFont.truetype(ns.font, ns.font_size)
pil = ImageDraw(Image.new('P', (1, 1)))
xheight = -pil.textbbox((0,0), 'x', font=font, anchor='ls')[1]
capheight = -pil.textbbox((0,0), 'A', font=font, anchor='ls')[1]
voffset = - capheight / 2
voffset = -capheight / 2
# Generate a discrete set of text placement candidates around each erfa dot
candidates = {}
for city, location in erfas.items():
text = city
@ -427,29 +466,35 @@ def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
meta = {'city': city, 'text': text, 'baseline': capheight}
textbox = pil.textbbox((0, 0), text, font=font, anchor='ls') # left, baseline at 0,0
mw, mh = textbox[2] - textbox[0], textbox[3] - textbox[1]
rbox = BoundingBox(erfax + ns.font_distance, erfay + voffset, width=mw, height=mh, meta=meta)
lbox = BoundingBox(erfax - ns.font_distance - mw, erfay + voffset, width=mw, height=mh, meta=meta)
if erfax > 0.8 * size[0]:
rbox.base_weight += 0.001
else:
lbox.base_weight += 0.001
candidates[city] = [lbox, rbox]
for i in range(-2, 3):
if i == 0:
continue
candidates[city].append(lbox + (0, ns.dotsize_erfa*i))
candidates[city].append(rbox + (0, ns.dotsize_erfa*i))
candidates[city].extend([
BoundingBox(erfax - mw/2, erfay - ns.font_distance - mh, width=mw, height=mh, meta=meta, base_weight=0.001),
BoundingBox(erfax - mw/2, erfay + ns.font_distance, width=mw, height=mh, meta=meta, base_weight=0.001),
BoundingBox(erfax - ns.dotsize_erfa, erfay - ns.font_distance - mh, width=mw, height=mh, meta=meta, base_weight=0.002),
BoundingBox(erfax - ns.dotsize_erfa, erfay + ns.font_distance, width=mw, height=mh, meta=meta, base_weight=0.003),
BoundingBox(erfax + ns.dotsize_erfa - mw, erfay - ns.font_distance - mh, width=mw, height=mh, meta=meta, base_weight=0.001),
BoundingBox(erfax + ns.dotsize_erfa - mw, erfay + ns.font_distance, width=mw, height=mh, meta=meta, base_weight=0.001),
])
rbox.base_weight -= 0.003
lbox.base_weight -= 0.003
candidates[city] = []
# Iterate over the dot-to-text distance range in discrete steps
it = max(0, ns.font_step_distance)
for j in range(it+1):
dist = ns.font_min_distance + (ns.font_max_distance - ns.font_min_distance) * j / it
bw = dist
# Generate 5 candidates each left and right of the dot, with varying vertical offset
for i in range(-1, 2):
if i == 0:
bw -= 0.003
bwl, bwr = bw, bw
if erfax > 0.8 * size[0]:
bwr = bw + 0.001
else:
bwl = bw + 0.001
candidates[city].append(BoundingBox(erfax - dist - mw, erfay + voffset + ns.dotsize_erfa*i*2, width=mw, height=mh, meta=meta, base_weight=bwl))
candidates[city].append(BoundingBox(erfax + dist, erfay + voffset + ns.dotsize_erfa*i*2, width=mw, height=mh, meta=meta, base_weight=bwr))
# Generate 3 candidates each above and beneath the dot, aligned left, centered and right
candidates[city].extend([
BoundingBox(erfax - mw/2, erfay - dist - mh, width=mw, height=mh, meta=meta, base_weight=bw + 0.001),
BoundingBox(erfax - mw/2, erfay + dist, width=mw, height=mh, meta=meta, base_weight=bw + 0.001),
BoundingBox(erfax - ns.dotsize_erfa, erfay - dist - mh, width=mw, height=mh, meta=meta, base_weight=bw + 0.002),
BoundingBox(erfax - ns.dotsize_erfa, erfay + dist, width=mw, height=mh, meta=meta, base_weight=bw + 0.003),
BoundingBox(erfax + ns.dotsize_erfa - mw, erfay - dist - mh, width=mw, height=mh, meta=meta, base_weight=bw + 0.001),
BoundingBox(erfax + ns.dotsize_erfa - mw, erfay + dist, width=mw, height=mh, meta=meta, base_weight=bw + 0.001),
])
# If debugging is enabled, render one rectangle around each label's bounding box, and one rectangle around each label's median box
if ns.debug:
for c in candidates[city]:
di = etree.Element('rect', x=str(c.left), y=str(c.top + capheight - xheight), width=str(c.width), height=str(xheight))
@ -463,6 +508,7 @@ def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
unfinished = {c for c in erfas.keys()}
finished = {}
# Greedily choose a candidate for each label
with tqdm.tqdm(total=len(erfas)) as progress:
while len(unfinished) > 0:
progress.update(len(erfas) - len(unfinished))
@ -473,7 +519,7 @@ def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
all_boxes.update(candidates[city])
unfinished_boxes.update(candidates[city])
for box in all_boxes:
box.compute_weight(all_boxes, erfas, chaostreffs, pdist=max(ns.font_distance, xheight), ns=ns)
box.compute_weight(all_boxes, erfas, chaostreffs, pdist=max(ns.font_min_distance, xheight), ns=ns)
# Get candidate with the least number of optimal solutions > 0
optcity = min(unfinished, key=lambda city: len([1 for box in candidates[city] if box.is_optimal]) or float('inf'))
@ -499,7 +545,7 @@ def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
for city in order:
all_boxes = set(finished.values())
for box in candidates[city]:
box.compute_weight(all_boxes, erfas, chaostreffs, pdist=max(ns.font_distance, xheight), ns=ns)
box.compute_weight(all_boxes, erfas, chaostreffs, pdist=max(ns.font_min_distance, xheight), ns=ns)
minbox = min(candidates[city], key=lambda box: box.weight)
if minbox is not finished[city]:
changed = True
@ -512,6 +558,48 @@ def optimize_text_layout(ns, erfas, chaostreffs, size, svg):
return finished
def create_imagemap(ns, size, parent,
erfas, erfa_urls, erfa_names, texts,
chaostreffs, chaostreff_urls, chaostreff_names):
img = etree.Element('img',
src=ns.output_directory.rel(ns.output_directory.png_path),
usemap='#erfamap',
width=str(size[0]), height=str(size[1]))
imgmap = etree.Element('map', name='erfamap')
for city, location in erfas.items():
if city not in erfa_urls:
continue
box = texts[city]
area = etree.Element('area',
shape='circle',
coords=f'{location[0]},{location[1]},{ns.dotsize_erfa}',
href=erfa_urls[city])
area2 = etree.Element('area',
shape='rect',
coords=f'{box.left},{box.top},{box.right},{box.bottom}',
href=erfa_urls[city])
if city in erfa_names:
area.set('title', erfa_names[city])
area2.set('title', erfa_names[city])
imgmap.append(area)
imgmap.append(area2)
for city, location in chaostreffs.items():
if city not in chaostreff_urls:
continue
area = etree.Element('area',
shape='circle',
coords=f'{location[0]},{location[1]},{ns.dotsize_treff}',
href=chaostreff_urls[city])
if city in chaostreff_names:
area.set('title', chaostreff_names[city])
imgmap.append(area)
parent.append(img)
parent.append(imgmap)
def create_svg(ns, bbox):
print('Creating SVG image')
@ -527,7 +615,8 @@ def create_svg(ns, bbox):
]
origin = trans_bounding_box[0]
svg_box = (trans_bounding_box[1][0] - origin[0], origin[1] - trans_bounding_box[1][1])
# Load state border lines from cached JSON files
shapes_states = []
files = os.listdir(ns.cache_directory.shapes_states)
for f in files:
@ -548,6 +637,7 @@ def create_svg(ns, bbox):
ts.append((xt*scalex - origin[0], origin[1] - yt*scaley))
shapes_states.append((name, ts))
# Load country border lines from cached JSON files
shapes_countries = []
files = os.listdir(ns.cache_directory.shapes_filtered)
for f in files:
@ -568,6 +658,7 @@ def create_svg(ns, bbox):
ts.append((xt*scalex - origin[0], origin[1] - yt*scaley))
shapes_countries.append((name, ts))
# Load Erfa infos from cached JSON files
erfas = {}
erfa_urls = {}
erfa_names = {}
@ -586,6 +677,7 @@ def create_svg(ns, bbox):
if name is not None:
erfa_names[city] = name
# Load Chaostreff infos from cached JSON files
chaostreffs = {}
chaostreff_urls = {}
chaostreff_names = {}
@ -617,20 +709,22 @@ def create_svg(ns, bbox):
rectbox[2] = max(lon, rectbox[2])
rectbox[3] = max(lat, rectbox[3])
print('Copying stylesheet and font')
dst = os.path.join(ns.output_directory.path, ns.stylesheet)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(ns.stylesheet, dst)
dst = os.path.join(ns.output_directory.path, ns.font)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(ns.font, dst)
svg = etree.Element('svg',
xmlns='http://www.w3.org/2000/svg',
viewBox=f'0 0 {svg_box[0]} {svg_box[1]}',
width=str(svg_box[0]), height=str(svg_box[1]))
print('Layouting labels')
texts = optimize_text_layout(ns, erfas, chaostreffs, (int(svg_box[0]), int(svg_box[1])), svg)
defs = etree.Element('defs')
style = etree.Element('style', type='text/css')
style.text = f'@import url({ns.stylesheet})'
defs.append(style)
svg.append(defs)
svg.append(style)
bg = etree.Element('rect',
id='background',
@ -641,13 +735,15 @@ def create_svg(ns, bbox):
bg.set('class', 'background')
svg.append(bg)
# Render country borders
for name, shape in shapes_countries:
points = ' '.join([f'{lon},{lat}' for lon, lat in shape])
poly = etree.Element('polygon', points=points)
poly.set('class', 'country')
poly.set('data-country', name)
svg.append(poly)
# Render state borders
# Render shortest shapes last s.t. Berlin, Hamburg and Bremen are rendered on top of their surrounding states
for name, shape in sorted(shapes_states, key=lambda x: -sum(len(s) for s in x[1])):
points = ' '.join([f'{lon},{lat}' for lon, lat in shape])
@ -656,6 +752,11 @@ def create_svg(ns, bbox):
poly.set('data-state', name)
svg.append(poly)
# This can take some time, especially if lots of candidates are generated
print('Layouting labels')
texts = optimize_text_layout(ns, erfas, chaostreffs, (int(svg_box[0]), int(svg_box[1])), svg)
# Render Erfa dots and their labels
for city, location in erfas.items():
box = texts[city]
if city in erfa_urls:
@ -678,6 +779,7 @@ def create_svg(ns, bbox):
group.append(text)
svg.append(group)
# Render Chaostreff dots
for city, location in chaostreffs.items():
circle = etree.Element('circle', cx=str(location[0]), cy=str(location[1]), r=str(ns.dotsize_treff))
circle.set('class', 'chaostreff')
@ -687,25 +789,59 @@ def create_svg(ns, bbox):
title.text = chaostreff_names[city]
circle.append(title)
if city in chaostreff_urls:
a = etree.Element('a', href=chaostreff_urls[city], target='_blank', title='foox')
a = etree.Element('a', href=chaostreff_urls[city], target='_blank')
a.append(circle)
svg.append(a)
else:
svg.append(circle)
print('Done, writing SVG')
with open('map.svg', 'wb') as mapfile:
# Generate SVG, PNG and HTML output files
print('Writing SVG')
with open(ns.output_directory.svg_path, 'wb') as mapfile:
root = etree.ElementTree(svg)
root.write(mapfile)
print('Done, writing PNG')
cairosvg.svg2png(url='map.svg', write_to='map.png')
print('Done')
print('Writing PNG')
cairosvg.svg2png(url=ns.output_directory.svg_path, write_to=ns.output_directory.png_path)
print('Writing HTML SVG page')
html = etree.Element('html')
head = etree.Element('head')
link = etree.Element('link', rel='stylesheet', href=ns.stylesheet)
head.append(link)
html.append(head)
body = etree.Element('body')
obj = etree.Element('object',
data=ns.output_directory.rel(ns.output_directory.svg_path),
width=str(svg_box[0]), height=str(svg_box[1]))
create_imagemap(ns, svg_box, obj,
erfas, erfa_urls, erfa_names, texts,
chaostreffs, chaostreff_urls, chaostreff_names)
body.append(obj)
html.append(body)
with open(ns.output_directory.html_path, 'wb') as f:
f.write(b'<!DOCTYLE html>\n')
etree.ElementTree(html).write(f)
print('Writing HTML Image Map')
html = etree.Element('html')
body = etree.Element('body')
html.append(body)
create_imagemap(ns, svg_box, body,
erfas, erfa_urls, erfa_names, texts,
chaostreffs, chaostreff_urls, chaostreff_names)
with open(ns.output_directory.imagemap_path, 'wb') as f:
f.write(b'<!DOCTYLE html>\n')
etree.ElementTree(html).write(f)
def main():
ap = argparse.ArgumentParser(sys.argv[0])
ap.add_argument('--cache-directory', type=CachePaths, default='cache', help='Path to the cache directory')
ap.add_argument('--output-directory', type=OutputPaths, default='out', help='Path to the output directory')
ap.add_argument('--update-borders', action='store_true', default=False, help='Update country borders from Wikidata')
ap.add_argument('--update-erfalist', action='store_true', default=False, help='Update Erfa/Chaostreff list from doku.ccc.de')
ap.add_argument('--bbox', type=float, nargs=4, metavar=('LON', 'LAT', 'LON', 'LAT') ,default=None, help='Override map bounding box')
@ -713,7 +849,9 @@ def main():
ap.add_argument('--stylesheet', type=str, default='style/erfamap.css', help='Stylesheet for the generated SVG')
ap.add_argument('--font', type=str, default='style/concertone-regular.ttf', help='Name of the font used in the stylesheet.')
ap.add_argument('--font-size', type=int, default=45, help='Size of the font used in the stylesheet.')
ap.add_argument('--font-distance', type=int, default=18, help='Distance of labels from their dots center')
ap.add_argument('--font-min-distance', type=int, default=18, help='Minimal distance of labels from their dots center')
ap.add_argument('--font-max-distance', type=int, default=40, help='Maximal distance of labels from their dots center')
ap.add_argument('--font-step-distance', type=int, default=3, help='Distance steps of labels from their dots center')
ap.add_argument('--dotsize-erfa', type=float, default=13, help='Radius of Erfa dots')
ap.add_argument('--dotsize-treff', type=float, default=8, help='Radius of Chaostreff dots')
ap.add_argument('--rename', type=str, action='append', nargs=2, metavar=('FROM', 'TO'), default=[], help='Rename a city with an overly long name (e.g. "Rothenburg ob der Tauber" to "Rothenburg")')

Binary file not shown.

Before

Width:  |  Height:  |  Size: 421 KiB

After

Width:  |  Height:  |  Size: 428 KiB

View file

@ -1,4 +1,11 @@
* {
margin: 0;
padding: 0;
}
object, img {
max-width: 100%;
}
rect.background {