WebHost: server render remaining markdown using mistune (#5276)

---------

Co-authored-by: Aaron Wagener <mmmcheese158@gmail.com>
Co-authored-by: qwint <qwint.42@gmail.com>
Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com>
This commit is contained in:
Fabian Dill
2025-08-02 21:12:58 +02:00
committed by GitHub
parent 277f21db7a
commit 72ae076ce7
16 changed files with 157 additions and 335 deletions

View File

@@ -1926,7 +1926,7 @@ class Tutorial(NamedTuple):
description: str
language: str
file_name: str
link: str
link: str # unused
authors: List[str]

View File

@@ -54,16 +54,15 @@ def get_app() -> "Flask":
return app
def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]:
import json
def copy_tutorials_files_to_static() -> None:
import shutil
import zipfile
from werkzeug.utils import secure_filename
zfile: zipfile.ZipInfo
from worlds.AutoWorld import AutoWorldRegister
worlds = {}
data = []
for game, world in AutoWorldRegister.world_types.items():
if hasattr(world.web, 'tutorials') and (not world.hidden or game == 'Archipelago'):
worlds[game] = world
@@ -72,7 +71,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
shutil.rmtree(base_target_path, ignore_errors=True)
for game, world in worlds.items():
# copy files from world's docs folder to the generated folder
target_path = os.path.join(base_target_path, get_file_safe_name(game))
target_path = os.path.join(base_target_path, secure_filename(game))
os.makedirs(target_path, exist_ok=True)
if world.zip_path:
@@ -85,45 +84,14 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
for zfile in zf.infolist():
if not zfile.is_dir() and "/docs/" in zfile.filename:
zfile.filename = os.path.basename(zfile.filename)
zf.extract(zfile, target_path)
with open(os.path.join(target_path, secure_filename(zfile.filename)), "wb") as f:
f.write(zf.read(zfile))
else:
source_path = Utils.local_path(os.path.dirname(world.__file__), "docs")
files = os.listdir(source_path)
for file in files:
shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file))
# build a json tutorial dict per game
game_data = {'gameTitle': game, 'tutorials': []}
for tutorial in world.web.tutorials:
# build dict for the json file
current_tutorial = {
'name': tutorial.tutorial_name,
'description': tutorial.description,
'files': [{
'language': tutorial.language,
'filename': game + '/' + tutorial.file_name,
'link': f'{game}/{tutorial.link}',
'authors': tutorial.authors
}]
}
# check if the name of the current guide exists already
for guide in game_data['tutorials']:
if guide and tutorial.tutorial_name == guide['name']:
guide['files'].append(current_tutorial['files'][0])
break
else:
game_data['tutorials'].append(current_tutorial)
data.append(game_data)
with open(Utils.local_path("WebHostLib", "static", "generated", "tutorials.json"), 'w', encoding='utf-8-sig') as json_target:
generic_data = {}
for games in data:
if 'Archipelago' in games['gameTitle']:
generic_data = data.pop(data.index(games))
sorted_data = [generic_data] + Utils.title_sorted(data, key=lambda entry: entry["gameTitle"])
json.dump(sorted_data, json_target, indent=2, ensure_ascii=False)
return sorted_data
shutil.copyfile(Utils.local_path(source_path, file),
Utils.local_path(target_path, secure_filename(file)))
if __name__ == "__main__":
@@ -142,7 +110,7 @@ if __name__ == "__main__":
logging.warning("Could not update LttP sprites.")
app = get_app()
create_options_files()
create_ordered_tutorials_file()
copy_tutorials_files_to_static()
if app.config["SELFLAUNCH"]:
autohost(app.config)
if app.config["SELFGEN"]:

View File

@@ -7,17 +7,69 @@ from flask import request, redirect, url_for, render_template, Response, session
from pony.orm import count, commit, db_session
from werkzeug.utils import secure_filename
from worlds.AutoWorld import AutoWorldRegister
from worlds.AutoWorld import AutoWorldRegister, World
from . import app, cache
from .models import Seed, Room, Command, UUID, uuid4
from Utils import title_sorted
def get_world_theme(game_name: str):
def get_world_theme(game_name: str) -> str:
if game_name in AutoWorldRegister.world_types:
return AutoWorldRegister.world_types[game_name].web.theme
return 'grass'
def get_visible_worlds() -> dict[str, type(World)]:
worlds = {}
for game, world in AutoWorldRegister.world_types.items():
if not world.hidden:
worlds[game] = world
return worlds
def render_markdown(path: str) -> str:
import mistune
from collections import Counter
markdown = mistune.create_markdown(
escape=False,
plugins=[
"strikethrough",
"footnotes",
"table",
"speedup",
],
)
heading_id_count: Counter[str] = Counter()
def heading_id(text: str) -> str:
nonlocal heading_id_count
import re # there is no good way to do this without regex
s = re.sub(r"[^\w\- ]", "", text.lower()).replace(" ", "-").strip("-")
n = heading_id_count[s]
heading_id_count[s] += 1
if n > 0:
s += f"-{n}"
return s
def id_hook(_: mistune.Markdown, state: mistune.BlockState) -> None:
for tok in state.tokens:
if tok["type"] == "heading" and tok["attrs"]["level"] < 4:
text = tok["text"]
assert isinstance(text, str)
unique_id = heading_id(text)
tok["attrs"]["id"] = unique_id
tok["text"] = f"<a href=\"#{unique_id}\">{text}</a>" # make header link to itself
markdown.before_render_hooks.append(id_hook)
with open(path, encoding="utf-8-sig") as f:
document = f.read()
return markdown(document)
@app.errorhandler(404)
@app.errorhandler(jinja2.exceptions.TemplateNotFound)
def page_not_found(err):
@@ -31,83 +83,88 @@ def start_playing():
return render_template(f"startPlaying.html")
# Game Info Pages
@app.route('/games/<string:game>/info/<string:lang>')
@cache.cached()
def game_info(game, lang):
try:
world = AutoWorldRegister.world_types[game]
if lang not in world.web.game_info_languages:
raise KeyError("Sorry, this game's info page is not available in that language yet.")
except KeyError:
return abort(404)
return render_template('gameInfo.html', game=game, lang=lang, theme=get_world_theme(game))
"""Game Info Pages"""
theme = get_world_theme(game)
secure_game_name = secure_filename(game)
lang = secure_filename(lang)
document = render_markdown(os.path.join(
app.static_folder, "generated", "docs",
secure_game_name, f"{lang}_{secure_game_name}.md"
))
return render_template(
"markdown_document.html",
title=f"{game} Guide",
html_from_markdown=document,
theme=theme,
)
# List of supported games
@app.route('/games')
@cache.cached()
def games():
worlds = {}
for game, world in AutoWorldRegister.world_types.items():
if not world.hidden:
worlds[game] = world
return render_template("supportedGames.html", worlds=worlds)
"""List of supported games"""
return render_template("supportedGames.html", worlds=get_visible_worlds())
@app.route('/tutorial/<string:game>/<string:file>/<string:lang>')
@app.route('/tutorial/<string:game>/<string:file>')
@cache.cached()
def tutorial(game, file, lang):
try:
world = AutoWorldRegister.world_types[game]
if lang not in [tut.link.split("/")[1] for tut in world.web.tutorials]:
raise KeyError("Sorry, the tutorial is not available in that language yet.")
except KeyError:
return abort(404)
return render_template("tutorial.html", game=game, file=file, lang=lang, theme=get_world_theme(game))
def tutorial(game: str, file: str):
theme = get_world_theme(game)
secure_game_name = secure_filename(game)
file = secure_filename(file)
document = render_markdown(os.path.join(
app.static_folder, "generated", "docs",
secure_game_name, file+".md"
))
return render_template(
"markdown_document.html",
title=f"{game} Guide",
html_from_markdown=document,
theme=theme,
)
@app.route('/tutorial/')
@cache.cached()
def tutorial_landing():
return render_template("tutorialLanding.html")
tutorials = {}
worlds = AutoWorldRegister.world_types
for world_name, world_type in worlds.items():
current_world = tutorials[world_name] = {}
for tutorial in world_type.web.tutorials:
current_tutorial = current_world.setdefault(tutorial.tutorial_name, {
"description": tutorial.description, "files": {}})
current_tutorial["files"][secure_filename(tutorial.file_name).rsplit(".", 1)[0]] = {
"authors": tutorial.authors,
"language": tutorial.language
}
tutorials = {world_name: tutorials for world_name, tutorials in title_sorted(
tutorials.items(), key=lambda element: "\x00" if element[0] == "Archipelago" else worlds[element[0]].game)}
return render_template("tutorialLanding.html", worlds=worlds, tutorials=tutorials)
@app.route('/faq/<string:lang>/')
@cache.cached()
def faq(lang: str):
import markdown
with open(os.path.join(app.static_folder, "assets", "faq", secure_filename(lang)+".md")) as f:
document = f.read()
document = render_markdown(os.path.join(app.static_folder, "assets", "faq", secure_filename(lang)+".md"))
return render_template(
"markdown_document.html",
title="Frequently Asked Questions",
html_from_markdown=markdown.markdown(
document,
extensions=["toc", "mdx_breakless_lists"],
extension_configs={
"toc": {"anchorlink": True}
}
),
html_from_markdown=document,
)
@app.route('/glossary/<string:lang>/')
@cache.cached()
def glossary(lang: str):
import markdown
with open(os.path.join(app.static_folder, "assets", "glossary", secure_filename(lang)+".md")) as f:
document = f.read()
document = render_markdown(os.path.join(app.static_folder, "assets", "glossary", secure_filename(lang)+".md"))
return render_template(
"markdown_document.html",
title="Glossary",
html_from_markdown=markdown.markdown(
document,
extensions=["toc", "mdx_breakless_lists"],
extension_configs={
"toc": {"anchorlink": True}
}
),
html_from_markdown=document,
)

View File

@@ -7,6 +7,5 @@ Flask-Compress>=1.17
Flask-Limiter>=3.12
bokeh>=3.6.3
markupsafe>=3.0.2
Markdown>=3.7
mdx-breakless-lists>=1.0.1
setproctitle>=1.3.5
mistune>=3.1.3

View File

@@ -1,45 +0,0 @@
window.addEventListener('load', () => {
const gameInfo = document.getElementById('game-info');
new Promise((resolve, reject) => {
const ajax = new XMLHttpRequest();
ajax.onreadystatechange = () => {
if (ajax.readyState !== 4) { return; }
if (ajax.status === 404) {
reject("Sorry, this game's info page is not available in that language yet.");
return;
}
if (ajax.status !== 200) {
reject("Something went wrong while loading the info page.");
return;
}
resolve(ajax.responseText);
};
ajax.open('GET', `${window.location.origin}/static/generated/docs/${gameInfo.getAttribute('data-game')}/` +
`${gameInfo.getAttribute('data-lang')}_${gameInfo.getAttribute('data-game')}.md`, true);
ajax.send();
}).then((results) => {
// Populate page with HTML generated from markdown
showdown.setOption('tables', true);
showdown.setOption('strikethrough', true);
showdown.setOption('literalMidWordUnderscores', true);
gameInfo.innerHTML += (new showdown.Converter()).makeHtml(results);
// Reset the id of all header divs to something nicer
for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
header.setAttribute('id', headerId);
header.addEventListener('click', () => {
window.location.hash = `#${headerId}`;
header.scrollIntoView();
});
}
// Manually scroll the user to the appropriate header if anchor navigation is used
document.fonts.ready.finally(() => {
if (window.location.hash) {
const scrollTarget = document.getElementById(window.location.hash.substring(1));
scrollTarget?.scrollIntoView();
}
});
});
});

View File

@@ -1,52 +0,0 @@
window.addEventListener('load', () => {
const tutorialWrapper = document.getElementById('tutorial-wrapper');
new Promise((resolve, reject) => {
const ajax = new XMLHttpRequest();
ajax.onreadystatechange = () => {
if (ajax.readyState !== 4) { return; }
if (ajax.status === 404) {
reject("Sorry, the tutorial is not available in that language yet.");
return;
}
if (ajax.status !== 200) {
reject("Something went wrong while loading the tutorial.");
return;
}
resolve(ajax.responseText);
};
ajax.open('GET', `${window.location.origin}/static/generated/docs/` +
`${tutorialWrapper.getAttribute('data-game')}/${tutorialWrapper.getAttribute('data-file')}_` +
`${tutorialWrapper.getAttribute('data-lang')}.md`, true);
ajax.send();
}).then((results) => {
// Populate page with HTML generated from markdown
showdown.setOption('tables', true);
showdown.setOption('strikethrough', true);
showdown.setOption('literalMidWordUnderscores', true);
showdown.setOption('disableForced4SpacesIndentedSublists', true);
tutorialWrapper.innerHTML += (new showdown.Converter()).makeHtml(results);
const title = document.querySelector('h1')
if (title) {
document.title = title.textContent;
}
// Reset the id of all header divs to something nicer
for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
header.setAttribute('id', headerId);
header.addEventListener('click', () => {
window.location.hash = `#${headerId}`;
header.scrollIntoView();
});
}
// Manually scroll the user to the appropriate header if anchor navigation is used
document.fonts.ready.finally(() => {
if (window.location.hash) {
const scrollTarget = document.getElementById(window.location.hash.substring(1));
scrollTarget?.scrollIntoView();
}
});
});
});

View File

@@ -1,81 +0,0 @@
const showError = () => {
const tutorial = document.getElementById('tutorial-landing');
document.getElementById('page-title').innerText = 'This page is out of logic!';
tutorial.removeChild(document.getElementById('loading'));
const userMessage = document.createElement('h3');
const homepageLink = document.createElement('a');
homepageLink.innerText = 'Click here';
homepageLink.setAttribute('href', '/');
userMessage.append(homepageLink);
userMessage.append(' to go back to safety!');
tutorial.append(userMessage);
};
window.addEventListener('load', () => {
const ajax = new XMLHttpRequest();
ajax.onreadystatechange = () => {
if (ajax.readyState !== 4) { return; }
const tutorialDiv = document.getElementById('tutorial-landing');
if (ajax.status !== 200) { return showError(); }
try {
const games = JSON.parse(ajax.responseText);
games.forEach((game) => {
const gameTitle = document.createElement('h2');
gameTitle.innerText = game.gameTitle;
gameTitle.id = `${encodeURIComponent(game.gameTitle)}`;
tutorialDiv.appendChild(gameTitle);
game.tutorials.forEach((tutorial) => {
const tutorialName = document.createElement('h3');
tutorialName.innerText = tutorial.name;
tutorialDiv.appendChild(tutorialName);
const tutorialDescription = document.createElement('p');
tutorialDescription.innerText = tutorial.description;
tutorialDiv.appendChild(tutorialDescription);
const intro = document.createElement('p');
intro.innerText = 'This guide is available in the following languages:';
tutorialDiv.appendChild(intro);
const fileList = document.createElement('ul');
tutorial.files.forEach((file) => {
const listItem = document.createElement('li');
const anchor = document.createElement('a');
anchor.innerText = file.language;
anchor.setAttribute('href', `${window.location.origin}/tutorial/${file.link}`);
listItem.appendChild(anchor);
listItem.append(' by ');
for (let author of file.authors) {
listItem.append(author);
if (file.authors.indexOf(author) !== (file.authors.length -1)) {
listItem.append(', ');
}
}
fileList.appendChild(listItem);
});
tutorialDiv.appendChild(fileList);
});
});
tutorialDiv.removeChild(document.getElementById('loading'));
} catch (error) {
showError();
console.error(error);
}
// Check if we are on an anchor when coming in, and scroll to it.
const hash = window.location.hash;
if (hash) {
const offset = 128; // To account for navbar banner at top of page.
window.scrollTo(0, 0);
const rect = document.getElementById(hash.slice(1)).getBoundingClientRect();
window.scrollTo(rect.left, rect.top - offset);
}
};
ajax.open('GET', `${window.location.origin}/static/generated/tutorials.json`, true);
ajax.send();
});

View File

@@ -1,17 +0,0 @@
{% extends 'pageWrapper.html' %}
{% block head %}
<title>{{ game }} Info</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/markdown.css") }}" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/1.9.1/showdown.min.js"
integrity="sha512-L03kznCrNOfVxOUovR6ESfCz9Gfny7gihUX/huVbQB9zjODtYpxaVtIaAkpetoiyV2eqWbvxMH9fiSv5enX7bw=="
crossorigin="anonymous"></script>
<script type="application/ecmascript" src="{{ url_for('static', filename="assets/gameInfo.js") }}"></script>
{% endblock %}
{% block body %}
{% include 'header/'+theme+'Header.html' %}
<div id="game-info" class="markdown" data-lang="{{ lang }}" data-game="{{ game | get_file_safe_name }}">
<!-- Populated my JS / MD -->
</div>
{% endblock %}

View File

@@ -1,7 +1,8 @@
{% extends 'pageWrapper.html' %}
{% block head %}
{% include 'header/grassHeader.html' %}
{% set theme_name = theme|default("grass", true) %}
{% include "header/"+theme_name+"Header.html" %}
<title>{{ title }}</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/markdown.css") }}" />
{% endblock %}

View File

@@ -1,17 +0,0 @@
{% extends 'pageWrapper.html' %}
{% block head %}
{% include 'header/'+theme+'Header.html' %}
<title>Archipelago</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/markdown.css") }}" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/1.9.1/showdown.min.js"
integrity="sha512-L03kznCrNOfVxOUovR6ESfCz9Gfny7gihUX/huVbQB9zjODtYpxaVtIaAkpetoiyV2eqWbvxMH9fiSv5enX7bw=="
crossorigin="anonymous"></script>
<script type="application/ecmascript" src="{{ url_for('static', filename="assets/tutorial.js") }}"></script>
{% endblock %}
{% block body %}
<div id="tutorial-wrapper" class="markdown" data-game="{{ game | get_file_safe_name }}" data-file="{{ file | get_file_safe_name }}" data-lang="{{ lang }}">
<!-- Content generated by JavaScript -->
</div>
{% endblock %}

View File

@@ -3,14 +3,32 @@
{% block head %}
{% include 'header/grassHeader.html' %}
<title>Archipelago Guides</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/markdown.css") }}" />
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/tutorialLanding.css") }}" />
<script type="application/ecmascript" src="{{ url_for('static', filename="assets/tutorialLanding.js") }}"></script>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/markdown.css") }}"/>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename="styles/tutorialLanding.css") }}"/>
{% endblock %}
{% block body %}
<div id="tutorial-landing" class="markdown" data-game="{{ game }}" data-file="{{ file }}" data-lang="{{ lang }}">
<h1 id="page-title">Archipelago Guides</h1>
<p id="loading">Loading...</p>
<div id="tutorial-landing" class="markdown">
<h1>Archipelago Guides</h1>
{% for world_name, world_type in worlds.items() %}
<h2 id="{{ world_type.game | urlencode }}">{{ world_type.game }}</h2>
{% for tutorial_name, tutorial_data in tutorials[world_name].items() %}
<h3>{{ tutorial_name }}</h3>
<p>{{ tutorial_data.description }}</p>
<p>This guide is available in the following languages:</p>
<ul>
{% for file_name, file_data in tutorial_data.files.items() %}
<li>
<a href="{{ url_for("tutorial", game=world_name, file=file_name) }}">{{ file_data.language }}</a>
by
{% for author in file_data.authors %}
{{ author }}
{% if not loop.last %}, {% endif %}
{% endfor %}
</li>
{% endfor %}
</ul>
{% endfor %}
{% endfor %}
</div>
{% endblock %}

View File

@@ -2,6 +2,8 @@ import unittest
import Utils
import os
from werkzeug.utils import secure_filename
import WebHost
from worlds.AutoWorld import AutoWorldRegister
@@ -9,36 +11,30 @@ from worlds.AutoWorld import AutoWorldRegister
class TestDocs(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.tutorials_data = WebHost.create_ordered_tutorials_file()
WebHost.copy_tutorials_files_to_static()
def test_has_tutorial(self):
games_with_tutorial = set(entry["gameTitle"] for entry in self.tutorials_data)
for game_name, world_type in AutoWorldRegister.world_types.items():
if not world_type.hidden:
with self.subTest(game_name):
try:
self.assertIn(game_name, games_with_tutorial)
except AssertionError:
# look for partial name in the tutorial name
for game in games_with_tutorial:
if game_name in game:
break
else:
self.fail(f"{game_name} has no setup tutorial. "
f"Games with Tutorial: {games_with_tutorial}")
tutorials = world_type.web.tutorials
self.assertGreater(len(tutorials), 0, msg=f"{game_name} has no setup tutorial.")
safe_name = secure_filename(game_name)
target_path = Utils.local_path("WebHostLib", "static", "generated", "docs", safe_name)
for tutorial in tutorials:
self.assertTrue(
os.path.isfile(Utils.local_path(target_path, secure_filename(tutorial.file_name))),
f'{game_name} missing tutorial file {tutorial.file_name}.'
)
def test_has_game_info(self):
for game_name, world_type in AutoWorldRegister.world_types.items():
if not world_type.hidden:
safe_name = Utils.get_file_safe_name(game_name)
safe_name = secure_filename(game_name)
target_path = Utils.local_path("WebHostLib", "static", "generated", "docs", safe_name)
for game_info_lang in world_type.web.game_info_languages:
with self.subTest(game_name):
self.assertTrue(
safe_name == game_name or
not os.path.isfile(Utils.local_path(target_path, f'{game_info_lang}_{game_name}.md')),
f'Info docs have be named <lang>_{safe_name}.md for {game_name}.'
)
self.assertTrue(
os.path.isfile(Utils.local_path(target_path, f'{game_info_lang}_{safe_name}.md')),
f'{game_name} missing game info file for "{game_info_lang}" language.'

View File

@@ -29,8 +29,3 @@ class TestFileGeneration(unittest.TestCase):
with open(file, encoding="utf-8-sig") as f:
for value in roll_options({file.name: f.read()})[0].values():
self.assertTrue(value is True, f"Default Options for template {file.name} cannot be run.")
def test_tutorial(self):
WebHost.create_ordered_tutorials_file()
self.assertTrue(os.path.exists(os.path.join(self.correct_path, "static", "generated", "tutorials.json")))
self.assertFalse(os.path.exists(os.path.join(self.incorrect_path, "static", "generated", "tutorials.json")))

View File

@@ -34,7 +34,7 @@ class AWebInTime(WebWorld):
"Multiworld Setup Guide",
"A guide for setting up A Hat in Time to be played in Archipelago.",
"English",
"ahit_en.md",
"setup_en.md",
"setup/en",
["CookieCat"]
)]

View File

@@ -25,7 +25,7 @@ class OSRSWeb(WebWorld):
"Multiworld Setup Guide",
"A guide to setting up the Old School Runescape Randomizer connected to an Archipelago Multiworld",
"English",
"docs/setup_en.md",
"setup_en.md",
"setup/en",
["digiholic"]
)

View File

@@ -56,7 +56,7 @@ class Yugioh06Web(WebWorld):
"A guide to setting up Yu-Gi-Oh! - Ultimate Masters Edition - World Championship Tournament 2006 "
"for Archipelago on your computer.",
"English",
"docs/setup_en.md",
"setup_en.md",
"setup/en",
["Rensen"],
)