requiremnets bump

unnecessarily duplicated now that I have div.header, and also useful for
when I start the error handling pages

incorporealcms/__init__.py

@@ -3,7 +3,6 @@ import logging
 import os
 from logging.config import dictConfig
 
-import markdown
 from flask import Flask, request, send_from_directory
 
 from ._version import get_versions
@@ -32,10 +31,6 @@ def create_app(instance_path=None, test_config=None):
 
     logger.debug("instance path: %s", app.instance_path)
 
-    # initialize markdown parser from config, but include
-    # extensions our app depends on, like the meta extension
-    app.config['md'] = markdown.Markdown(extensions=app.config['MARKDOWN_EXTENSIONS'] + ['meta'])
-
     @app.before_request
     def log_request():
         logger.info("REQUEST:  %s %s", request.method, request.path)
@@ -50,7 +45,10 @@ def create_app(instance_path=None, test_config=None):
         return send_from_directory(os.path.join(app.instance_path, app.config['MEDIA_DIR']),
                                    filename)
 
-    from . import pages
+    from . import error_pages, pages
     app.register_blueprint(pages.bp)
+    app.register_error_handler(400, error_pages.bad_request)
+    app.register_error_handler(404, error_pages.page_not_found)
+    app.register_error_handler(500, error_pages.internal_server_error)
 
     return app

incorporealcms/config.py

@@ -32,7 +32,21 @@ class Config(object):
         },
     }
 
-    MARKDOWN_EXTENSIONS = ['meta', 'tables']
+    MARKDOWN_EXTENSIONS = ['extra', 'mdx_linkify', 'sane_lists', 'smarty', 'tables']
+    MARKDOWN_EXTENSION_CONFIGS = {
+        'extra': {
+            'attr_list': {},
+            'footnotes': {
+                'UNIQUE_IDS': True,
+            },
+        },
+        'smarty': {
+            'smart_dashes': True,
+            'smart_quotes': False,
+            'smart_angled_quotes': False,
+            'smart_ellipses': True,
+        },
+    }
 
     DEFAULT_PAGE_STYLE = 'light'
 

incorporealcms/error_pages.py

@@ -0,0 +1,17 @@
+"""Error page views for 400, 404, etc."""
+from incorporealcms.lib import render
+
+
+def bad_request(error):
+    """Display 400 error messaging."""
+    return render('400.html'), 400
+
+
+def internal_server_error(error):
+    """Display 500 error messaging."""
+    return render('500.html'), 500
+
+
+def page_not_found(error):
+    """Display 404 error messaging."""
+    return render('404.html'), 404

incorporealcms/lib.py

@@ -0,0 +1,50 @@
+"""Miscellaneous helper functions and whatnot."""
+import logging
+
+import markdown
+from flask import current_app as app
+from flask import make_response, render_template, request
+
+logger = logging.getLogger(__name__)
+
+
+def get_meta_str(md, key):
+    """Provide the page's (parsed in Markup obj md) metadata for the specified key, or '' if unset."""
+    return " ".join(md.Meta.get(key)) if md.Meta.get(key) else ""
+
+
+def init_md():
+    """Initialize the Markdown parser.
+
+    This used to done at the app level in __init__, but extensions like footnotes apparently
+    assume the parser to only live for the length of parsing one document, and create double
+    footnote ref links if the one parser sees the same document multiple times.
+    """
+    # initialize markdown parser from config, but include
+    # extensions our app depends on, like the meta extension
+    return markdown.Markdown(extensions=app.config['MARKDOWN_EXTENSIONS'] + ['meta'],
+                             extension_configs=app.config['MARKDOWN_EXTENSION_CONFIGS'])
+
+
+def render(template_name_or_list, **context):
+    """Wrap Flask's render_template.
+
+    * Determine the proper site theme to use in the template and provide it.
+    """
+    PAGE_STYLES = {
+        'dark': 'css/dark.css',
+        'light': 'css/light.css',
+    }
+
+    selected_style = request.args.get('style', None)
+    if selected_style:
+        user_style = selected_style
+    else:
+        user_style = request.cookies.get('user-style')
+        logger.debug("user style cookie: %s", user_style)
+    context['user_style'] = PAGE_STYLES.get(user_style, PAGE_STYLES.get(app.config['DEFAULT_PAGE_STYLE']))
+
+    resp = make_response(render_template(template_name_or_list, **context))
+    if selected_style:
+        resp.set_cookie('user-style', selected_style)
+    return resp

incorporealcms/pages.py

@@ -2,12 +2,15 @@
 import datetime
 import logging
 import os
+import re
 
 from flask import Blueprint, Markup, abort
 from flask import current_app as app
-from flask import make_response, redirect, render_template, request
+from flask import redirect, request
 from tzlocal import get_localzone
 
+from incorporealcms.lib import get_meta_str, init_md, render
+
 logger = logging.getLogger(__name__)
 
 bp = Blueprint('pages', __name__, url_prefix='/')
@@ -17,97 +20,115 @@ bp = Blueprint('pages', __name__, url_prefix='/')
 @bp.route('/<path:path>')
 def display_page(path):
     """Get the file contents of the requested path and render the file."""
-    if is_file_path_actually_dir_path(path):
+    try:
-        return redirect(f'{path}/', code=301)
+        resolved_path = request_path_to_instance_resource_path(path)
-
-    resolved_path = resolve_page_file(path)
         logger.debug("received request for path '%s', resolved to '%s'", path, resolved_path)
+    except PermissionError:
+        abort(400)
+    except IsADirectoryError:
+        return redirect(f'{path}/', code=301)
+    except FileNotFoundError:
+        abort(404)
+
     try:
         with app.open_instance_resource(resolved_path, 'r') as entry_file:
-            logger.debug("file '%s' found", resolved_path)
-            parent_navs = generate_parent_navs(path)
             mtime = datetime.datetime.fromtimestamp(os.path.getmtime(entry_file.name), get_localzone())
             entry = entry_file.read()
-    except FileNotFoundError:
+    except OSError:
-        logger.warning("requested path '%s' (resolved path '%s') not found!", path, resolved_path)
+        logger.exception("resolved path '%s' could not be opened!", resolved_path)
-        abort(404)
+        abort(500)
     else:
-        content = Markup(app.config['md'].convert(entry))
+        md = init_md()
-        logger.debug("file metadata: %s", app.config['md'].Meta)
+        content = Markup(md.convert(entry))
-        title = " ".join(app.config['md'].Meta.get('title')) if app.config['md'].Meta.get('title') else ""
+        logger.debug("file metadata: %s", md.Meta)
-        return render('base.html', title=title, content=content, navs=parent_navs,
+
+        parent_navs = generate_parent_navs(resolved_path)
+
+        page_name = (get_meta_str(md, 'title') if md.Meta.get('title') else
+                     f'/{instance_resource_path_to_request_path(resolved_path)}')
+        page_title = f'{page_name} - {app.config["TITLE_SUFFIX"]}' if page_name else app.config['TITLE_SUFFIX']
+        logger.debug("title (potentially derived): %s", page_title)
+
+        return render('base.html', title=page_title, description=get_meta_str(md, 'description'),
+                      image=get_meta_str(md, 'image'), base_url=request.base_url, content=content, navs=parent_navs,
                       mtime=mtime.strftime('%Y-%m-%d %H:%M:%S %Z'))
 
 
-def render(template_name_or_list, **context):
+def request_path_to_instance_resource_path(path):
-    """Wrap Flask's render_template.
+    """Turn a request URL path to the full page path.
 
-    * Determine the proper site theme to use in the template and provide it.
+    flask.Flask.open_instance_resource will open a file like /etc/hosts if you tell it to,
+    which sucks, so we do a lot of work here to make sure we have a valid request to
+    something inside the pages dir.
     """
-    PAGE_STYLES = {
+    # check if the path is allowed
-        'dark': 'css/dark.css',
+    base_dir = os.path.realpath(f'{app.instance_path}/pages/')
-        'light': 'css/light.css',
+    resolved_path = os.path.realpath(os.path.join(base_dir, path))
-    }
+    logger.debug("base_dir: %s, constructed resolved_path: %s", base_dir, resolved_path)
 
-    selected_style = request.args.get('style', None)
+    # bail if the requested real path isn't inside the base directory
-    if selected_style:
+    if base_dir != os.path.commonpath((base_dir, resolved_path)):
-        user_style = selected_style
+        logger.warning("client tried to request a path '%s' outside of the base_dir!", path)
+        raise PermissionError
+
+    # if this is a file-like requset but actually a directory, redirect the user
+    if os.path.isdir(resolved_path) and not path.endswith('/'):
+        logger.info("client requested a path '%s' that is actually a directory", path)
+        raise IsADirectoryError
+
+    # derive the proper markdown file depending on if this is a dir or file
+    if os.path.isdir(resolved_path):
+        absolute_resource = os.path.join(resolved_path, 'index.md')
     else:
-        user_style = request.cookies.get('user-style')
+        absolute_resource = f'{resolved_path}.md'
-        logger.debug("user style cookie: %s", user_style)
-    context['user_style'] = PAGE_STYLES.get(user_style, PAGE_STYLES.get(app.config['DEFAULT_PAGE_STYLE']))
 
-    resp = make_response(render_template(template_name_or_list, **context))
+    logger.info("final path = '%s' for request '%s'", absolute_resource, path)
-    if selected_style:
+
-        resp.set_cookie('user-style', selected_style)
+    # does the final file actually exist?
-    return resp
+    if not os.path.exists(absolute_resource):
+        logger.warning("requested final path '%s' does not exist!", absolute_resource)
+        raise FileNotFoundError
+
+    # we checked that the file exists via absolute path, but now we need to give the path relative to instance dir
+    return absolute_resource.replace(f'{app.instance_path}{os.path.sep}', '')
 
 
-def resolve_page_file(path):
+def instance_resource_path_to_request_path(path):
-    """Manipulate the request path to find appropriate page file.
+    """Reverse a (presumed to exist) disk path to the canonical path that would show up in a Flask route.
 
-    * convert dir requests to index files
+    This does not include the leading /, so aside from the root index case, this should be
-
+    bidirectional.
-    Worth noting, Flask already does stuff like convert '/foo/../../../bar' to
-    '/bar', so we don't need to take care around file access here.
     """
-    if path.endswith('/'):
+    return re.sub(r'^pages/', '', re.sub(r'.md$', '', re.sub(r'index.md$', '', path)))
-        path = f'{path}index'
-    path = f'pages/{path}.md'
-    return path
-
-
-def is_file_path_actually_dir_path(path):
-    """Check if requested path which looks like a file is actually a directory.
-
-    If, for example, /foo used to be a file (foo.md) which later became a directory,
-    foo/, this returns True. Useful for when I make my structure more complicated
-    than it originally was, or if users are just weird.
-    """
-    if not path.endswith('/'):
-        logger.debug("requested path '%s' looks like a file", path)
-        if os.path.isdir(f'{app.instance_path}/pages/{path}'):
-            logger.debug("...and it's actually a dir")
-            return True
-
-    return False
 
 
 def generate_parent_navs(path):
-    """Create a series of paths/links to navigate up from the given path."""
+    """Create a series of paths/links to navigate up from the given resource path."""
-    # derive additional path/location stuff based on path
+    if path == 'pages/index.md':
-    resolved_path = resolve_page_file(path)
+        # bail and return the title suffix (generally the domain name) as a terminal case
-    parent_dir = os.path.dirname(resolved_path)
-    parent_path = '/'.join(path[:-1].split('/')[:-1]) + '/'
-
-    logger.debug("path: '%s'; parent path: '%s'; resolved path: '%s'; parent dir: '%s'",
-                 path, parent_path, resolved_path, parent_dir)
-
-    if path in ('index', '/'):
         return [(app.config['TITLE_SUFFIX'], '/')]
     else:
-        with app.open_instance_resource(resolved_path, 'r') as entry_file:
+        if path.endswith('index.md'):
+            # index case: one dirname for foo/bar/index.md -> foo/bar, one for foo/bar -> foo
+            parent_resource_dir = os.path.dirname(os.path.dirname(path))
+        else:
+            # usual case: foo/buh.md -> foo
+            parent_resource_dir = os.path.dirname(path)
+
+        # generate the request path (i.e. what the link will be) for this path, and
+        # also the resource path of this parent (which is always a dir, so always index.md)
+        request_path = f'/{instance_resource_path_to_request_path(path)}'
+        parent_resource_path = os.path.join(parent_resource_dir, 'index.md')
+
+        logger.debug("resource path: '%s'; request path: '%s'; parent resource path: '%s'", path,
+                     request_path, parent_resource_path)
+
+        # for issues regarding parser reuse (see lib.init_md) we reinitialize the parser here
+        md = init_md()
+
+        # read the resource
+        with app.open_instance_resource(path, 'r') as entry_file:
             entry = entry_file.read()
-        _ = Markup(app.config['md'].convert(entry))
+        _ = Markup(md.convert(entry))
-        page_name = " ".join(app.config['md'].Meta.get('title')) if app.config['md'].Meta.get('title') else f'/{path}'
+        page_name = " ".join(md.Meta.get('title')) if md.Meta.get('title') else request_path
-        return generate_parent_navs(parent_path) + [(page_name, f'/{path}')]
+        return generate_parent_navs(parent_resource_path) + [(page_name, request_path)]

incorporealcms/static/css/dark.css

@@ -1,11 +1,15 @@
 html {
-    color: #DDD;
+    color: #CCC;
 }
 
 body {
     background: black;
 }
 
+strong {
+    color: #EEE;
+}
+
 .site-wrap {
     background: #111;
 
@@ -25,16 +29,16 @@ a:link, a:visited {
 
 a:hover, a:active {
     color: #B31D15;
-    border-bottom: 1px dotted #EEE;
+    border-bottom: 1px dotted #B31D15;
 }
 
-section.nav, section.styles {
+div.header {
-    color: #BBB;
     background: #222;
     border-bottom: 1px solid #222;
+    color: #BBB;
 }
 
-section.nav a, section.styles a {
+div.header a {
     color: #BBB;
 }
 
@@ -50,3 +54,17 @@ blockquote {
     background-color: rgba(120, 120, 120, 0.3);
     border: 1px solid #222;
 }
+
+.img-frame {
+    background-color: rgba(255, 255, 255, 0.1);
+    border: 1px solid #333;
+}
+
+figure {
+    background: #222;
+    border: 1px solid #333;
+}
+
+figcaption {
+    color: #BBB;
+}

incorporealcms/static/css/light.css

@@ -3,7 +3,11 @@ html {
 }
 
 body {
-    background: #888;
+    background: #999;
+}
+
+strong {
+    color: #111;
 }
 
 .site-wrap {
@@ -25,16 +29,16 @@ a:link, a:visited {
 
 a:hover, a:active {
     color: #811610;
-    border-bottom: 1px dotted #111;
+    border-bottom: 1px dotted #811610;
 }
 
-section.nav, section.styles {
+div.header {
-    color: #666;
     background: #EEE;
     border-bottom: 1px solid #CCC;
+    color: #666;
 }
 
-section.nav a, section.styles a {
+div.header a {
     color: #666;
 }
 
@@ -50,3 +54,17 @@ blockquote {
     background-color: rgba(120, 120, 120, 0.1);
     border: 1px solid #CCC;
 }
+
+.img-frame {
+    background-color: rgba(0, 0, 0, 0.1);
+    border: 1px solid #BBB;
+}
+
+figure {
+    background: #EFEFEF;
+    border: 1px solid #CCCCCC;
+}
+
+figcaption {
+    color: #777777;
+}

incorporealcms/static/css/style.css

@@ -63,27 +63,33 @@ a:active {
     text-decoration: none;
 }
 
-section.nav, section.styles {
+div.header {
+    display: flex;
+    justify-content: space-between;
     font-size: 0.75em;
-    border-bottom: 1px solid #444;
     padding: 0.25em 0.5em;
 }
 
-section.nav a, section.styles a {
+div.header a {
     border-bottom: none;
 }
 
-section.styles {
+div.content {
-    float: right;
-}
-
-section.content {
     font-size: 11pt;
 
     padding: 0 1em;
     line-height: 1.5em;
 }
 
+sup, sub {
+	vertical-align: baseline;
+	position: relative;
+	top: -0.4em;
+}
+sub {
+	top: 0.4em;
+}
+
 footer {
     display: block;
     font-size: 75%;
@@ -109,9 +115,104 @@ blockquote {
 }
 
 .splash {
+    margin-top: 1em;
+    margin-bottom: 1em;
     text-align: center;
 }
 
-.img-50 {
+img {
-    max-width: 50%;
+    max-width: 100%;
+}
+
+.img-25 {
+    max-width: 25% !important;
+}
+
+.img-50 {
+    max-width: 50% !important;
+}
+
+.img-center {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+.img-left {
+    float: left;
+    margin-right: 1em;
+}
+
+.img-right {
+    float: right;
+    margin-left: 1em;
+}
+
+.img-frame {
+    padding: 5px;
+}
+
+/* For screens with width smaller than 400px */
+.figure-left .figure-right {
+    max-width: 95%;
+    float: none;
+    margin-left: 10px;
+    margin-right: 10px;
+}
+
+/* For larger screens */
+@media only screen and (min-width: 400px) {
+    .figure-left {
+        float: left;
+        margin-top: 0;
+        margin-left: 0;
+    }
+
+    .figure-right {
+        float: right;
+        margin-top: 0;
+        margin-right: 0;
+    }
+}
+
+figure {
+    max-width: 400px;
+    padding: 5px;
+    margin: 10px;
+    margin-top: 0;
+    margin-bottom: 5px;
+}
+
+figure img {
+    max-width: 100%;
+    height: auto;
+}
+
+figcaption {
+    font-family: "Times New Roman", serif;
+    color: #777777;
+    text-align: center;
+    font-style: italic;
+    line-height: 1.3em;
+    margin-top: 5px;
+}
+
+.thumbnail-image {
+    width: 150px;
+    height: auto;
+    margin: 5px;
+    display: inline;
+}
+
+.footnote {
+    font-size: 0.8em;
+}
+
+.footnote p {
+    margin: 0;
+}
+
+.footnote-ref:link, .footnote-ref:visited, .footnote-ref:hover, .footnote-ref:active {
+    border-bottom: none;
+    font-weight: normal;
 }

incorporealcms/templates/400.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block header %}
+<div class="header">
+    <a href="/">{{ config.TITLE_SUFFIX }}</a>
+</div>
+{% endblock %}
+
+{% block body %}
+<div class="content">
+    <h1>BAD REQUEST</h1>
+    <p>You're doing something you're not supposed to. Stop it?</p>
+</div>
+{% endblock %}

incorporealcms/templates/404.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+
+{% block header %}
+<div class="header">
+    <a href="/">{{ config.TITLE_SUFFIX }}</a>
+</div>
+{% endblock %}
+
+{% block body %}
+<div class="content">
+    <h1>NOT FOUND</h1>
+    <p>Sorry, <b><tt>{{ request.path }}</tt></b> does not seem to exist, at least not anymore.</p>
+    <p>It's possible you followed a dead link on this site, in which case I would appreciate it if you could email me via:
+    bss @ &lt;this domain&gt; and I can take a look. I make an effort to symlink old content to its new location,
+    so old links and URLs should, generally speaking, work.</p>
+    <p>Otherwise, I suggest you go <a href="/">to the index</a> and navigate your way (hopefully) to what
+    you're looking for.</p>
+</div>
+{% endblock %}

incorporealcms/templates/500.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block header %}
+<div class="header">
+    <a href="/">{{ config.TITLE_SUFFIX }}</a>
+</div>
+{% endblock %}
+
+{% block body %}
+<div class="content">
+    <h1>INTERNAL SERVER ERROR</h1>
+    <p>Something bad happened! Please email me at bss @ &lt;this domain&gt; and tell me what happened.</p>
+</div>
+{% endblock %}

incorporealcms/templates/base.html

@@ -1,24 +1,38 @@
 <!doctype html>
-<title>{{ title }}{% if title %} - {% endif %}{{ config.TITLE_SUFFIX }}</title>
+<html lang="en">
+<title>{{ title }}</title>
+{% if title %}<meta property="og:title" content="{{ title }}">{% endif %}
+{% if description %}<meta property="og:description" content="{{ description }}">{% endif %}
+{% if image %}<meta property="og:image" content="{{ image }}">{% endif %}
+<meta property="og:url" content="{{ base_url }}">
+<meta name="twitter:card" content="summary_large_image">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
 <link rel="stylesheet" href="{{ url_for('static', filename=user_style) }}">
 <link rel="icon" href="{{ url_for('static', filename='img/favicon.png') }}">
-<section class="site-wrap">
+
-    <section class="styles">
+<div class="site-wrap">
-        <a href="?style=dark">[dark]</a>
+    {% block header %}
-        <a href="?style=light">[light]</a>
+    <div class="header">
-    </section>
+        <div class="nav">
-    <section class="nav">
             {% for nav in navs %}
             <a href="{{ nav.1 }}">{{ nav.0 }}</a>
             {% if not loop.last %} &raquo; {% endif %}
             {% endfor %}
-    </section>
+        </div>
-    <section class="content">
+        <div class="styles">
+            <a href="?style=dark">[dark]</a>
+            <a href="?style=light">[light]</a>
+        </div>
+    </div>
+    {% endblock %}
+    {% block body %}
+    <div class="content">
         {{ content }}
-    </section>
+    </div>
     <footer>
         <i>Last modified: {{ mtime }}</i>
     </footer>
-</section>
+    {% endblock %}
+</div>
+</html>

requirements/requirements-dev.in

@@ -5,7 +5,7 @@ pytest
 pytest-cov
 
 # linting and other static code analysis
-bandit
+bandit==1.6.2               # pinned because 1.7.0 wasn't running right in tox
 dlint
 flake8
 flake8-blind-except

requirements/requirements-dev.txt

@@ -4,59 +4,148 @@
 #
 #    pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
 #
-appdirs==1.4.4            # via virtualenv
+appdirs==1.4.4
-attrs==20.3.0             # via pytest
+    # via virtualenv
-bandit==1.6.3             # via -r requirements/requirements-dev.in
+attrs==20.3.0
-click==7.1.2              # via flask, pip-tools
+    # via pytest
-coverage==5.3             # via pytest-cov
+bandit==1.6.2
-distlib==0.3.1            # via virtualenv
+    # via -r requirements/requirements-dev.in
-dlint==0.11.0             # via -r requirements/requirements-dev.in
+bleach==3.3.0
-filelock==3.0.12          # via tox, virtualenv
+    # via mdx-linkify
-flake8-blind-except==0.1.1  # via -r requirements/requirements-dev.in
+click==7.1.2
-flake8-builtins==1.5.3    # via -r requirements/requirements-dev.in
+    # via
-flake8-docstrings==1.5.0  # via -r requirements/requirements-dev.in
+    #   flask
-flake8-executable==2.1.0  # via -r requirements/requirements-dev.in
+    #   pip-tools
-flake8-fixme==1.1.1       # via -r requirements/requirements-dev.in
+coverage==5.4
-flake8-isort==4.0.0       # via -r requirements/requirements-dev.in
+    # via pytest-cov
-flake8-logging-format==0.6.0  # via -r requirements/requirements-dev.in
+distlib==0.3.1
-flake8-mutable==1.2.0     # via -r requirements/requirements-dev.in
+    # via virtualenv
-flake8==3.8.4             # via -r requirements/requirements-dev.in, dlint, flake8-builtins, flake8-docstrings, flake8-executable, flake8-isort, flake8-mutable
+dlint==0.11.0
-flask==1.1.2              # via -r requirements/requirements.in
+    # via -r requirements/requirements-dev.in
-gitdb==4.0.5              # via gitpython
+filelock==3.0.12
-gitpython==3.1.11         # via bandit
+    # via
-iniconfig==1.1.1          # via pytest
+    #   tox
-isort==5.6.4              # via flake8-isort
+    #   virtualenv
-itsdangerous==1.1.0       # via flask
+flake8-blind-except==0.2.0
-jinja2==2.11.2            # via flask
+    # via -r requirements/requirements-dev.in
-markdown==3.3.3           # via -r requirements/requirements.in
+flake8-builtins==1.5.3
-markupsafe==1.1.1         # via jinja2
+    # via -r requirements/requirements-dev.in
-mccabe==0.6.1             # via flake8
+flake8-docstrings==1.5.0
-packaging==20.7           # via pytest, tox
+    # via -r requirements/requirements-dev.in
-pbr==5.5.1                # via stevedore
+flake8-executable==2.1.1
-pip-tools==5.4.0          # via -r requirements/requirements-dev.in
+    # via -r requirements/requirements-dev.in
-pluggy==0.13.1            # via pytest, tox
+flake8-fixme==1.1.1
-py==1.9.0                 # via pytest, tox
+    # via -r requirements/requirements-dev.in
-pycodestyle==2.6.0        # via flake8
+flake8-isort==4.0.0
-pydocstyle==5.1.1         # via flake8-docstrings
+    # via -r requirements/requirements-dev.in
-pyflakes==2.2.0           # via flake8
+flake8-logging-format==0.6.0
-pyparsing==2.4.7          # via packaging
+    # via -r requirements/requirements-dev.in
-pytest-cov==2.10.1        # via -r requirements/requirements-dev.in
+flake8-mutable==1.2.0
-pytest==6.1.2             # via -r requirements/requirements-dev.in, pytest-cov
+    # via -r requirements/requirements-dev.in
-pytz==2020.4              # via tzlocal
+flake8==3.8.4
-pyyaml==5.3.1             # via bandit
+    # via
-six==1.15.0               # via bandit, pip-tools, tox, virtualenv
+    #   -r requirements/requirements-dev.in
-smmap==3.0.4              # via gitdb
+    #   dlint
-snowballstemmer==2.0.0    # via pydocstyle
+    #   flake8-builtins
-stevedore==3.3.0          # via bandit
+    #   flake8-docstrings
-testfixtures==6.15.0      # via flake8-isort
+    #   flake8-executable
-toml==0.10.2              # via pytest, tox
+    #   flake8-isort
-tox-wheel==0.6.0          # via -r requirements/requirements-dev.in
+    #   flake8-mutable
-tox==3.20.1               # via -r requirements/requirements-dev.in, tox-wheel
+flask==1.1.2
-tzlocal==2.1              # via -r requirements/requirements.in
+    # via -r requirements/requirements.in
-versioneer==0.19          # via -r requirements/requirements-dev.in
+gitdb==4.0.5
-virtualenv==20.2.2        # via tox
+    # via gitpython
-werkzeug==1.0.1           # via flask
+gitpython==3.1.13
-wheel==0.36.1             # via tox-wheel
+    # via bandit
+iniconfig==1.1.1
+    # via pytest
+isort==5.7.0
+    # via flake8-isort
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via flask
+markdown==3.3.3
+    # via
+    #   -r requirements/requirements.in
+    #   mdx-linkify
+markupsafe==1.1.1
+    # via jinja2
+mccabe==0.6.1
+    # via flake8
+mdx-linkify==2.1
+    # via -r requirements/requirements.in
+packaging==20.9
+    # via
+    #   bleach
+    #   pytest
+    #   tox
+pbr==5.5.1
+    # via stevedore
+pip-tools==5.5.0
+    # via -r requirements/requirements-dev.in
+pluggy==0.13.1
+    # via
+    #   pytest
+    #   tox
+py==1.10.0
+    # via
+    #   pytest
+    #   tox
+pycodestyle==2.6.0
+    # via flake8
+pydocstyle==5.1.1
+    # via flake8-docstrings
+pyflakes==2.2.0
+    # via flake8
+pyparsing==2.4.7
+    # via packaging
+pytest-cov==2.11.1
+    # via -r requirements/requirements-dev.in
+pytest==6.2.2
+    # via
+    #   -r requirements/requirements-dev.in
+    #   pytest-cov
+pytz==2021.1
+    # via tzlocal
+pyyaml==5.4.1
+    # via bandit
+six==1.15.0
+    # via
+    #   bandit
+    #   bleach
+    #   tox
+    #   virtualenv
+smmap==3.0.5
+    # via gitdb
+snowballstemmer==2.1.0
+    # via pydocstyle
+stevedore==3.3.0
+    # via bandit
+testfixtures==6.17.1
+    # via flake8-isort
+toml==0.10.2
+    # via
+    #   pytest
+    #   tox
+tox-wheel==0.6.0
+    # via -r requirements/requirements-dev.in
+tox==3.22.0
+    # via
+    #   -r requirements/requirements-dev.in
+    #   tox-wheel
+tzlocal==2.1
+    # via -r requirements/requirements.in
+versioneer==0.19
+    # via -r requirements/requirements-dev.in
+virtualenv==20.4.2
+    # via tox
+webencodings==0.5.1
+    # via bleach
+werkzeug==1.0.1
+    # via flask
+wheel==0.36.2
+    # via tox-wheel
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/requirements.in

@@ -1,3 +1,4 @@
 Flask                       # general purpose web service and web server stuff
 Markdown                    # markdown rendering in templates
+mdx-linkify                 # convert URLs in the text to clickable links
 tzlocal                     # identifying system's local timezone

requirements/requirements.txt

@@ -4,12 +4,35 @@
 #
 #    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
 #
-click==7.1.2              # via flask
+bleach==3.3.0
-flask==1.1.2              # via -r requirements/requirements.in
+    # via mdx-linkify
-itsdangerous==1.1.0       # via flask
+click==7.1.2
-jinja2==2.11.2            # via flask
+    # via flask
-markdown==3.3.3           # via -r requirements/requirements.in
+flask==1.1.2
-markupsafe==1.1.1         # via jinja2
+    # via -r requirements/requirements.in
-pytz==2020.4              # via tzlocal
+itsdangerous==1.1.0
-tzlocal==2.1              # via -r requirements/requirements.in
+    # via flask
-werkzeug==1.0.1           # via flask
+jinja2==2.11.3
+    # via flask
+markdown==3.3.3
+    # via
+    #   -r requirements/requirements.in
+    #   mdx-linkify
+markupsafe==1.1.1
+    # via jinja2
+mdx-linkify==2.1
+    # via -r requirements/requirements.in
+packaging==20.9
+    # via bleach
+pyparsing==2.4.7
+    # via packaging
+pytz==2021.1
+    # via tzlocal
+six==1.15.0
+    # via bleach
+tzlocal==2.1
+    # via -r requirements/requirements.in
+webencodings==0.5.1
+    # via bleach
+werkzeug==1.0.1
+    # via flask

tests/functional_tests.py

@@ -10,9 +10,30 @@ def test_page_that_exists(client):
 
 
 def test_page_that_doesnt_exist(client):
-    """Test that the app returns 404 for nonsense requests."""
+    """Test that the app returns 404 for nonsense requests and they use my error page."""
     response = client.get('/ohuesthaoeusth')
     assert response.status_code == 404
+    assert b'<b><tt>/ohuesthaoeusth</tt></b> does not seem to exist' in response.data
+
+
+def test_files_outside_pages_do_not_get_served(client):
+    """Test that page pathing doesn't break out of the instance/pages/ dir, and the error uses my error page."""
+    response = client.get('/../unreachable')
+    assert response.status_code == 400
+    assert b'You\'re doing something you\'re not supposed to. Stop it?' in response.data
+
+
+def test_internal_server_error_serves_error_page(client):
+    """Test that various exceptions serve up the 500 page."""
+    response = client.get('/actually-a-png')
+    assert response.status_code == 500
+    assert b'INTERNAL SERVER ERROR' in response.data
+
+
+def test_weird_paths_do_not_get_served(client):
+    """Test that we clean up requests as desired."""
+    response = client.get('/../../')
+    assert response.status_code == 400
 
 
 def test_page_with_title_metadata(client):
@@ -26,8 +47,39 @@ def test_page_without_title_metadata(client):
     """Test that a page without title metadata gets the default title."""
     response = client.get('/no-title')
     assert response.status_code == 200
+    assert b'<title>/no-title - incorporeal.org</title>' in response.data
+
+
+def test_page_in_subdir_without_title_metadata(client):
+    """Test that the title-less page display is as expected."""
+    response = client.get('/subdir//page-no-title')
+    assert response.status_code == 200
+    assert b'<title>/subdir/page-no-title - incorporeal.org</title>' in response.data
+
+
+def test_page_with_card_metadata(client):
+    """Test that a page with opengraph metadata."""
+    response = client.get('/more-metadata')
+    assert response.status_code == 200
+    assert b'<meta property="og:title" content="title for the page - incorporeal.org">' in response.data
+    assert b'<meta property="og:description" content="description of this page made even longer">' in response.data
+    assert b'<meta property="og:image" content="http://buh.com/test.img">' in response.data
+
+
+def test_page_with_card_title_even_when_no_metadata(client):
+    """Test that a page without metadata still has a card with the derived title."""
+    response = client.get('/no-title')
+    assert response.status_code == 200
+    assert b'<meta property="og:title" content="/no-title - incorporeal.org">' in response.data
+    assert b'<meta property="og:description"' not in response.data
+    assert b'<meta property="og:image"' not in response.data
+
+
+def test_page_with_forced_empty_title_just_shows_suffix(client):
+    """Test that if a page specifies a blank Title meta tag explicitly, only the suffix is used in the title."""
+    response = client.get('/forced-no-title')
+    assert response.status_code == 200
     assert b'<title>incorporeal.org</title>' in response.data
-    assert b'<h1>this page doesn\'t have a title!</h1>' in response.data
 
 
 def test_page_has_modified_timestamp(client):

tests/instance/pages/forced-no-title.md

@@ -0,0 +1,3 @@
+Title:
+
+some words are here

tests/instance/pages/more-metadata.md

@@ -0,0 +1,6 @@
+Title:          title for the page
+Description:    description of this page
+                made even longer
+Image:          http://buh.com/test.img
+
+hello

tests/instance/pages/subdir/page-no-title.md

@@ -0,0 +1 @@
+this is just a page

tests/instance/unreachable.md

@@ -0,0 +1 @@
+this file exists but the app should not serve it.

tests/test_factory.py

@@ -23,20 +23,24 @@ def test_markdown_meta_extension_always():
     assert b'<title>Index - incorporeal.org</title>' in response.data
 
 
-def test_extra_markdown_extensions_work():
+def test_custom_markdown_extensions_work():
-    """Test we can load more extensions via config, and that they work."""
+    """Test we can change extensions via config, and that they work.
+
+    This used to test smarty, but that's added by default now, so we test
+    that it can be removed by overriding the option.
+    """
     app = create_app(instance_path=os.path.join(HERE, 'instance'))
     client = app.test_client()
     response = client.get('/mdash-or-triple-dash')
     assert response.status_code == 200
-    assert b'word --- word' in response.data
+    assert b'word &mdash; word' in response.data
 
     app = create_app(instance_path=os.path.join(HERE, 'instance'),
-                     test_config={'MARKDOWN_EXTENSIONS': ['smarty']})
+                     test_config={'MARKDOWN_EXTENSIONS': []})
     client = app.test_client()
     response = client.get('/mdash-or-triple-dash')
     assert response.status_code == 200
-    assert b'word &mdash; word' in response.data
+    assert b'word --- word' in response.data
 
 
 def test_title_override():
@@ -46,7 +50,7 @@ def test_title_override():
     client = app.test_client()
     response = client.get('/no-title')
     assert response.status_code == 200
-    assert b'<title>suou.net</title>' in response.data
+    assert b'<title>/no-title - suou.net</title>' in response.data
 
 
 def test_media_file_access(client):

tests/test_pages.py

@@ -1,83 +1,40 @@
 """Unit test helper methods."""
+import pytest
 from werkzeug.http import dump_cookie
 
-from incorporealcms.pages import generate_parent_navs, is_file_path_actually_dir_path, render, resolve_page_file
+from incorporealcms.pages import (generate_parent_navs, instance_resource_path_to_request_path, render,
-
+                                  request_path_to_instance_resource_path)
-
-def test_resolve_page_file_dir_to_index():
-    """Test that a request to a directory path results in the dir's index.md."""
-    assert resolve_page_file('foo/') == 'pages/foo/index.md'
-
-
-def test_resolve_page_file_subdir_to_index():
-    """Test that a request to a dir's subdir path results in the subdir's index.md."""
-    assert resolve_page_file('foo/bar/') == 'pages/foo/bar/index.md'
-
-
-def test_resolve_page_file_other_requests_fine():
-    """Test that a request to non-dir path results in a Markdown file."""
-    assert resolve_page_file('foo/baz') == 'pages/foo/baz.md'
 
 
 def test_generate_page_navs_index(app):
     """Test that the index page has navs to the root (itself)."""
     with app.app_context():
-        assert generate_parent_navs('/') == [('incorporeal.org', '/')]
+        assert generate_parent_navs('pages/index.md') == [('incorporeal.org', '/')]
-
-
-def test_generate_page_navs_alternate_index(app):
-    """Test that the index page (as a page, not a path) also has navs only to the root (by path)."""
-    with app.app_context():
-        assert generate_parent_navs('index') == [('incorporeal.org', '/')]
 
 
 def test_generate_page_navs_subdir_index(app):
     """Test that dir pages have navs to the root and themselves."""
     with app.app_context():
-        assert generate_parent_navs('subdir/') == [('incorporeal.org', '/'), ('/subdir/', '/subdir/')]
+        assert generate_parent_navs('pages/subdir/index.md') == [('incorporeal.org', '/'), ('/subdir/', '/subdir/')]
 
 
 def test_generate_page_navs_subdir_real_page(app):
     """Test that real pages have navs to the root, their parent, and themselves."""
     with app.app_context():
-        assert generate_parent_navs('subdir/page') == [('incorporeal.org', '/'), ('/subdir/', '/subdir/'),
+        assert generate_parent_navs('pages/subdir/page.md') == [('incorporeal.org', '/'), ('/subdir/', '/subdir/'),
                                                                 ('Page', '/subdir/page')]
 
 
 def test_generate_page_navs_subdir_with_title_parsing_real_page(app):
     """Test that title metadata is used in the nav text."""
     with app.app_context():
-        assert generate_parent_navs('subdir-with-title/page') == [
+        assert generate_parent_navs('pages/subdir-with-title/page.md') == [
             ('incorporeal.org', '/'),
             ('SUB!', '/subdir-with-title/'),
             ('/subdir-with-title/page', '/subdir-with-title/page')
         ]
 
 
-def test_is_file_path_actually_dir_path_valid_file_is_yes(app):
-    """Test that a file request for what's actually a directory is detected as such."""
-    with app.app_context():
-        assert is_file_path_actually_dir_path('/subdir')
-
-
-def test_is_file_path_actually_dir_path_valid_dir_is_no(app):
-    """Test that a directory request is still a directory request."""
-    with app.app_context():
-        assert not is_file_path_actually_dir_path('/subdir/')
-
-
-def test_is_file_path_actually_dir_path_nonsense_file_is_no(app):
-    """Test that requests to nonsense file-looking paths aren't treated as dirs."""
-    with app.app_context():
-        assert not is_file_path_actually_dir_path('/antphnathpnthapnthsnthax')
-
-
-def test_is_file_path_actually_dir_path_nonsense_dir_is_no(app):
-    """Test that a directory request is a directory request even if the dir doesn't exist."""
-    with app.app_context():
-        assert not is_file_path_actually_dir_path('/antphnathpnthapnthsnthax/')
-
-
 def test_render_with_user_dark_theme(app):
     """Test that a request with the dark theme selected renders the dark theme."""
     cookie = dump_cookie("user-style", 'dark')
@@ -98,3 +55,110 @@ def test_render_with_no_user_theme(app):
     with app.test_request_context():
         assert b'light.css' in render('base.html').data
         assert b'dark.css' not in render('base.html').data
+
+
+def test_request_path_to_instance_resource_path(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('index') == 'pages/index.md'
+
+
+def test_request_path_to_instance_resource_path_direct_file(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('no-title') == 'pages/no-title.md'
+
+
+def test_request_path_to_instance_resource_path_in_subdir(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('subdir/page') == 'pages/subdir/page.md'
+
+
+def test_request_path_to_instance_resource_path_subdir_index(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('subdir/') == 'pages/subdir/index.md'
+
+
+def test_request_path_to_instance_resource_path_relatives_walked(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert (request_path_to_instance_resource_path('subdir/more-subdir/../../more-metadata') ==
+                'pages/more-metadata.md')
+
+
+def test_request_path_to_instance_resource_path_relatives_walked_indexes_work_too(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('subdir/more-subdir/../../') == 'pages/index.md'
+
+
+def test_request_path_to_instance_resource_path_relatives_walked_into_subdirs_also_fine(app):
+    """Test a normal URL request is transformed into the file path."""
+    with app.test_request_context():
+        assert request_path_to_instance_resource_path('subdir/more-subdir/../../subdir/page') == 'pages/subdir/page.md'
+
+
+def test_request_path_to_instance_resource_path_permission_error_on_ref_above_pages(app):
+    """Test that attempts to get above the base dir ("/../../foo") fail."""
+    with app.test_request_context():
+        with pytest.raises(PermissionError):
+            assert request_path_to_instance_resource_path('../unreachable')
+
+
+def test_request_path_to_instance_resource_path_isadirectory_on_file_like_req_for_dir(app):
+    """Test that a request for e.g. '/foo' when foo is a dir indicate to redirect."""
+    with app.test_request_context():
+        with pytest.raises(IsADirectoryError):
+            assert request_path_to_instance_resource_path('subdir')
+
+
+def test_request_path_to_instance_resource_path_nonexistant_file_errors(app):
+    """Test that a request for something not on disk errors."""
+    with app.test_request_context():
+        with pytest.raises(FileNotFoundError):
+            assert request_path_to_instance_resource_path('nthanpthpnh')
+
+
+def test_request_path_to_instance_resource_path_absolute_file_errors(app):
+    """Test that a request for something not on disk errors."""
+    with app.test_request_context():
+        with pytest.raises(PermissionError):
+            assert request_path_to_instance_resource_path('/etc/hosts')
+
+
+def test_instance_resource_path_to_request_path_on_index(app):
+    """Test index.md -> /."""
+    with app.test_request_context():
+        assert instance_resource_path_to_request_path('index.md') == ''
+
+
+def test_instance_resource_path_to_request_path_on_page(app):
+    """Test no-title.md -> no-title."""
+    with app.test_request_context():
+        assert instance_resource_path_to_request_path('no-title.md') == 'no-title'
+
+
+def test_instance_resource_path_to_request_path_on_subdir(app):
+    """Test subdir/index.md -> subdir/."""
+    with app.test_request_context():
+        assert instance_resource_path_to_request_path('subdir/index.md') == 'subdir/'
+
+
+def test_instance_resource_path_to_request_path_on_subdir_and_page(app):
+    """Test subdir/page.md -> subdir/page."""
+    with app.test_request_context():
+        assert instance_resource_path_to_request_path('subdir/page.md') == 'subdir/page'
+
+
+def test_request_resource_request_root(app):
+    """Test that a request can resolve to a resource and back to a request."""
+    with app.test_request_context():
+        instance_resource_path_to_request_path(request_path_to_instance_resource_path('index')) == ''
+
+
+def test_request_resource_request_page(app):
+    """Test that a request can resolve to a resource and back to a request."""
+    with app.test_request_context():
+        instance_resource_path_to_request_path(request_path_to_instance_resource_path('no-title')) == 'no-title'