version bumps, especially flask and werkzeug 3

no tests changed, so my implementation might have been good, but let's
use the provided check

incorporealcms/__init__.py

@@ -40,8 +40,9 @@ def create_app(instance_path=None, test_config=None):
         logger.info("RESPONSE: %s %s: %s", request.method, request.path, response.status)
         return response
 
-    from . import error_pages, pages
+    from . import error_pages, pages, static
     app.register_blueprint(pages.bp)
+    app.register_blueprint(static.bp)
     app.register_error_handler(400, error_pages.bad_request)
     app.register_error_handler(404, error_pages.page_not_found)
     app.register_error_handler(500, error_pages.internal_server_error)

incorporealcms/config.py

@@ -50,6 +50,12 @@ class Config(object):
     MEDIA_DIR = 'media'
 
     # customizations
+    PAGE_STYLES = {
+        'dark': '/static/css/dark.css',
+        'light': '/static/css/light.css',
+        'plain': '/static/css/plain.css',
+    }
+
     DEFAULT_PAGE_STYLE = 'light'
     TITLE_SUFFIX = 'example.com'
     CONTACT_EMAIL = 'admin@example.com'

incorporealcms/lib.py

@@ -31,19 +31,15 @@ def render(template_name_or_list, **context):
 
     * Determine the proper site theme to use in the template and provide it.
     """
-    PAGE_STYLES = {
+    page_styles = app.config['PAGE_STYLES']
-        'dark': 'css/dark.css',
-        'light': 'css/light.css',
-        'plain': 'css/plain.css',
-    }
-
     selected_style = request.args.get('style', None)
     if selected_style:
         user_style = selected_style
     else:
         user_style = request.cookies.get('user-style')
         logger.debug("user style cookie: %s", user_style)
-    context['user_style'] = PAGE_STYLES.get(user_style, PAGE_STYLES.get(app.config['DEFAULT_PAGE_STYLE']))
+    context['user_style'] = page_styles.get(user_style, page_styles.get(app.config['DEFAULT_PAGE_STYLE']))
+    context['page_styles'] = page_styles
 
     resp = make_response(render_template(template_name_or_list, **context))
     if selected_style:

incorporealcms/pages.py

@@ -4,9 +4,11 @@ import logging
 import os
 import re
 
-from flask import Blueprint, Markup, abort
+from flask import Blueprint, abort
 from flask import current_app as app
 from flask import redirect, request, send_from_directory
+from markupsafe import Markup
+from werkzeug.security import safe_join
 
 from incorporealcms.lib import get_meta_str, init_md, render
 
@@ -101,15 +103,16 @@ def request_path_to_instance_resource_path(path):
     """
     # check if the path is allowed
     base_dir = os.path.realpath(f'{app.instance_path}/pages/')
-    verbatim_path = os.path.abspath(os.path.join(base_dir, path))
+    safe_path = safe_join(base_dir, path)
-    resolved_path = os.path.realpath(verbatim_path)
-    logger.debug("base_dir '%s', constructed resolved_path '%s' for path '%s'", base_dir, resolved_path, path)
-
     # bail if the requested real path isn't inside the base directory
-    if base_dir != os.path.commonpath((base_dir, resolved_path)):
+    if not safe_path:
         logger.warning("client tried to request a path '%s' outside of the base_dir!", path)
         raise PermissionError
 
+    verbatim_path = os.path.abspath(safe_path)
+    resolved_path = os.path.realpath(verbatim_path)
+    logger.debug("base_dir '%s', constructed resolved_path '%s' for path '%s'", base_dir, resolved_path, path)
+
     # see if we have a real file or if we should infer markdown rendering
     if os.path.exists(resolved_path):
         # if this is a file-like request but actually a directory, redirect the user
@@ -131,7 +134,7 @@ def request_path_to_instance_resource_path(path):
         return resolved_path.replace(f'{app.instance_path}{os.path.sep}', ''), 'file'
 
     # if we're here, this isn't direct file access, so try markdown inference
-    verbatim_path = os.path.abspath(os.path.join(base_dir, f'{path}.md'))
+    verbatim_path = f'{safe_path}.md'
     resolved_path = os.path.realpath(verbatim_path)
 
     # does the final file actually exist?

incorporealcms/static.py

@@ -0,0 +1,14 @@
+"""Serve static files from the instance directory."""
+import os
+
+from flask import Blueprint
+from flask import current_app as app
+from flask import send_from_directory
+
+bp = Blueprint('static', __name__, url_prefix='/custom-static')
+
+
+@bp.route('/<path:name>')
+def serve_instance_static_file(name):
+    """Serve a static file from the instance directory, used for customization."""
+    return send_from_directory(os.path.join(app.instance_path, 'custom-static'), name)

incorporealcms/static/css/base.css

@@ -28,7 +28,8 @@ body {
 
 a {
     font-weight: bold;
-    text-decoration: none;
+    text-decoration-line: underline;
+    text-decoration-thickness: 1px;
 }
 
 div.header {
@@ -39,10 +40,6 @@ div.header {
     padding-bottom: 0;
 }
 
-div.header a {
-    border-bottom: none;
-}
-
 div.content {
     font-size: 11pt;
     padding: 0 1rem;
@@ -170,6 +167,5 @@ figcaption {
 }
 
 .footnote-ref:link, .footnote-ref:visited, .footnote-ref:hover, .footnote-ref:active {
-    border-bottom: none;
     font-weight: normal;
 }

incorporealcms/static/css/dark.css

@@ -15,22 +15,14 @@ h1, h2, h3, h4, h5, h6 {
 
 p a, ul a, ol a {
     color: #DDD;
-    border-bottom: 1px solid #DDD;
 }
 
-p a:hover, ul a:hover, ol a:hover {
+footer a {
-    color: #B31D15;
-    border-bottom: 1px solid #B31D15;
-}
-
-footer a, ul a, ol a {
     color: #999;
-    border-bottom: 1px solid #999;
 }
 
-footer a:hover, ul a:hover, ol a:hover {
+p a:hover, ul a:hover, ol a:hover, footer a:hover {
-    color: #811610;
+    color: #B31D15;
-    border-bottom: 1px solid #811610;
 }
 
 div.site-wrap {
@@ -39,10 +31,7 @@ div.site-wrap {
 
 div.header, div.header a {
     color: #555;
-}
+    text-decoration: none;
-
-div.header a:hover, div.header a:active {
-    border-bottom: 1px solid #555;
 }
 
 table, th, td {

incorporealcms/static/css/light.css

@@ -15,22 +15,14 @@ h1, h2, h3, h4, h5, h6 {
 
 p a, ul a, ol a {
     color: #222;
-    border-bottom: 1px solid #222;
 }
 
-p a:hover, ul a:hover, ol a:hover {
+footer a {
-    color: #811610;
-    border-bottom: 1px solid #811610;
-}
-
-footer a, ul a, ol a {
     color: #999;
-    border-bottom: 1px solid #999;
 }
 
-footer a:hover, ul a:hover, ol a:hover {
+p a:hover, ul a:hover, ol a:hover, footer a:hover {
     color: #811610;
-    border-bottom: 1px solid #811610;
 }
 
 div.site-wrap {
@@ -39,10 +31,7 @@ div.site-wrap {
 
 div.header, div.header a {
     color: #AAA;
-}
+    text-decoration: none;
-
-div.header a:hover, div.header a:active {
-    border-bottom: 1px solid #AAA;
 }
 
 table, th, td {

incorporealcms/templates/base.html

@@ -7,7 +7,7 @@
 <meta property="og:url" content="{{ base_url }}">
 <meta name="twitter:card" content="summary_large_image">
 <meta name="viewport" content="width=device-width, initial-scale=1">
-<link rel="stylesheet" href="{{ url_for('static', filename=user_style) }}">
+<link rel="stylesheet" href="{{ user_style }}">
 <link rel="icon" href="{% if config.FAVICON %}{{ config.FAVICON }}{% else %}{{ url_for('static', filename='img/favicon.png') }}{% endif %}">
 
 <div {% block site_class %}class="site-wrap site-wrap-normal-width"{% endblock %}>
@@ -20,9 +20,9 @@
             {% endfor %}
         </div>
         <div class="styles">
-            <a href="?style=dark">[dark]</a>
+            {% for style in page_styles %}
-            <a href="?style=light">[light]</a>
+            <a href="?style={{ style }}">[{{ style }}]</a>
-            <a href="?style=plain">[plain]</a>
+            {% endfor %}
         </div>
     </div>
     {% endblock %}

requirements/requirements-dev.in

@@ -21,6 +21,6 @@ safety                      # check requirements file for issues
 
 # maintenance utilities and tox
 pip-tools                   # pip-compile
-tox                         # CI stuff
+tox<4                       # CI stuff, pinned for now to avoid packaging conflict w/safety
 tox-wheel                   # build wheels in tox
 versioneer                  # automatic version numbering

requirements/requirements-dev.txt

@@ -1,37 +1,37 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
-# To update, run:
+# by the following command:
 #
 #    pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
 #
-attrs==22.1.0
+bandit==1.7.5
-    # via pytest
-bandit==1.7.4
     # via -r requirements/requirements-dev.in
-build==0.8.0
+blinker==1.6.3
+    # via flask
+build==1.0.3
     # via pip-tools
-certifi==2022.9.14
+certifi==2023.7.22
     # via requests
-charset-normalizer==2.1.1
+charset-normalizer==3.3.0
     # via requests
-click==8.1.3
+click==8.1.7
     # via
     #   flask
     #   pip-tools
     #   safety
-coverage[toml]==6.4.4
+coverage[toml]==7.3.2
     # via pytest-cov
-distlib==0.3.6
+distlib==0.3.7
     # via virtualenv
-dlint==0.13.0
+dlint==0.14.1
     # via -r requirements/requirements-dev.in
-dparse==0.6.0
+dparse==0.6.3
     # via safety
-filelock==3.8.0
+filelock==3.12.4
     # via
     #   tox
     #   virtualenv
-flake8==5.0.4
+flake8==6.1.0
     # via
     #   -r requirements/requirements-dev.in
     #   dlint
@@ -42,44 +42,48 @@ flake8==5.0.4
     #   flake8-mutable
 flake8-blind-except==0.2.1
     # via -r requirements/requirements-dev.in
-flake8-builtins==1.5.3
+flake8-builtins==2.1.0
     # via -r requirements/requirements-dev.in
-flake8-docstrings==1.6.0
+flake8-docstrings==1.7.0
     # via -r requirements/requirements-dev.in
-flake8-executable==2.1.1
+flake8-executable==2.1.3
     # via -r requirements/requirements-dev.in
 flake8-fixme==1.1.1
     # via -r requirements/requirements-dev.in
-flake8-isort==4.2.0
+flake8-isort==6.1.0
     # via -r requirements/requirements-dev.in
-flake8-logging-format==0.7.5
+flake8-logging-format==0.9.0
     # via -r requirements/requirements-dev.in
 flake8-mutable==1.2.0
     # via -r requirements/requirements-dev.in
-flask==2.2.2
+flask==3.0.0
     # via -r requirements/requirements.in
-gitdb==4.0.9
+gitdb==4.0.10
     # via gitpython
-gitpython==3.1.27
+gitpython==3.1.37
     # via bandit
 idna==3.4
     # via requests
-iniconfig==1.1.1
+iniconfig==2.0.0
     # via pytest
-isort==5.10.1
+isort==5.12.0
     # via flake8-isort
 itsdangerous==2.1.2
     # via flask
 jinja2==3.1.2
     # via flask
-markdown==3.4.1
+markdown==3.5
     # via -r requirements/requirements.in
-markupsafe==2.1.1
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
 mccabe==0.7.0
     # via flake8
+mdurl==0.1.2
+    # via markdown-it-py
 packaging==21.3
     # via
     #   build
@@ -87,82 +91,75 @@ packaging==21.3
     #   pytest
     #   safety
     #   tox
-pbr==5.10.0
+pbr==5.11.1
     # via stevedore
-pep517==0.13.0
+pip-tools==7.3.0
-    # via build
-pip-tools==6.8.0
     # via -r requirements/requirements-dev.in
-platformdirs==2.5.2
+platformdirs==3.11.0
     # via virtualenv
-pluggy==1.0.0
+pluggy==1.3.0
     # via
     #   pytest
     #   tox
 py==1.11.0
-    # via
+    # via tox
-    #   pytest
+pycodestyle==2.11.0
-    #   tox
-pycodestyle==2.9.1
     # via flake8
-pydocstyle==6.1.1
+pydocstyle==6.3.0
     # via flake8-docstrings
 pydot==1.4.2
     # via -r requirements/requirements-dev.in
-pyflakes==2.5.0
+pyflakes==3.1.0
     # via flake8
-pyparsing==3.0.9
+pygments==2.16.1
+    # via rich
+pyparsing==3.1.1
     # via
     #   packaging
     #   pydot
-pytest==7.1.3
+pyproject-hooks==1.0.0
+    # via build
+pytest==7.4.2
     # via
     #   -r requirements/requirements-dev.in
     #   pytest-cov
-pytest-cov==3.0.0
+pytest-cov==4.1.0
     # via -r requirements/requirements-dev.in
-pyyaml==6.0
+pyyaml==6.0.1
     # via bandit
-requests==2.28.1
+requests==2.31.0
     # via safety
-ruamel-yaml==0.17.21
+rich==13.6.0
+    # via bandit
+ruamel-yaml==0.17.35
     # via safety
-ruamel-yaml-clib==0.2.6
+ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-safety==2.1.1
+safety==2.3.5
     # via -r requirements/requirements-dev.in
 six==1.16.0
     # via tox
-smmap==5.0.0
+smmap==5.0.1
     # via gitdb
 snowballstemmer==2.2.0
     # via pydocstyle
-stevedore==4.0.0
+stevedore==5.1.0
     # via bandit
-toml==0.10.2
+tox==3.28.0
-    # via dparse
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   pep517
-    #   pytest
-    #   tox
-tox==3.26.0
     # via
     #   -r requirements/requirements-dev.in
     #   tox-wheel
-tox-wheel==0.7.0
+tox-wheel==1.0.0
     # via -r requirements/requirements-dev.in
-urllib3==1.26.12
+urllib3==2.0.6
     # via requests
-versioneer==0.26
+versioneer==0.29
     # via -r requirements/requirements-dev.in
-virtualenv==20.16.5
+virtualenv==20.24.5
     # via tox
-werkzeug==2.2.2
+werkzeug==3.0.0
     # via flask
-wheel==0.37.1
+wheel==0.41.2
     # via
     #   pip-tools
     #   tox-wheel

requirements/requirements.txt

@@ -1,22 +1,24 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
-# To update, run:
+# by the following command:
 #
 #    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
 #
-click==8.1.3
+blinker==1.6.3
     # via flask
-flask==2.2.2
+click==8.1.7
+    # via flask
+flask==3.0.0
     # via -r requirements/requirements.in
 itsdangerous==2.1.2
     # via flask
 jinja2==3.1.2
     # via flask
-markdown==3.4.1
+markdown==3.5
     # via -r requirements/requirements.in
-markupsafe==2.1.1
+markupsafe==2.1.3
     # via
     #   jinja2
     #   werkzeug
-werkzeug==2.2.2
+werkzeug==3.0.0
     # via flask

tests/functional_tests.py

@@ -184,21 +184,21 @@ def test_that_dir_request_does_not_redirect(client):
 def test_setting_selected_style_includes_cookie(client):
     """Test that a request with style=foo sets the cookie and renders appropriately."""
     response = client.get('/')
-    style_cookie = next((cookie for cookie in client.cookie_jar if cookie.name == 'user-style'), None)
+    style_cookie = client.get_cookie('user-style')
     assert style_cookie is None
 
     response = client.get('/?style=light')
-    style_cookie = next((cookie for cookie in client.cookie_jar if cookie.name == 'user-style'), None)
+    style_cookie = client.get_cookie('user-style')
     assert response.status_code == 200
-    assert b'light.css' in response.data
+    assert b'/static/css/light.css' in response.data
-    assert b'dark.css' not in response.data
+    assert b'/static/css/dark.css' not in response.data
     assert style_cookie.value == 'light'
 
     response = client.get('/?style=dark')
-    style_cookie = next((cookie for cookie in client.cookie_jar if cookie.name == 'user-style'), None)
+    style_cookie = client.get_cookie('user-style')
     assert response.status_code == 200
-    assert b'dark.css' in response.data
+    assert b'/static/css/dark.css' in response.data
-    assert b'light.css' not in response.data
+    assert b'/static/css/light.css' not in response.data
     assert style_cookie.value == 'dark'
 
 
@@ -218,3 +218,23 @@ def test_extra_footer_per_page(client):
     assert b'<div class="extra-footer">' not in response.data
     response = client.get('/index-but-with-footer')
     assert b'<div class="extra-footer"><i>ooo <a href="a">a</a></i>' in response.data
+
+
+def test_serving_static_files(client):
+    """Test the usage of send_from_directory to serve extra static files."""
+    response = client.get('/custom-static/css/warm.css')
+    assert response.status_code == 200
+
+    # can't serve directories, just files
+    response = client.get('/custom-static/')
+    assert response.status_code == 404
+    response = client.get('/custom-static/css/')
+    assert response.status_code == 404
+    response = client.get('/custom-static/css')
+    assert response.status_code == 404
+
+    # can't serve files that don't exist or bad paths
+    response = client.get('/custom-static/css/cold.css')
+    assert response.status_code == 404
+    response = client.get('/custom-static/css/../../unreachable.md')
+    assert response.status_code == 404

tests/instance/custom-static/css/warm.css

@@ -0,0 +1,3 @@
+* {
+    color: red;
+}

tests/test_pages.py

@@ -1,10 +1,15 @@
 """Unit test helper methods."""
+import os
+
 import pytest
 from werkzeug.http import dump_cookie
 
+from incorporealcms import create_app
 from incorporealcms.pages import (generate_parent_navs, instance_resource_path_to_request_path, render,
                                   request_path_to_breadcrumb_display, request_path_to_instance_resource_path)
 
+HERE = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_generate_page_navs_index(app):
     """Test that the index page has navs to the root (itself)."""
@@ -49,22 +54,74 @@ def test_render_with_user_dark_theme(app):
     """Test that a request with the dark theme selected renders the dark theme."""
     cookie = dump_cookie("user-style", 'dark')
     with app.test_request_context(headers={'COOKIE': cookie}):
-        assert b'dark.css' in render('base.html').data
+        assert b'/static/css/dark.css' in render('base.html').data
-        assert b'light.css' not in render('base.html').data
+        assert b'/static/css/light.css' not in render('base.html').data
 
 
 def test_render_with_user_light_theme(app):
     """Test that a request with the light theme selected renders the light theme."""
     with app.test_request_context():
-        assert b'light.css' in render('base.html').data
+        assert b'/static/css/light.css' in render('base.html').data
-        assert b'dark.css' not in render('base.html').data
+        assert b'/static/css/dark.css' not in render('base.html').data
 
 
 def test_render_with_no_user_theme(app):
     """Test that a request with no theme set renders the light theme."""
     with app.test_request_context():
-        assert b'light.css' in render('base.html').data
+        assert b'/static/css/light.css' in render('base.html').data
-        assert b'dark.css' not in render('base.html').data
+        assert b'/static/css/dark.css' not in render('base.html').data
+
+
+def test_render_with_theme_defaults_affects_html(app):
+    """Test that the base themes are all that's presented in the HTML."""
+    # test we can remove stuff from the default
+    with app.test_request_context():
+        assert b'?style=light' in render('base.html').data
+        assert b'?style=dark' in render('base.html').data
+        assert b'?style=plain' in render('base.html').data
+
+
+def test_render_with_theme_overrides_affects_html(app):
+    """Test that the overridden themes are presented in the HTML."""
+    # test we can remove stuff from the default
+    restyled_app = create_app(instance_path=os.path.join(HERE, 'instance'),
+                              test_config={'PAGE_STYLES': {'light': '/static/css/light.css'}})
+    with restyled_app.test_request_context():
+        assert b'?style=light' in render('base.html').data
+        assert b'?style=dark' not in render('base.html').data
+        assert b'?style=plain' not in render('base.html').data
+
+    # test that we can add new stuff too/instead
+    restyled_app = create_app(instance_path=os.path.join(HERE, 'instance'),
+                              test_config={'PAGE_STYLES': {'cool': '/static/css/cool.css',
+                                                           'warm': '/static/css/warm.css'},
+                                           'DEFAULT_PAGE_STYLE': 'warm'})
+    with restyled_app.test_request_context():
+        assert b'?style=cool' in render('base.html').data
+        assert b'?style=warm' in render('base.html').data
+
+
+def test_render_with_theme_overrides(app):
+    """Test that the loaded themes can be overridden from the default."""
+    cookie = dump_cookie("user-style", 'cool')
+    restyled_app = create_app(instance_path=os.path.join(HERE, 'instance'),
+                              test_config={'PAGE_STYLES': {'cool': '/static/css/cool.css',
+                                                           'warm': '/static/css/warm.css'}})
+    with restyled_app.test_request_context(headers={'COOKIE': cookie}):
+        assert b'/static/css/cool.css' in render('base.html').data
+        assert b'/static/css/warm.css' not in render('base.html').data
+
+
+def test_render_with_theme_overrides_not_found_is_default(app):
+    """Test that theme overrides work, and if a requested theme doesn't exist, the default is loaded."""
+    cookie = dump_cookie("user-style", 'nonexistent')
+    restyled_app = create_app(instance_path=os.path.join(HERE, 'instance'),
+                              test_config={'PAGE_STYLES': {'cool': '/static/css/cool.css',
+                                                           'warm': '/static/css/warm.css'},
+                                           'DEFAULT_PAGE_STYLE': 'warm'})
+    with restyled_app.test_request_context(headers={'COOKIE': cookie}):
+        assert b'/static/css/warm.css' in render('base.html').data
+        assert b'/static/css/nonexistent.css' not in render('base.html').data
 
 
 def test_request_path_to_instance_resource_path(app):

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = begin,py38,py39,py310,coverage,security,lint,bundle
+envlist = begin,py38,py39,py310,py311,coverage,security,lint,bundle
 
 [testenv]
 # build a wheel and test it
@@ -46,6 +46,11 @@ commands =
 commands =
     pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch
 
+[testenv:py311]
+# run pytest with coverage
+commands =
+    pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch
+
 [testenv:coverage]
 # report on coverage runs from above
 skip_install = true
@@ -56,9 +61,10 @@ commands =
 # run security checks
 #
 # again it seems the most valuable here to run against the packaged code
+# 51457 is nearly a red herring that I'm stuck with because tox is pinned, try removing occasionally
 commands =
     bandit {envsitepackagesdir}/incorporealcms/ -r
-    safety check -r requirements/requirements-dev.txt
+    safety check -r requirements/requirements-dev.txt -i 51457
 
 [testenv:lint]
 # run style checks
@@ -90,6 +96,7 @@ exclude =
     versioneer.py
     _version.py
     instance/
+    venv/
 extend-ignore = T101
 max-complexity = 10
 max-line-length = 120