requirements bump

in the end, a lot of this doesn't really matter, and right now, I
slightly prefer the site with a touch bigger font, so let's just let the
default do its thing

incorporealcms/mdx/__init__.py

@@ -0,0 +1 @@
+"""Markdown extensions."""

incorporealcms/mdx/pydot.py

@@ -0,0 +1,52 @@
+"""Serve dot diagrams inline."""
+import base64
+import logging
+import re
+
+import markdown
+import pydot
+
+logger = logging.getLogger(__name__)
+
+
+class InlinePydot(markdown.Extension):
+    """Wrap the markdown prepcoressor."""
+
+    def extendMarkdown(self, md, md_globals):
+        """Add InlinePydotPreprocessor to the Markdown instance."""
+        md.registerExtension(self)
+        md.preprocessors.add('dot_block', InlinePydotPreprocessor(md), '_begin')
+
+
+class InlinePydotPreprocessor(markdown.preprocessors.Preprocessor):
+    """Identify dot codeblocks and run them through pydot."""
+
+    BLOCK_RE = re.compile(r'~~~pydot:(?P<filename>[^\s]+)\n(?P<content>.*?)~~~', re.DOTALL)
+
+    def run(self, lines):
+        """Match and generate diagrams from dot code blocks."""
+        text = '\n'.join(lines)
+        for match in self.BLOCK_RE.finditer(text):
+            filename = match.group(1)
+            dot_string = match.group(2)
+
+            # use pydot to turn the text into pydot
+            graphs = pydot.graph_from_dot_data(dot_string)
+            if not graphs:
+                logger.debug("some kind of issue with parsed 'dot' %s", dot_string)
+                raise ValueError("error parsing dot text!")
+
+            # encode the image and provide as an inline image in markdown
+            encoded_image = base64.b64encode(graphs[0].create_png()).decode('ascii')
+            data_path = f'data:image/png;base64,{encoded_image}'
+            inline_image = f'![{filename}]({data_path})'
+
+            # replace the image in the output markdown
+            text = f'{text[:match.start()]}\n{inline_image}\n{text[match.end():]}'
+
+        return text.split('\n')
+
+
+def makeExtension(*args, **kwargs):
+    """Provide the extension to the markdown extension loader."""
+    return InlinePydot(*args, **kwargs)

incorporealcms/pages.py

@@ -49,15 +49,25 @@ def display_page(path):
 def handle_markdown_file_path(resolved_path):
     """Given a location on disk, attempt to open it and render the markdown within."""
     try:
+        logger.debug("opening resolved path '%s'", resolved_path)
         with app.open_instance_resource(resolved_path, 'r') as entry_file:
             mtime = datetime.datetime.fromtimestamp(os.path.getmtime(entry_file.name), get_localzone())
             entry = entry_file.read()
+        logger.debug("resolved path '%s' read", resolved_path)
     except OSError:
         logger.exception("resolved path '%s' could not be opened!", resolved_path)
         abort(500)
     else:
-        md = init_md()
+        try:
-        content = Markup(md.convert(entry))
+            md = init_md()
+            content = Markup(md.convert(entry))
+        except ValueError:
+            logger.exception("error parsing/rendering markdown!")
+            abort(500)
+        except TypeError:
+            logger.exception("error loading/rendering markdown!")
+            abort(500)
+
         logger.debug("file metadata: %s", md.Meta)
 
         parent_navs = generate_parent_navs(resolved_path)

incorporealcms/static/css/base.css

@@ -26,30 +26,6 @@ body {
     margin-right: auto;
 }
 
-h1 {
-    font-size: 2em;
-}
-
-h2 {
-    font-size: 1.5em;
-}
-
-h3 {
-    font-size: 1.25em;
-}
-
-h4 {
-    font-size: 1.17em;
-}
-
-h5 {
-    font-size: 1em;
-}
-
-h6 {
-    font-size: .83em;
-}
-
 a:link {
     font-weight: bold;
     text-decoration: none;
@@ -82,8 +58,6 @@ div.header a {
 }
 
 div.content {
-    font-size: 11pt;
-
     padding: 0 1em;
     line-height: 1.5em;
 }

requirements/requirements-dev.in

@@ -1,6 +1,7 @@
 -r requirements.in
 
 # testing runner, test reporting, packages used during testing (e.g. requests-mock), etc.
+pydot
 pytest
 pytest-cov
 
@@ -16,6 +17,7 @@ flake8-fixme
 flake8-isort
 flake8-logging-format
 flake8-mutable
+safety                      # check requirements file for issues
 
 # maintenance utilities and tox
 pip-tools                   # pip-compile

requirements/requirements-dev.txt

@@ -1,31 +1,49 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
 #
-appdirs==1.4.4
+attrs==21.2.0
-    # via virtualenv
-attrs==20.3.0
     # via pytest
+backports.entry-points-selectable==1.1.0
+    # via virtualenv
+backports.zoneinfo==0.2.1
+    # via tzlocal
 bandit==1.6.2
     # via -r requirements/requirements-dev.in
-bleach==3.3.0
+bleach==4.1.0
     # via mdx-linkify
-click==7.1.2
+certifi==2021.5.30
+    # via requests
+charset-normalizer==2.0.6
+    # via requests
+click==8.0.1
     # via
     #   flask
     #   pip-tools
-coverage==5.5
+    #   safety
+coverage[toml]==6.0.1
     # via pytest-cov
-distlib==0.3.1
+distlib==0.3.3
     # via virtualenv
 dlint==0.11.0
     # via -r requirements/requirements-dev.in
-filelock==3.0.12
+dparse==0.5.1
+    # via safety
+filelock==3.3.0
     # via
     #   tox
     #   virtualenv
+flake8==3.9.2
+    # via
+    #   -r requirements/requirements-dev.in
+    #   dlint
+    #   flake8-builtins
+    #   flake8-docstrings
+    #   flake8-executable
+    #   flake8-isort
+    #   flake8-mutable
 flake8-blind-except==0.2.0
     # via -r requirements/requirements-dev.in
 flake8-builtins==1.5.3
@@ -42,51 +60,48 @@ flake8-logging-format==0.6.0
     # via -r requirements/requirements-dev.in
 flake8-mutable==1.2.0
     # via -r requirements/requirements-dev.in
-flake8==3.9.1
+flask==2.0.2
-    # via
-    #   -r requirements/requirements-dev.in
-    #   dlint
-    #   flake8-builtins
-    #   flake8-docstrings
-    #   flake8-executable
-    #   flake8-isort
-    #   flake8-mutable
-flask==1.1.2
     # via -r requirements/requirements.in
 gitdb==4.0.7
     # via gitpython
-gitpython==3.1.14
+gitpython==3.1.24
     # via bandit
+idna==3.2
+    # via requests
 iniconfig==1.1.1
     # via pytest
-isort==5.8.0
+isort==5.9.3
     # via flake8-isort
-itsdangerous==1.1.0
+itsdangerous==2.0.1
     # via flask
-jinja2==2.11.3
+jinja2==3.0.2
     # via flask
 markdown==3.3.4
     # via
     #   -r requirements/requirements.in
     #   mdx-linkify
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
 mccabe==0.6.1
     # via flake8
 mdx-linkify==2.1
     # via -r requirements/requirements.in
-packaging==20.9
+packaging==21.0
     # via
     #   bleach
+    #   dparse
     #   pytest
+    #   safety
     #   tox
-pbr==5.5.1
+pbr==5.6.0
     # via stevedore
-pep517==0.10.0
+pep517==0.11.0
     # via pip-tools
-pip-tools==6.1.0
+pip-tools==6.3.0
     # via -r requirements/requirements-dev.in
-pluggy==0.13.1
+platformdirs==2.4.0
+    # via virtualenv
+pluggy==1.0.0
     # via
     #   pytest
     #   tox
@@ -96,23 +111,31 @@ py==1.10.0
     #   tox
 pycodestyle==2.7.0
     # via flake8
-pydocstyle==6.0.0
+pydocstyle==6.1.1
     # via flake8-docstrings
+pydot==1.4.2
+    # via -r requirements/requirements-dev.in
 pyflakes==2.3.1
     # via flake8
 pyparsing==2.4.7
-    # via packaging
+    # via
-pytest-cov==2.11.1
+    #   packaging
-    # via -r requirements/requirements-dev.in
+    #   pydot
-pytest==6.2.3
+pytest==6.2.5
     # via
     #   -r requirements/requirements-dev.in
     #   pytest-cov
-pytz==2021.1
+pytest-cov==3.0.0
-    # via tzlocal
+    # via -r requirements/requirements-dev.in
 pyyaml==5.4.1
-    # via bandit
+    # via
-six==1.15.0
+    #   bandit
+    #   dparse
+requests==2.26.0
+    # via safety
+safety==1.10.3
+    # via -r requirements/requirements-dev.in
+six==1.16.0
     # via
     #   bandit
     #   bleach
@@ -122,33 +145,43 @@ smmap==4.0.0
     # via gitdb
 snowballstemmer==2.1.0
     # via pydocstyle
-stevedore==3.3.0
+stevedore==3.4.0
     # via bandit
-testfixtures==6.17.1
+testfixtures==6.18.3
     # via flake8-isort
 toml==0.10.2
     # via
-    #   pep517
+    #   dparse
     #   pytest
     #   tox
-tox-wheel==0.6.0
+tomli==1.2.1
-    # via -r requirements/requirements-dev.in
+    # via
-tox==3.23.0
+    #   coverage
+    #   pep517
+tox==3.24.4
     # via
     #   -r requirements/requirements-dev.in
     #   tox-wheel
-tzlocal==2.1
+tox-wheel==0.6.0
-    # via -r requirements/requirements.in
-versioneer==0.19
     # via -r requirements/requirements-dev.in
-virtualenv==20.4.3
+typing-extensions==3.10.0.2
+    # via gitpython
+tzlocal==3.0
+    # via -r requirements/requirements.in
+urllib3==1.26.7
+    # via requests
+versioneer==0.20
+    # via -r requirements/requirements-dev.in
+virtualenv==20.8.1
     # via tox
 webencodings==0.5.1
     # via bleach
-werkzeug==1.0.1
+werkzeug==2.0.2
     # via flask
-wheel==0.36.2
+wheel==0.37.0
-    # via tox-wheel
+    # via
+    #   pip-tools
+    #   tox-wheel
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/requirements.txt

@@ -1,38 +1,38 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
 #
-bleach==3.3.0
+backports.zoneinfo==0.2.1
+    # via tzlocal
+bleach==4.1.0
     # via mdx-linkify
-click==7.1.2
+click==8.0.1
     # via flask
-flask==1.1.2
+flask==2.0.2
     # via -r requirements/requirements.in
-itsdangerous==1.1.0
+itsdangerous==2.0.1
     # via flask
-jinja2==2.11.3
+jinja2==3.0.2
     # via flask
 markdown==3.3.4
     # via
     #   -r requirements/requirements.in
     #   mdx-linkify
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
 mdx-linkify==2.1
     # via -r requirements/requirements.in
-packaging==20.9
+packaging==21.0
     # via bleach
 pyparsing==2.4.7
     # via packaging
-pytz==2021.1
+six==1.16.0
-    # via tzlocal
-six==1.15.0
     # via bleach
-tzlocal==2.1
+tzlocal==3.0
     # via -r requirements/requirements.in
 webencodings==0.5.1
     # via bleach
-werkzeug==1.0.1
+werkzeug==2.0.2
     # via flask

setup.py

@@ -18,7 +18,7 @@ setup(
     name='incorporeal-cms',
     description='Flask project for running https://suou.net (and eventually others).',
     url='https://git.incorporeal.org/bss/incorporeal-cms',
-    license='GPL3',
+    license='GPLv2+',
     author='Brian S. Stephan',
     author_email='bss@incorporeal.org',
     version=versioneer.get_version(),
@@ -27,4 +27,7 @@ setup(
     include_package_data=True,
     zip_safe=False,
     install_requires=extract_requires(),
+    extras_require={
+        'dot': ['pydot'],
+    },
 )

tests/functional_markdown_tests.py

@@ -0,0 +1,39 @@
+"""Test graphviz functionality."""
+import os
+
+from incorporealcms import create_app
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+def app_with_pydot():
+    """Create the test app, including the pydot extension."""
+    return create_app(instance_path=os.path.join(HERE, 'instance'),
+                      test_config={'MARKDOWN_EXTENSIONS': ['incorporealcms.mdx.pydot']})
+
+
+def test_functional_initialization():
+    """Test initialization with the graphviz config."""
+    app = app_with_pydot()
+    assert app is not None
+
+
+def test_graphviz_is_rendered():
+    """Initialize the app with the graphviz extension and ensure it does something."""
+    app = app_with_pydot()
+    client = app.test_client()
+
+    response = client.get('/test-graphviz')
+    assert response.status_code == 200
+    assert b'~~~pydot' not in response.data
+    assert b'data:image/png;base64' in response.data
+
+
+def test_invalid_graphviz_is_not_rendered():
+    """Check that invalid graphviz doesn't blow things up."""
+    app = app_with_pydot()
+    client = app.test_client()
+
+    response = client.get('/test-invalid-graphviz')
+    assert response.status_code == 500
+    assert b'INTERNAL SERVER ERROR' in response.data

tests/instance/pages/test-graphviz.md

@@ -0,0 +1,12 @@
+# test
+
+test
+~~~pydot:attack-plan
+digraph G {
+    rankdir=LR
+    Earth
+    Mars
+    Earth -> Mars
+}
+~~~
+more test

tests/instance/pages/test-invalid-graphviz.md

@@ -0,0 +1,11 @@
+# test
+
+test
+~~~pydot:attack-plan
+    rankdir=LR
+    Earth
+    Mars
+    Earth -> Mars
+}
+~~~
+more test

tests/test_factory.py

@@ -68,3 +68,12 @@ def test_favicon_override():
     response = client.get('/no-title')
     assert response.status_code == 200
     assert b'<link rel="icon" href="/media/foo.png">' in response.data
+
+
+def test_misconfigured_markdown_extensions():
+    """Test that a misconfigured markdown extensions leads to a 500 at render time."""
+    instance_path = os.path.join(HERE, 'instance')
+    app = create_app(instance_path=instance_path, test_config={'MARKDOWN_EXTENSIONS': 'WRONG'})
+    client = app.test_client()
+    response = client.get('/no-title')
+    assert response.status_code == 500

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = begin,py37,py38,coverage,security,lint,bundle
+envlist = begin,py37,py38,py39,coverage,security,lint,bundle
 
 [testenv]
 # build a wheel and test it
@@ -41,6 +41,11 @@ commands =
 commands =
     pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch
 
+[testenv:py39]
+# run pytest with coverage
+commands =
+    pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch
+
 [testenv:coverage]
 # report on coverage runs from above
 skip_install = true
@@ -53,6 +58,7 @@ commands =
 # again it seems the most valuable here to run against the packaged code
 commands =
     bandit {envsitepackagesdir}/incorporealcms/ -r
+    safety check -r requirements/requirements-dev.txt
 
 [testenv:lint]
 # run style checks