implement a rudimentary Atom/RSS feed module

this provides a somewhat unconfigurable (at the moment) feed module which provides Atom and RSS feeds. entries are determined by symlinks to content pages, because my core CMS usage is still more general and not blog-like. the symlinks allow for arbitrarily adding entries as I see fit. this also moves core Markdown parser stuff to the library module, since that's used by the feed as well as normal pages Signed-off-by: Brian S. Stephan <bss@incorporeal.org>
deduplicate TITLE_SUFFIX from new DOMAIN_NAME
2023-12-30 14:55:02 -06:00 · 2023-12-30 11:55:01 -06:00 · 2023-12-30 11:13:08 -06:00 · 2023-12-30 11:12:59 -06:00 · 2023-12-30 10:33:28 -06:00
20 changed files with 297 additions and 2542 deletions
--- a/.gitignore
+++ b/.gitignore
@ -117,3 +117,6 @@ dmypy.json

 # vim stuff
 *.swp
+
+# autogenerated versioning
+_version.py
--- a/incorporealcms/init.py
+++ b/incorporealcms/init.py
@ -9,11 +9,6 @@ from logging.config import dictConfig

 from flask import Flask, request

-from ._version import get_versions
-
-__version__ = get_versions()['version']
-del get_versions
-

 def create_app(instance_path=None, test_config=None):
    """Create the Flask app, with allowances for customizing path and test settings."""
@ -44,7 +39,8 @@ def create_app(instance_path=None, test_config=None):
        logger.info("RESPONSE: %s %s: %s", request.method, request.path, response.status)
        return response

-    from . import error_pages, pages, static
+    from . import error_pages, feed, pages, static
+    app.register_blueprint(feed.bp)
    app.register_blueprint(pages.bp)
    app.register_blueprint(static.bp)
    app.register_error_handler(400, error_pages.bad_request)
--- a/incorporealcms/_version.py
+++ b/incorporealcms/_version.py
@ -1,520 +0,0 @@
-
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
-
-"""Git implementation of _version.py."""
-
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
-    """Get the keywords needed to look up the version information."""
-    # these strings will be replaced by git during git-archive.
-    # setup.py/versioneer.py will grep for the variable names, so they must
-    # each be defined on a line of their own. _version.py will just call
-    # get_keywords().
-    git_refnames = "$Format:%d$"
-    git_full = "$Format:%H$"
-    git_date = "$Format:%ci$"
-    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
-    return keywords
-
-
-class VersioneerConfig:
-    """Container for Versioneer configuration parameters."""
-
-
-def get_config():
-    """Create, populate and return the VersioneerConfig() object."""
-    # these strings are filled in when 'setup.py versioneer' creates
-    # _version.py
-    cfg = VersioneerConfig()
-    cfg.VCS = "git"
-    cfg.style = "pep440-post"
-    cfg.tag_prefix = "v"
-    cfg.parentdir_prefix = "None"
-    cfg.versionfile_source = "incorporealcms/_version.py"
-    cfg.verbose = False
-    return cfg
-
-
-class NotThisMethod(Exception):
-    """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
-    def decorate(f):
-        """Store f in HANDLERS[vcs][method]."""
-        if vcs not in HANDLERS:
-            HANDLERS[vcs] = {}
-        HANDLERS[vcs][method] = f
-        return f
-    return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
-                env=None):
-    """Call the given command(s)."""
-    assert isinstance(commands, list)
-    p = None
-    for c in commands:
-        try:
-            dispcmd = str([c] + args)
-            # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
-                                 stdout=subprocess.PIPE,
-                                 stderr=(subprocess.PIPE if hide_stderr
-                                         else None))
-            break
-        except EnvironmentError:
-            e = sys.exc_info()[1]
-            if e.errno == errno.ENOENT:
-                continue
-            if verbose:
-                print("unable to run %s" % dispcmd)
-                print(e)
-            return None, None
-    else:
-        if verbose:
-            print("unable to find command, tried %s" % (commands,))
-        return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % dispcmd)
-            print("stdout was %s" % stdout)
-        return None, p.returncode
-    return stdout, p.returncode
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
-    """Try to determine the version from the parent directory name.
-
-    Source tarballs conventionally unpack into a directory that includes both
-    the project name and a version string. We will also support searching up
-    two directory levels for an appropriately named parent directory
-    """
-    rootdirs = []
-
-    for i in range(3):
-        dirname = os.path.basename(root)
-        if dirname.startswith(parentdir_prefix):
-            return {"version": dirname[len(parentdir_prefix):],
-                    "full-revisionid": None,
-                    "dirty": False, "error": None, "date": None}
-        else:
-            rootdirs.append(root)
-            root = os.path.dirname(root)  # up a level
-
-    if verbose:
-        print("Tried directories %s but none started with prefix %s" %
-              (str(rootdirs), parentdir_prefix))
-    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
-    """Extract version information from the given file."""
-    # the code embedded in _version.py can just fetch the value of these
-    # keywords. When used from setup.py, we don't want to import _version.py,
-    # so we do it with a regexp instead. This function is not used from
-    # _version.py.
-    keywords = {}
-    try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
-    except EnvironmentError:
-        pass
-    return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
-    """Get version information from git keywords."""
-    if not keywords:
-        raise NotThisMethod("no keywords at all, weird")
-    date = keywords.get("date")
-    if date is not None:
-        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
-        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
-        # -like" string, which we must then edit to make compliant), because
-        # it's been around since git-1.5.3, and it's too difficult to
-        # discover which version we're using, or to work around using an
-        # older one.
-        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-    refnames = keywords["refnames"].strip()
-    if refnames.startswith("$Format"):
-        if verbose:
-            print("keywords are unexpanded, not using")
-        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
-    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
-    TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
-    if not tags:
-        # Either we're using git < 1.8.3, or there really are no tags. We use
-        # a heuristic: assume all version tags have a digit. The old git %d
-        # expansion behaves like git log --decorate=short and strips out the
-        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
-        # between branches and tags. By ignoring refnames without digits, we
-        # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
-        if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
-    if verbose:
-        print("likely tags: %s" % ",".join(sorted(tags)))
-    for ref in sorted(tags):
-        # sorting will prefer e.g. "2.0" over "2.0rc1"
-        if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix):]
-            if verbose:
-                print("picking %s" % r)
-            return {"version": r,
-                    "full-revisionid": keywords["full"].strip(),
-                    "dirty": False, "error": None,
-                    "date": date}
-    # no suitable tags, so version is "0+unknown", but full hex is still there
-    if verbose:
-        print("no suitable tags, using unknown + full revision id")
-    return {"version": "0+unknown",
-            "full-revisionid": keywords["full"].strip(),
-            "dirty": False, "error": "no suitable tags", "date": None}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
-    """Get version from 'git describe' in the root of the source tree.
-
-    This only gets called if the git-archive 'subst' keywords were *not*
-    expanded, and _version.py hasn't already been rewritten with a short
-    version string, meaning we're inside a checked out source tree.
-    """
-    GITS = ["git"]
-    if sys.platform == "win32":
-        GITS = ["git.cmd", "git.exe"]
-
-    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
-                          hide_stderr=True)
-    if rc != 0:
-        if verbose:
-            print("Directory %s not under git control" % root)
-        raise NotThisMethod("'git rev-parse --git-dir' returned error")
-
-    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
-    # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
-                                          "--always", "--long",
-                                          "--match", "%s*" % tag_prefix],
-                                   cwd=root)
-    # --long was added in git-1.5.5
-    if describe_out is None:
-        raise NotThisMethod("'git describe' failed")
-    describe_out = describe_out.strip()
-    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
-    if full_out is None:
-        raise NotThisMethod("'git rev-parse' failed")
-    full_out = full_out.strip()
-
-    pieces = {}
-    pieces["long"] = full_out
-    pieces["short"] = full_out[:7]  # maybe improved later
-    pieces["error"] = None
-
-    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
-    # TAG might have hyphens.
-    git_describe = describe_out
-
-    # look for -dirty suffix
-    dirty = git_describe.endswith("-dirty")
-    pieces["dirty"] = dirty
-    if dirty:
-        git_describe = git_describe[:git_describe.rindex("-dirty")]
-
-    # now we have TAG-NUM-gHEX or HEX
-
-    if "-" in git_describe:
-        # TAG-NUM-gHEX
-        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
-        if not mo:
-            # unparseable. Maybe git-describe is misbehaving?
-            pieces["error"] = ("unable to parse git-describe output: '%s'"
-                               % describe_out)
-            return pieces
-
-        # tag
-        full_tag = mo.group(1)
-        if not full_tag.startswith(tag_prefix):
-            if verbose:
-                fmt = "tag '%s' doesn't start with prefix '%s'"
-                print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
-                               % (full_tag, tag_prefix))
-            return pieces
-        pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
-        # distance: number of commits since tag
-        pieces["distance"] = int(mo.group(2))
-
-        # commit: short hex revision ID
-        pieces["short"] = mo.group(3)
-
-    else:
-        # HEX: no tags
-        pieces["closest-tag"] = None
-        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                    cwd=root)
-        pieces["distance"] = int(count_out)  # total number of commits
-
-    # commit date: see ISO-8601 comment in git_versions_from_keywords()
-    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
-                       cwd=root)[0].strip()
-    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-
-    return pieces
-
-
-def plus_or_dot(pieces):
-    """Return a + if we don't already have one, else return a ."""
-    if "+" in pieces.get("closest-tag", ""):
-        return "."
-    return "+"
-
-
-def render_pep440(pieces):
-    """Build up version string, with post-release "local version identifier".
-
-    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
-    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
-    Exceptions:
-    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += plus_or_dot(pieces)
-            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
-            if pieces["dirty"]:
-                rendered += ".dirty"
-    else:
-        # exception #1
-        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
-                                          pieces["short"])
-        if pieces["dirty"]:
-            rendered += ".dirty"
-    return rendered
-
-
-def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
-
-    Exceptions:
-    1: no tags. 0.post.devDISTANCE
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
-    else:
-        # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
-    return rendered
-
-
-def render_pep440_post(pieces):
-    """TAG[.postDISTANCE[.dev0]+gHEX] .
-
-    The ".dev0" means dirty. Note that .dev0 sorts backwards
-    (a dirty tree will appear "older" than the corresponding clean one),
-    but you shouldn't be releasing software with -dirty anyways.
-
-    Exceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-            rendered += plus_or_dot(pieces)
-            rendered += "g%s" % pieces["short"]
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-        rendered += "+g%s" % pieces["short"]
-    return rendered
-
-
-def render_pep440_old(pieces):
-    """TAG[.postDISTANCE[.dev0]] .
-
-    The ".dev0" means dirty.
-
-    Eexceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-    return rendered
-
-
-def render_git_describe(pieces):
-    """TAG[-DISTANCE-gHEX][-dirty].
-
-    Like 'git describe --tags --dirty --always'.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render_git_describe_long(pieces):
-    """TAG-DISTANCE-gHEX[-dirty].
-
-    Like 'git describe --tags --dirty --always -long'.
-    The distance/hash is unconditional.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render(pieces, style):
-    """Render the given version pieces into the requested style."""
-    if pieces["error"]:
-        return {"version": "unknown",
-                "full-revisionid": pieces.get("long"),
-                "dirty": None,
-                "error": pieces["error"],
-                "date": None}
-
-    if not style or style == "default":
-        style = "pep440"  # the default
-
-    if style == "pep440":
-        rendered = render_pep440(pieces)
-    elif style == "pep440-pre":
-        rendered = render_pep440_pre(pieces)
-    elif style == "pep440-post":
-        rendered = render_pep440_post(pieces)
-    elif style == "pep440-old":
-        rendered = render_pep440_old(pieces)
-    elif style == "git-describe":
-        rendered = render_git_describe(pieces)
-    elif style == "git-describe-long":
-        rendered = render_git_describe_long(pieces)
-    else:
-        raise ValueError("unknown style '%s'" % style)
-
-    return {"version": rendered, "full-revisionid": pieces["long"],
-            "dirty": pieces["dirty"], "error": None,
-            "date": pieces.get("date")}
-
-
-def get_versions():
-    """Get version information or return default if unable to do so."""
-    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
-    # __file__, we can work backwards from there to the root. Some
-    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
-    # case we can only use expanded keywords.
-
-    cfg = get_config()
-    verbose = cfg.verbose
-
-    try:
-        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
-                                          verbose)
-    except NotThisMethod:
-        pass
-
-    try:
-        root = os.path.realpath(__file__)
-        # versionfile_source is the relative path from the top of the source
-        # tree (where the .git directory might live) to this file. Invert
-        # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
-            root = os.path.dirname(root)
-    except NameError:
-        return {"version": "0+unknown", "full-revisionid": None,
-                "dirty": None,
-                "error": "unable to find root of source tree",
-                "date": None}
-
-    try:
-        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
-        return render(pieces, cfg.style)
-    except NotThisMethod:
-        pass
-
-    try:
-        if cfg.parentdir_prefix:
-            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
-    except NotThisMethod:
-        pass
-
-    return {"version": "0+unknown", "full-revisionid": None,
-            "dirty": None,
-            "error": "unable to compute version", "date": None}
--- a/incorporealcms/config.py
+++ b/incorporealcms/config.py
@ -61,7 +61,8 @@ class Config(object):
    }

    DEFAULT_PAGE_STYLE = 'light'
-    TITLE_SUFFIX = 'example.com'
+    DOMAIN_NAME = 'example.com'
+    TITLE_SUFFIX = DOMAIN_NAME
    CONTACT_EMAIL = 'admin@example.com'

    # specify FAVICON in your instance config.py to override the provided icon
--- a/incorporealcms/feed.py
+++ b/incorporealcms/feed.py
@ -0,0 +1,71 @@
+"""Generate Atom and RSS feeds based on content in a blog-ish location.
+
+This parses a special root directory, feed/, for feed/YYYY/MM/DD/file files,
+and combines them into an Atom or RSS feed. These files *should* be symlinks
+to the real pages, which may mirror the same YYYY/MM/DD/file naming scheme
+under pages/ (which may make sense for a blog) if they want, but could just
+as well be pages/foo content.
+
+SPDX-FileCopyrightText: © 2023 Brian S. Stephan <bss@incorporeal.org>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+import logging
+import os
+import re
+
+from feedgen.feed import FeedGenerator
+from flask import Blueprint, abort
+from flask import current_app as app
+
+from incorporealcms.lib import instance_resource_path_to_request_path, parse_md
+
+logger = logging.getLogger(__name__)
+
+bp = Blueprint('feed', __name__, url_prefix='/feed')
+
+
+@bp.route('/<feed_type>')
+def serve_feed(feed_type):
+    """Serve the Atom or RSS feed as requested."""
+    logger.warning("wat")
+    if feed_type not in ('atom', 'rss'):
+        abort(404)
+
+    fg = FeedGenerator()
+    fg.id(f'{app.config["DOMAIN_NAME"]}')
+    fg.title(f'{app.config["TITLE_SUFFIX"]}')
+    fg.link(href=f'https://{app.config["DOMAIN_NAME"]}/feed/{feed_type}', rel='self')
+    fg.link(href=f'https://{app.config["DOMAIN_NAME"]}', rel='alternate')
+    fg.subtitle(f"Blog posts and other dated materials from {app.config['TITLE_SUFFIX']}")
+
+    # get recent feeds
+    feed_path = os.path.join(app.instance_path, 'feed')
+    feed_entry_paths = [os.path.join(dirpath, filename) for dirpath, _, filenames in os.walk(feed_path)
+                        for filename in filenames if os.path.islink(os.path.join(dirpath, filename))]
+    for feed_entry_path in sorted(feed_entry_paths):
+        # get the actual file to parse it
+        resolved_path = os.path.realpath(feed_entry_path).replace(f'{app.instance_path}/', '')
+        try:
+            content, md, page_name, page_title, mtime = parse_md(resolved_path)
+            link = f'https://{app.config["DOMAIN_NAME"]}/{instance_resource_path_to_request_path(resolved_path)}'
+        except (OSError, ValueError, TypeError):
+            logger.exception("error loading/rendering markdown!")
+            abort(500)
+
+        fe = fg.add_entry()
+        fe.id(_generate_feed_id(feed_entry_path))
+        fe.title(page_name if page_name else page_title)
+        fe.link(href=link)
+        fe.content(content, type='html')
+
+    if feed_type == 'atom':
+        return fg.atom_str(pretty=True)
+    else:
+        return fg.rss_str(pretty=True)
+
+
+def _generate_feed_id(feed_entry_path):
+    """For a relative file path, generate the Atom/RSS feed ID for it."""
+    date = re.sub(r'.*/(\d+)/(\d+)/(\d+).*', r'\1-\2-\3', feed_entry_path)
+    cleaned = feed_entry_path.replace('#', '/').replace('feed/', '', 1).replace(app.instance_path, '')
+    return f'tag:{app.config["DOMAIN_NAME"]},{date}:{cleaned}'
--- a/incorporealcms/lib.py
+++ b/incorporealcms/lib.py
@ -3,11 +3,15 @@
 SPDX-FileCopyrightText: © 2021 Brian S. Stephan <bss@incorporeal.org>
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
+import datetime
 import logging
+import os
+import re

 import markdown
 from flask import current_app as app
 from flask import make_response, render_template, request
+from markupsafe import Markup

 logger = logging.getLogger(__name__)

@ -30,6 +34,45 @@ def init_md():
                             extension_configs=app.config['MARKDOWN_EXTENSION_CONFIGS'])


+def instance_resource_path_to_request_path(path):
+    """Reverse a (presumed to exist) RELATIVE disk path to the canonical path that would show up in a Flask route.
+
+    This does not include the leading /, so aside from the root index case, this should be
+    bidirectional.
+    """
+    return re.sub(r'^pages/', '', re.sub(r'.md$', '', re.sub(r'index.md$', '', path)))
+
+
+def parse_md(resolved_path):
+    """Given a file to parse, return file content and other derived data along with the md object."""
+    try:
+        logger.debug("opening resolved path '%s'", resolved_path)
+        with app.open_instance_resource(resolved_path, 'r') as entry_file:
+            mtime = datetime.datetime.fromtimestamp(os.path.getmtime(entry_file.name), tz=datetime.timezone.utc)
+            entry = entry_file.read()
+        logger.debug("resolved path '%s' read", resolved_path)
+        md = init_md()
+        content = Markup(md.convert(entry))
+    except OSError:
+        logger.exception("resolved path '%s' could not be opened!", resolved_path)
+        raise
+    except ValueError:
+        logger.exception("error parsing/rendering markdown!")
+        raise
+    except TypeError:
+        logger.exception("error loading/rendering markdown!")
+        raise
+
+    logger.debug("file metadata: %s", md.Meta)
+
+    page_name = (get_meta_str(md, 'title') if md.Meta.get('title') else
+                 f'/{instance_resource_path_to_request_path(resolved_path)}')
+    page_title = f'{page_name} - {app.config["TITLE_SUFFIX"]}' if page_name else app.config['TITLE_SUFFIX']
+    logger.debug("title (potentially derived): %s", page_title)
+
+    return content, md, page_name, page_title, mtime
+
+
 def render(template_name_or_list, **context):
    """Wrap Flask's render_template.

--- a/incorporealcms/mdx/figures.py
+++ b/incorporealcms/mdx/figures.py
@ -4,7 +4,7 @@ SPDX-FileCopyrightText: © 2022 Brian S. Stephan <bss@incorporeal.org>
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
 import re
-from xml.etree.ElementTree import SubElement
+from xml.etree.ElementTree import SubElement  # nosec B405 - not parsing untrusted XML here

 import markdown

--- a/incorporealcms/pages.py
+++ b/incorporealcms/pages.py
@ -3,10 +3,8 @@
 SPDX-FileCopyrightText: © 2020 Brian S. Stephan <bss@incorporeal.org>
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
-import datetime
 import logging
 import os
-import re

 from flask import Blueprint, abort
 from flask import current_app as app
@ -14,7 +12,7 @@ from flask import redirect, request, send_from_directory
 from markupsafe import Markup
 from werkzeug.security import safe_join

-from incorporealcms.lib import get_meta_str, init_md, render
+from incorporealcms.lib import get_meta_str, init_md, instance_resource_path_to_request_path, parse_md, render

 logger = logging.getLogger(__name__)

@ -54,36 +52,19 @@ def display_page(path):
 def handle_markdown_file_path(resolved_path):
    """Given a location on disk, attempt to open it and render the markdown within."""
    try:
-        logger.debug("opening resolved path '%s'", resolved_path)
-        with app.open_instance_resource(resolved_path, 'r') as entry_file:
-            mtime = datetime.datetime.fromtimestamp(os.path.getmtime(entry_file.name), tz=datetime.timezone.utc)
-            entry = entry_file.read()
-        logger.debug("resolved path '%s' read", resolved_path)
+        content, md, page_name, page_title, mtime = parse_md(resolved_path)
    except OSError:
        logger.exception("resolved path '%s' could not be opened!", resolved_path)
        abort(500)
+    except ValueError:
+        logger.exception("error parsing/rendering markdown!")
+        abort(500)
+    except TypeError:
+        logger.exception("error loading/rendering markdown!")
+        abort(500)
    else:
-        try:
-            md = init_md()
-            content = Markup(md.convert(entry))
-        except ValueError:
-            logger.exception("error parsing/rendering markdown!")
-            abort(500)
-        except TypeError:
-            logger.exception("error loading/rendering markdown!")
-            abort(500)
-
-        logger.debug("file metadata: %s", md.Meta)
-
        parent_navs = generate_parent_navs(resolved_path)
-
-        page_name = (get_meta_str(md, 'title') if md.Meta.get('title') else
-                     f'/{instance_resource_path_to_request_path(resolved_path)}')
-        page_title = f'{page_name} - {app.config["TITLE_SUFFIX"]}' if page_name else app.config['TITLE_SUFFIX']
-        logger.debug("title (potentially derived): %s", page_title)
-
        extra_footer = get_meta_str(md, 'footer') if md.Meta.get('footer') else None
-
        template = get_meta_str(md, 'template') if md.Meta.get('template') else 'base.html'

        # check if this has a HTTP redirect
@ -156,20 +137,11 @@ def request_path_to_instance_resource_path(path):
    return resolved_path.replace(f'{app.instance_path}{os.path.sep}', ''), 'markdown'


-def instance_resource_path_to_request_path(path):
-    """Reverse a (presumed to exist) disk path to the canonical path that would show up in a Flask route.
-
-    This does not include the leading /, so aside from the root index case, this should be
-    bidirectional.
-    """
-    return re.sub(r'^pages/', '', re.sub(r'.md$', '', re.sub(r'index.md$', '', path)))
-
-
 def generate_parent_navs(path):
    """Create a series of paths/links to navigate up from the given resource path."""
    if path == 'pages/index.md':
-        # bail and return the title suffix (generally the domain name) as a terminal case
-        return [(app.config['TITLE_SUFFIX'], '/')]
+        # bail and return the domain name as a terminal case
+        return [(app.config['DOMAIN_NAME'], '/')]
    else:
        if path.endswith('index.md'):
            # index case: one dirname for foo/bar/index.md -> foo/bar, one for foo/bar -> foo
--- a/pyproject.toml
+++ b/pyproject.toml
@ -0,0 +1,54 @@
+[build-system]
+requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "incorporeal-cms"
+description = "A CMS for serving Markdown files with a bit of dynamicism."
+readme = "README.md"
+license = {text = "AGPL-3.0-or-later"}
+authors = [
+	{name = "Brian S. Stephan", email = "bss@incorporeal.org"},
+]
+requires-python = ">=3.8"
+dependencies = ["feedgen", "Flask", "Markdown"]
+dynamic = ["version"]
+classifiers = [
+	"Framework :: Flask",
+	"Programming Language :: Python :: 3",
+	"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+	"Operating System :: OS Independent",
+	"Topic :: Text Processing :: Markup :: Markdown",
+]
+
+[project.urls]
+"Homepage" = "https://git.incorporeal.org/bss/incorporeal-cms"
+"Changelog" = "https://git.incorporeal.org/bss/incorporeal-cms/releases"
+"Bug Tracker" = "https://git.incorporeal.org/bss/incorporeal-cms/issues"
+
+
+[project.optional-dependencies]
+dev = ["bandit", "dlint", "flake8", "flake8-blind-except", "flake8-builtins", "flake8-docstrings",
+       "flake8-executable", "flake8-fixme", "flake8-isort", "flake8-logging-format", "flake8-mutable",
+       "flake8-pyproject", "mypy", "pip-tools", "pydot", "pytest", "pytest-cov", "safety",
+       "setuptools-scm", "tox"]
+dot = ["pydot"]
+
+[tool.flake8]
+enable-extensions = "G,M"
+exclude = [".tox/", "venv/", "_version.py"]
+extend-ignore = "T101"
+max-complexity = 10
+max-line-length = 120
+
+[tool.isort]
+line_length = 120
+
+[tool.mypy]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+python_files = ["*_tests.py", "tests.py", "test_*.py"]
+
+[tool.setuptools_scm]
+write_to = "incorporealcms/_version.py"
--- a/requirements/requirements-dev.in
+++ b/requirements/requirements-dev.in
@ -1,26 +0,0 @@
-r requirements.in
-
-# testing runner, test reporting, packages used during testing (e.g. requests-mock), etc.
-pydot
-pytest
-pytest-cov
-
-# linting and other static code analysis
-bandit
-dlint
-flake8
-flake8-blind-except
-flake8-builtins
-flake8-docstrings
-flake8-executable
-flake8-fixme
-flake8-isort
-flake8-logging-format
-flake8-mutable
-safety                      # check requirements file for issues
-
-# maintenance utilities and tox
-pip-tools                   # pip-compile
-tox<4                       # CI stuff, pinned for now to avoid packaging conflict w/safety
-tox-wheel                   # build wheels in tox
-versioneer                  # automatic version numbering
--- a/requirements/requirements-dev.txt
+++ b/requirements/requirements-dev.txt
@ -1,17 +1,21 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
+#    pip-compile --extra=dev --output-file=requirements/requirements-dev.txt
 #
 bandit==1.7.6
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 blinker==1.7.0
    # via flask
 build==1.0.3
    # via pip-tools
+cachetools==5.3.2
+    # via tox
 certifi==2023.11.17
    # via requests
+chardet==5.2.0
+    # via tox
 charset-normalizer==3.3.2
    # via requests
 click==8.1.7
@ -19,6 +23,8 @@ click==8.1.7
    #   flask
    #   pip-tools
    #   safety
+colorama==0.4.6
+    # via tox
 coverage[toml]==7.4.0
    # via
    #   coverage
@ -26,40 +32,45 @@ coverage[toml]==7.4.0
 distlib==0.3.8
    # via virtualenv
 dlint==0.14.1
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 dparse==0.6.3
    # via safety
+feedgen==1.0.0
+    # via incorporeal-cms (pyproject.toml)
 filelock==3.13.1
    # via
    #   tox
    #   virtualenv
 flake8==6.1.0
    # via
-    #   -r requirements/requirements-dev.in
    #   dlint
    #   flake8-builtins
    #   flake8-docstrings
    #   flake8-executable
    #   flake8-isort
    #   flake8-mutable
+    #   flake8-pyproject
+    #   incorporeal-cms (pyproject.toml)
 flake8-blind-except==0.2.1
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-builtins==2.2.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-docstrings==1.7.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-executable==2.1.3
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-fixme==1.1.1
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-isort==6.1.1
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-logging-format==0.9.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 flake8-mutable==1.2.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
+flake8-pyproject==1.2.3
+    # via incorporeal-cms (pyproject.toml)
 flask==3.0.0
-    # via -r requirements/requirements.in
+    # via incorporeal-cms (pyproject.toml)
 gitdb==4.0.11
    # via gitpython
 gitpython==3.1.40
@ -74,8 +85,10 @@ itsdangerous==2.1.2
    # via flask
 jinja2==3.1.2
    # via flask
+lxml==5.0.0
+    # via feedgen
 markdown==3.5.1
-    # via -r requirements/requirements.in
+    # via incorporeal-cms (pyproject.toml)
 markdown-it-py==3.0.0
    # via rich
 markupsafe==2.1.3
@ -86,31 +99,37 @@ mccabe==0.7.0
    # via flake8
 mdurl==0.1.2
    # via markdown-it-py
+mypy==1.8.0
+    # via incorporeal-cms (pyproject.toml)
+mypy-extensions==1.0.0
+    # via mypy
 packaging==21.3
    # via
    #   build
    #   dparse
+    #   pyproject-api
    #   pytest
    #   safety
+    #   setuptools-scm
    #   tox
 pbr==6.0.0
    # via stevedore
 pip-tools==7.3.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 platformdirs==4.1.0
-    # via virtualenv
+    # via
+    #   tox
+    #   virtualenv
 pluggy==1.3.0
    # via
    #   pytest
    #   tox
-py==1.11.0
-    # via tox
 pycodestyle==2.11.1
    # via flake8
 pydocstyle==6.3.0
    # via flake8-docstrings
 pydot==1.4.2
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
 pyflakes==3.1.0
    # via flake8
 pygments==2.17.2
@ -119,14 +138,18 @@ pyparsing==3.1.1
    # via
    #   packaging
    #   pydot
+pyproject-api==1.5.0
+    # via tox
 pyproject-hooks==1.0.0
    # via build
 pytest==7.4.3
    # via
-    #   -r requirements/requirements-dev.in
+    #   incorporeal-cms (pyproject.toml)
    #   pytest-cov
 pytest-cov==4.1.0
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
+python-dateutil==2.8.2
+    # via feedgen
 pyyaml==6.0.1
    # via bandit
 requests==2.31.0
@ -138,33 +161,31 @@ ruamel-yaml==0.18.5
 ruamel-yaml-clib==0.2.8
    # via ruamel-yaml
 safety==2.3.5
-    # via -r requirements/requirements-dev.in
+    # via incorporeal-cms (pyproject.toml)
+setuptools-scm==8.0.4
+    # via incorporeal-cms (pyproject.toml)
 six==1.16.0
-    # via tox
+    # via python-dateutil
 smmap==5.0.1
    # via gitdb
 snowballstemmer==2.2.0
    # via pydocstyle
 stevedore==5.1.0
    # via bandit
-tox==3.28.0
+tox==4.0.0
+    # via incorporeal-cms (pyproject.toml)
+typing-extensions==4.9.0
    # via
-    #   -r requirements/requirements-dev.in
-    #   tox-wheel
-tox-wheel==1.0.0
-    # via -r requirements/requirements-dev.in
+    #   mypy
+    #   setuptools-scm
 urllib3==2.1.0
    # via requests
-versioneer==0.29
-    # via -r requirements/requirements-dev.in
 virtualenv==20.25.0
    # via tox
 werkzeug==3.0.1
    # via flask
 wheel==0.42.0
-    # via
-    #   pip-tools
-    #   tox-wheel
+    # via pip-tools

 # The following packages are considered to be unsafe in a requirements file:
 # pip
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@ -1,2 +0,0 @@
-Flask                       # general purpose web service and web server stuff
-Markdown                    # markdown rendering in templates
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@ -1,24 +1,32 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
+#    pip-compile --output-file=requirements/requirements.txt
 #
 blinker==1.7.0
    # via flask
 click==8.1.7
    # via flask
+feedgen==1.0.0
+    # via incorporeal-cms (pyproject.toml)
 flask==3.0.0
-    # via -r requirements/requirements.in
+    # via incorporeal-cms (pyproject.toml)
 itsdangerous==2.1.2
    # via flask
 jinja2==3.1.2
    # via flask
+lxml==5.0.0
+    # via feedgen
 markdown==3.5.1
-    # via -r requirements/requirements.in
+    # via incorporeal-cms (pyproject.toml)
 markupsafe==2.1.3
    # via
    #   jinja2
    #   werkzeug
+python-dateutil==2.8.2
+    # via feedgen
+six==1.16.0
+    # via python-dateutil
 werkzeug==3.0.1
    # via flask
--- a/setup.cfg
+++ b/setup.cfg
@ -1,6 +0,0 @@
-[versioneer]
-VCS = git
-style = pep440-post
-versionfile_source = incorporealcms/_version.py
-versionfile_build = incorporealcms/_version.py
-tag_prefix = v
--- a/setup.py
+++ b/setup.py
@ -1,40 +0,0 @@
-"""Setuptools configuration.
-
-SPDX-FileCopyrightText: © 2020 Brian S. Stephan <bss@incorporeal.org>
-SPDX-License-Identifier: AGPL-3.0-or-later
-"""
-import os
-
-from setuptools import find_packages, setup
-
-import versioneer
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-
-
-def extract_requires():
-    """Get pinned requirements from requirements.txt."""
-    with open(os.path.join(HERE, 'requirements/requirements.txt'), 'r') as reqs:
-        return [line.split(' ')[0] for line in reqs if not line[0] in ('-', '#')]
-
-
-setup(
-    name='incorporeal-cms',
-    description='Flask project for running https://suou.net (and eventually others).',
-    url='https://git.incorporeal.org/bss/incorporeal-cms',
-    author='Brian S. Stephan',
-    author_email='bss@incorporeal.org',
-    license='AGPLv3+',
-    classifiers=[
-        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
-    ],
-    version=versioneer.get_version(),
-    cmdclass=versioneer.get_cmdclass(),
-    packages=find_packages(),
-    include_package_data=True,
-    zip_safe=False,
-    install_requires=extract_requires(),
-    extras_require={
-        'dot': ['pydot'],
-    },
-)
--- a/tests/instance/feed/2023/12/01/forced-no-title.md
+++ b/tests/instance/feed/2023/12/01/forced-no-title.md
@ -0,0 +1 @@
+../../../../pages/forced-no-title.md
--- a/tests/instance/feed/2023/12/30/page.md
+++ b/tests/instance/feed/2023/12/30/page.md
@ -0,0 +1 @@
+../../../../pages/subdir-with-title/page.md
--- a/tests/test_feed.py
+++ b/tests/test_feed.py
@ -0,0 +1,32 @@
+"""Test the feed methods.
+
+SPDX-FileCopyrightText: © 2023 Brian S. Stephan <bss@incorporeal.org>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+from incorporealcms.feed import serve_feed
+
+
+def test_unknown_type_is_404(client):
+    """Test that requesting a feed type that doesn't exist is a 404."""
+    response = client.get('/feed/wat')
+    assert response.status_code == 404
+
+
+def test_atom_type_is_200(client):
+    """Test that requesting an ATOM feed is found."""
+    response = client.get('/feed/atom')
+    assert response.status_code == 200
+    print(response.text)
+
+
+def test_rss_type_is_200(client):
+    """Test that requesting an RSS feed is found."""
+    response = client.get('/feed/rss')
+    assert response.status_code == 200
+    print(response.text)
+
+
+def test_feed_generator(app):
+    """Test the root feed generator."""
+    with app.test_request_context():
+        serve_feed('atom')
--- a/tox.ini
+++ b/tox.ini
@ -4,21 +4,11 @@
 # and then run "tox" from this directory.

 [tox]
-envlist = begin,py38,py39,py310,py311,coverage,security,lint,bundle
+isolated_build = true
+envlist = begin,py38,py39,py310,py311,py312,coverage,security,lint

 [testenv]
-# build a wheel and test it
-wheel = true
-wheel_build_env = build
-
-# whitelist commands we need
-whitelist_externals = cp
-
-# install everything via requirements-dev.txt, so that developer environment
-# is the same as the tox environment (for ease of use/no weird gotchas in
-# local dev results vs. tox results) and also to avoid ticky-tacky maintenance
-# of "oh this particular env has weird results unless I install foo" --- just
-# shotgun blast install everything everywhere
+allow_externals = pytest, coverage
 deps =
    -rrequirements/requirements-dev.txt

@ -51,6 +41,11 @@ commands =
 commands =
    pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch

+[testenv:py312]
+# run pytest with coverage
+commands =
+    pytest --cov-append --cov={envsitepackagesdir}/incorporealcms/ --cov-branch
+
 [testenv:coverage]
 # report on coverage runs from above
 skip_install = true
@ -61,23 +56,17 @@ commands =
 # run security checks
 #
 # again it seems the most valuable here to run against the packaged code
-# 51457 is nearly a red herring that I'm stuck with because tox is pinned, try removing occasionally
 commands =
    bandit {envsitepackagesdir}/incorporealcms/ -r
-    safety check -r requirements/requirements-dev.txt -i 51457
+    safety check -r requirements/requirements-dev.txt

 [testenv:lint]
 # run style checks
+# TODO: mypy incorporealcms
 commands =
    flake8
    - flake8 --disable-noqa --ignore= --select=E,W,F,C,D,A,G,B,I,T,M,DUO

-[testenv:bundle]
-# take extra actions (build sdist, sphinx, whatever) to completely package the app
-commands =
-    cp -r {distdir} .
-    python setup.py sdist
-
 [coverage:paths]
 source =
    ./
@ -88,24 +77,3 @@ branch = True

 omit =
    **/_version.py
-
-[flake8]
-enable-extensions = G,M
-exclude =
-    .tox/
-    versioneer.py
-    _version.py
-    instance/
-    venv/
-extend-ignore = T101
-max-complexity = 10
-max-line-length = 120
-
-[isort]
-line_length = 120
-
-[pytest]
-python_files =
-    *_tests.py
-    tests.py
-    test_*.py
--- a/versioneer.py
+++ b/versioneer.py
Author	SHA1	Message	Date
Brian S. Stephan	6dc443e59f	implement a rudimentary Atom/RSS feed module this provides a somewhat unconfigurable (at the moment) feed module which provides Atom and RSS feeds. entries are determined by symlinks to content pages, because my core CMS usage is still more general and not blog-like. the symlinks allow for arbitrarily adding entries as I see fit. this also moves core Markdown parser stuff to the library module, since that's used by the feed as well as normal pages Signed-off-by: Brian S. Stephan <bss@incorporeal.org>	2023-12-30 14:55:02 -06:00
Brian S. Stephan	5a9a36f463	deduplicate TITLE_SUFFIX from new DOMAIN_NAME I will need the domain name for feed stuff, and I'm already crudely using the title suffix in the nav as if it was a domain name, so let's just be explicit in the case I ever change my mind on domain-in-title styling Signed-off-by: Brian S. Stephan <bss@incorporeal.org>	2023-12-30 11:55:01 -06:00
Brian S. Stephan	680a2bc764	add python 3.12 to tox environments Signed-off-by: Brian S. Stephan <bss@incorporeal.org>	2023-12-30 11:13:08 -06:00
Brian S. Stephan	713632fe7a	unpin tox in requirements for some reason bandit wasn't earlier catching the SubElement usage but now it is, but it's harmless anyway so we'll just suppress it. Signed-off-by: Brian S. Stephan <bss@incorporeal.org>	2023-12-30 11:12:59 -06:00
Brian S. Stephan	bf646db1e8	convert tooling to pyproject.toml based still has dynamic versioning and etc. Signed-off-by: Brian S. Stephan <bss@incorporeal.org>	2023-12-30 10:33:28 -06:00