use beautifulsoup to parse the description from the first paragraph

Signed-off-by: Brian S. Stephan <bss@incorporeal.org>

CHANGELOG.md

@@ -2,35 +2,6 @@
 
 Included is a summary of changes to the project, by version. Details can be found in the commit history.
 
-## v2.1.1
-
-### Improvements
-
-* Use the h1-as-name feature from v2.1.0 also to generate the page name in breadcrumbs. This changes the behavior on
-  pages with an h1 but no Title: meta tag to have a better name, of course, but also changes the behavior on pages with
-  neither a h1 nor a Title: meta tag to have a leading slash (e.g. /page-filename) where there previously was not one
-  (e.g. just page-filename). This seems like an acceptable trade-off.
-
-### Miscellaneous
-
-* With the minor breadcrumb change, a method used to finagle the breadcrumb no-name name is no longer necessary.
-
-## v2.1.0
-
-### Features
-
-* The page title (also used in the `og:title` header) and the optional description used in the `og:description` header
-  can be derived from the contents of the page content, if the markdown meta tags are not supplied. The first `h1` is
-  used for the title, and the first `p` is used for the description. This is largely to save some time writing pages
-  that one wants to look nice, especially in a social media card, and removes some repetition.
-
-### Miscellaneous
-
-* Requirements bumped, which led to...
-* Python 3.9 has been removed from the supported versions.
-* Added some miscellaneous unit tests and coverage changes to keep us at 95% (which only dropped for a library reason I
-  don't understand).
-
 ## v2.0.5
 
 ### Features

README.md

@@ -7,8 +7,8 @@ A lightweight static site generator for Markdown-based sites.
 Something like the following should suffice:
 
 ```
-% virtualenv --python=python3.10 env-py3.10
+% virtualenv --python=python3.9 env-py3.9
-% source env-py3.10/bin/activate
+% source env-py3.9/bin/activate
 % pip install -U pip
 % pip install incorporeal-cms
 % incorporealcms-build ./path/to/instance ./path/to/output/www/root

incorporealcms/markdown.py

@@ -84,21 +84,6 @@ def parse_md(path: str, pages_root: str):
 
     rel_path = os.path.relpath(path, pages_root)
 
-    page_name, page_description = _get_metadata_from_parsed_page(md, content, rel_path)
-    page_title = f'{page_name} - {Config.TITLE_SUFFIX}' if page_name else Config.TITLE_SUFFIX
-    logger.debug("title (potentially derived): %s", page_title)
-
-    return content, md, page_name, page_title, page_description, mtime
-
-
-def _get_metadata_from_parsed_page(md, content, path: str):
-    """Get the page name and description from a Markdown object and/or HTML output of a page.
-
-    Args:
-        md: the parsed Markdown object, potentially including Meta tags
-        content: the Markdown page content converted to HTML, to run through BeautifulSoup
-        path: path of the page, to derive the name from as a fallback
-    """
     soup = BeautifulSoup(content, features='lxml')
 
     # get the page title first from the markdown tags, second from the first h1, last from the path
@@ -108,7 +93,7 @@ def _get_metadata_from_parsed_page(md, content, path: str):
     elif h1_tag := soup.find('h1'):
         page_name = h1_tag.string
     elif not page_name:
-        page_name = instance_resource_path_to_request_path(path)
+        page_name = instance_resource_path_to_request_path(rel_path)
 
     # get the page description from the markdown tags or first paragraph
     page_description = None
@@ -118,7 +103,10 @@ def _get_metadata_from_parsed_page(md, content, path: str):
         if page_description := p_tag.string:
             page_description = page_description.replace('\n', ' ')
 
-    return page_name, page_description
+    page_title = f'{page_name} - {Config.TITLE_SUFFIX}' if page_name else Config.TITLE_SUFFIX
+    logger.debug("title (potentially derived): %s", page_title)
+
+    return content, md, page_name, page_title, page_description, mtime
 
 
 def handle_markdown_file_path(path: str, pages_root: str) -> str:
@@ -187,8 +175,16 @@ def generate_parent_navs(path, pages_root: str):
         try:
             with open(os.path.join(pages_root, path), 'r') as entry_file:
                 entry = entry_file.read()
-            content = Markup(md.convert(entry))  # nosec B704
+            _ = Markup(md.convert(entry))       # nosec B704
-            page_name, _ = _get_metadata_from_parsed_page(md, content, os.path.relpath(path, parent_resource_dir))
+            page_name = (" ".join(md.Meta.get('title')) if md.Meta.get('title')
+                         else request_path_to_breadcrumb_display(request_path))
             return generate_parent_navs(parent_resource_path, pages_root) + [(page_name, request_path)]
         except FileNotFoundError:
             return generate_parent_navs(parent_resource_path, pages_root) + [(request_path, request_path)]
+
+
+def request_path_to_breadcrumb_display(path):
+    """Given a request path, e.g. "/foo/bar/baz/", turn it into breadcrumby text "baz"."""
+    undired = path.rstrip('/')
+    leaf = undired[undired.rfind('/'):]
+    return leaf.strip('/')

tests/test_markdown.py

@@ -10,7 +10,8 @@ import pytest
 
 from incorporealcms import init_instance
 from incorporealcms.markdown import (generate_parent_navs, handle_markdown_file_path,
-                                     instance_resource_path_to_request_path, parse_md)
+                                     instance_resource_path_to_request_path, parse_md,
+                                     request_path_to_breadcrumb_display)
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 INSTANCE_DIR = os.path.join(HERE, 'instance')
@@ -25,21 +26,14 @@ def test_generate_page_navs_index():
     assert generate_parent_navs('index.md', PAGES_DIR) == [('example.org', '/')]
 
 
-def test_generate_page_navs_title_from_h1():
-    """Test that the index page has navs to the root (itself)."""
-    assert generate_parent_navs('no-title.md', PAGES_DIR) == [('example.org', '/'),
-                                                              ('this page doesn\'t have a title!', '/no-title')]
-
-
 def test_generate_page_navs_subdir_index():
     """Test that dir pages have navs to the root and themselves."""
-    assert generate_parent_navs('subdir/index.md', PAGES_DIR) == [('example.org', '/'), ('another page', '/subdir/')]
+    assert generate_parent_navs('subdir/index.md', PAGES_DIR) == [('example.org', '/'), ('subdir', '/subdir/')]
 
 
 def test_generate_page_navs_subdir_real_page():
     """Test that real pages have navs to the root, their parent, and themselves."""
-    assert generate_parent_navs('subdir/page.md', PAGES_DIR) == [('example.org', '/'),
+    assert generate_parent_navs('subdir/page.md', PAGES_DIR) == [('example.org', '/'), ('subdir', '/subdir/'),
-                                                                 ('another page', '/subdir/'),
                                                                  ('Page', '/subdir/page')]
 
 
@@ -48,7 +42,7 @@ def test_generate_page_navs_subdir_with_title_parsing_real_page():
     assert generate_parent_navs('subdir-with-title/page.md', PAGES_DIR) == [
         ('example.org', '/'),
         ('SUB!', '/subdir-with-title/'),
-        ('/page', '/subdir-with-title/page')
+        ('page', '/subdir-with-title/page')
     ]
 
 
@@ -57,7 +51,7 @@ def test_generate_page_navs_subdir_with_no_index():
     assert generate_parent_navs('no-index-dir/page.md', PAGES_DIR) == [
         ('example.org', '/'),
         ('/no-index-dir/', '/no-index-dir/'),
-        ('/page', '/no-index-dir/page')
+        ('page', '/no-index-dir/page')
     ]
 
 
@@ -129,6 +123,15 @@ def test_instance_resource_path_to_request_path_on_subdir_and_page():
     assert instance_resource_path_to_request_path('subdir/page.md') == '/subdir/page'
 
 
+def test_request_path_to_breadcrumb_display_patterns():
+    """Test various conversions from request path to leaf nodes for display in the breadcrumbs."""
+    assert request_path_to_breadcrumb_display('/foo') == 'foo'
+    assert request_path_to_breadcrumb_display('/foo/') == 'foo'
+    assert request_path_to_breadcrumb_display('/foo/bar') == 'bar'
+    assert request_path_to_breadcrumb_display('/foo/bar/') == 'bar'
+    assert request_path_to_breadcrumb_display('/') == ''
+
+
 def test_parse_md_metadata():
     """Test the direct results of parsing a markdown file."""
     content, md, page_name, page_title, page_desc, mtime = parse_md(