properly test protobuf structure packaging options

this should all be tested now:
1. invoking against precompiled _pb2.py files provided by user
2. invoking against .proto files provided by user which must be compiled
3. invoking with a special option to use shipped (by us) .proto files
   which must be compiled
4. erroring because none of the above occurred

this took some reorganization, but this should finally give me stability
in using this in GP2040-CE's build process

Signed-off-by: Brian S. Stephan <bss@incorporeal.org>

gp2040ce_bintools/__init__.py

@@ -40,6 +40,8 @@ core_parser.add_argument('-d', '--debug', action='store_true', help="enable debu
 core_parser.add_argument('-P', '--proto-files-path', type=pathlib.Path, default=list(), action='append',
                          help="path to .proto files to read, including dependencies; you will likely need "
                               "to supply this twice, once for GP2040-CE's .proto files and once for nanopb's")
+core_parser.add_argument('-S', '--use-shipped-fallback', action='store_true',
+                         help="utilize shipped (potentially stale) .proto files because you can't supply your own")
 args, _ = core_parser.parse_known_args()
 for path in args.proto_files_path:
     sys.path.append(os.path.abspath(os.path.expanduser(path)))
@@ -50,7 +52,7 @@ else:
     handler.setLevel(logging.WARNING)
 
 
-def get_config_pb2():
+def get_config_pb2(with_fallback: bool = args.use_shipped_fallback):
     """Retrieve prebuilt _pb2 file or attempt to compile it live."""
     # try to just import a precompiled module if we have been given it in our path
     # (perhaps someone already compiled it for us for whatever reason)
@@ -65,6 +67,9 @@ def get_config_pb2():
             return grpc.protos('config.proto')
         except (ModuleNotFoundError, TypeError):
             # (TypeError could be the windows bug https://github.com/protocolbuffers/protobuf/issues/14345)
+            if not with_fallback:
+                raise
+
             # that failed, import the snapshot (may be lagging what's in GP2040-CE)
             logger.warning("using the fallback .proto files! please supply your files with -P if you can!")
             sys.path.append(os.path.join(pathlib.Path(__file__).parent.resolve(), 'proto_snapshot'))

pyproject.toml

@@ -74,6 +74,10 @@ ignore_errors = true
 [tool.pytest]
 python_files = ["*_tests.py", "tests.py", "test_*.py"]
 
+[tool.pytest.ini_options]
+log_cli = 0
+log_cli_level = "WARNING"
+
 [tool.setuptools]
 packages = [
     "gp2040ce_bintools",

tests/test-files/pb2-files/nanopb_pb2.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: nanopb.proto
+# Protobuf Python Version: 5.27.2
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    5,
+    27,
+    2,
+    '',
+    'nanopb.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\xc4\x07\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x12\n\nmax_length\x18\x0e \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12&\n\x08int_size\x18\x07 \x01(\x0e\x32\x08.IntSize:\nIS_DEFAULT\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\x1a\n\x0bpacked_enum\x18\n \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0cskip_message\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\x18\n\tno_unions\x18\x08 \x01(\x08:\x05\x66\x61lse\x12\r\n\x05msgid\x18\t \x01(\r\x12\x1e\n\x0f\x61nonymous_oneof\x18\x0b \x01(\x08:\x05\x66\x61lse\x12\x15\n\x06proto3\x18\x0c \x01(\x08:\x05\x66\x61lse\x12#\n\x14proto3_singular_msgs\x18\x15 \x01(\x08:\x05\x66\x61lse\x12\x1d\n\x0e\x65num_to_string\x18\r \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0c\x66ixed_length\x18\x0f \x01(\x08:\x05\x66\x61lse\x12\x1a\n\x0b\x66ixed_count\x18\x10 \x01(\x08:\x05\x66\x61lse\x12\x1e\n\x0fsubmsg_callback\x18\x16 \x01(\x08:\x05\x66\x61lse\x12/\n\x0cmangle_names\x18\x11 \x01(\x0e\x32\x11.TypenameMangling:\x06M_NONE\x12(\n\x11\x63\x61llback_datatype\x18\x12 \x01(\t:\rpb_callback_t\x12\x34\n\x11\x63\x61llback_function\x18\x13 \x01(\t:\x19pb_default_field_callback\x12\x30\n\x0e\x64\x65scriptorsize\x18\x14 \x01(\x0e\x32\x0f.DescriptorSize:\x07\x44S_AUTO\x12\x1a\n\x0b\x64\x65\x66\x61ult_has\x18\x17 \x01(\x08:\x05\x66\x61lse\x12\x0f\n\x07include\x18\x18 \x03(\t\x12\x0f\n\x07\x65xclude\x18\x1a \x03(\t\x12\x0f\n\x07package\x18\x19 \x01(\t\x12\x41\n\rtype_override\x18\x1b \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.Type\x12\x19\n\x0bsort_by_tag\x18\x1c \x01(\x08:\x04true\x12.\n\rfallback_type\x18\x1d \x01(\x0e\x32\n.FieldType:\x0b\x46T_CALLBACK\x12\x1e\n\x0f\x64isallow_export\x18\x1e \x01(\x08:\x05\x66\x61lse*i\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03\x12\r\n\tFT_INLINE\x10\x05*D\n\x07IntSize\x12\x0e\n\nIS_DEFAULT\x10\x00\x12\x08\n\x04IS_8\x10\x08\x12\t\n\x05IS_16\x10\x10\x12\t\n\x05IS_32\x10 \x12\t\n\x05IS_64\x10@*Z\n\x10TypenameMangling\x12\n\n\x06M_NONE\x10\x00\x12\x13\n\x0fM_STRIP_PACKAGE\x10\x01\x12\r\n\tM_FLATTEN\x10\x02\x12\x16\n\x12M_PACKAGE_INITIALS\x10\x03*E\n\x0e\x44\x65scriptorSize\x12\x0b\n\x07\x44S_AUTO\x10\x00\x12\x08\n\x04\x44S_1\x10\x01\x12\x08\n\x04\x44S_2\x10\x02\x12\x08\n\x04\x44S_4\x10\x04\x12\x08\n\x04\x44S_8\x10\x08:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'nanopb_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\030fi.kapsi.koti.jpa.nanopb'
+  _globals['_FIELDTYPE']._serialized_start=1017
+  _globals['_FIELDTYPE']._serialized_end=1122
+  _globals['_INTSIZE']._serialized_start=1124
+  _globals['_INTSIZE']._serialized_end=1192
+  _globals['_TYPENAMEMANGLING']._serialized_start=1194
+  _globals['_TYPENAMEMANGLING']._serialized_end=1284
+  _globals['_DESCRIPTORSIZE']._serialized_start=1286
+  _globals['_DESCRIPTORSIZE']._serialized_end=1355
+  _globals['_NANOPBOPTIONS']._serialized_start=51
+  _globals['_NANOPBOPTIONS']._serialized_end=1015
+# @@protoc_insertion_point(module_scope)

tests/test_builder.py

@@ -25,13 +25,15 @@ logger = logging.getLogger(__name__)
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_concatenate_to_file(tmp_path):

tests/test_commands.py

@@ -18,13 +18,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_version_flag():

tests/test_gui.py

@@ -21,13 +21,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 async def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     await test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 @pytest.mark.asyncio

tests/test_package.py

@@ -6,43 +6,69 @@ SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import sys
 
+import pytest
+from decorator import decorator
+
 from gp2040ce_bintools import get_config_pb2
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
 
-def test_get_config_pb2_compile():
-    """Without any precompiled files on the path, test we can read proto files and compile them."""
-    # append to path as -P would
+@decorator
+def with_pb2s(test, *args, **kwargs):
+    """Wrap a test with precompiled pb2 files on the path."""
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
+    sys.path.append(proto_path)
+
+    test(*args, **kwargs)
+
+    sys.path.pop()
+    del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
+
+
+@decorator
+def with_protos(test, *args, **kwargs):
+    """Wrap a test with .proto files on the path."""
     proto_path = os.path.join(HERE, 'test-files', 'proto-files')
     sys.path.append(proto_path)
 
-    # let grpc tools compile the proto files on demand and give us the module
-    config_pb2 = get_config_pb2()
-    _ = config_pb2.Config()
+    test(*args, **kwargs)
 
-    # clean up the path and unload config_pb2
-    sys.path.pop()
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
+
+
+@with_pb2s
+def test_get_config_pb2_precompiled():
+    """With precompiled files on the path, test we can read and use them."""
+    # get the module from the provided files
+    config_pb2 = get_config_pb2()
+    _ = config_pb2.Config()
 
 
 def test_get_config_pb2_exception():
-    """Without any precompiled files or proto files on the path, test we DO NOT raise an exception."""
-    # this used to raise ModuleNotFoundError, but with our snapshot included now,
-    # we should always have a config to import
+    """Test that we fail if no config .proto files are available."""
+    with pytest.raises(ModuleNotFoundError):
         _ = get_config_pb2()
 
 
-def test_get_config_pb2_precompile():
-    """Test we can import precompiled protobuf files."""
-    proto_path = os.path.join(HERE, 'test-files')
-    sys.path.append(proto_path)
+def test_get_config_pb2_shipped_config_files():
+    """Without any precompiled files or proto files on the path, test we DO NOT raise an exception."""
+    # use the shipped .proto files to generate the config
+    config_pb2 = get_config_pb2(with_fallback=True)
+    _ = config_pb2.Config()
+    del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
-    # let grpc tools import the proto files normally
+
+@with_protos
+def test_get_config_pb2_compile():
+    """Without any precompiled files on the path, test we can read proto files and compile them."""
+    # let grpc tools compile the proto files on demand and give us the module
     config_pb2 = get_config_pb2()
     _ = config_pb2.Config()
-
-    # clean up the path and unload config_pb2
-    sys.path.pop()
-    del sys.modules['config_pb2']

tests/test_rp2040.py

@@ -20,13 +20,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_get_bootsel_endpoints():

tests/test_storage.py

@@ -20,13 +20,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_config_footer(storage_dump):