changelog bump

Signed-off-by: Brian S. Stephan <bss@incorporeal.org>

CHANGELOG.md

@@ -3,6 +3,26 @@
 Included is a summary of changes to the project. For full details, especially on behind-the-scenes code changes and
 development tools, see the commit history.
 
+## v0.11.1
+
+### Miscellaneous
+
+* The storage size and subsequent tests have been updated to expect a 32 KB config section (rather than the previous 16
+  KB).
+
+## v0.11.0
+
+### Improvements
+
+* The usage of provided GP2040-CE config Protobuf files must be explicitly specified via the `-S` flag. This is in order
+  to not accidentally fall back to them and be confused by the results. The net effect is that in most situations, users
+  will have to provide either `-P PATH` or `-S`.
+* The testing of the above fallbacks covers the options far better than before now.
+
+### Miscellaneous
+
+* Version bumps, which brought about a couple updates to the tests.
+
 ## v0.10.0
 
 ### Features

README.md

@@ -232,11 +232,11 @@ The latest update of the configuration snapshot is from **v0.7.8**.
 ### Dumping the GP2040-CE board with picotool
 
 Some of these tools require a dump of your GP2040-CE board, either the storage section or the whole board, depending on
-the context. The storage section of a GP2040-CE board is a reserved 16 KB starting at `0x101FC000`. To dump your board's
-storage with picotool:
+the context. The user config storage section of a GP2040-CE board is a reserved 32 KB starting at `0x101F8000`. To dump
+your board's storage with picotool:
 
 ```
-% picotool save -r 101FC000 10200000 memory.bin
+% picotool save -r 101F8000 10200000 memory.bin
 ```
 
 And to dump your whole board:

gp2040ce_bintools/__init__.py

@@ -40,6 +40,8 @@ core_parser.add_argument('-d', '--debug', action='store_true', help="enable debu
 core_parser.add_argument('-P', '--proto-files-path', type=pathlib.Path, default=list(), action='append',
                          help="path to .proto files to read, including dependencies; you will likely need "
                               "to supply this twice, once for GP2040-CE's .proto files and once for nanopb's")
+core_parser.add_argument('-S', '--use-shipped-fallback', action='store_true',
+                         help="utilize shipped (potentially stale) .proto files because you can't supply your own")
 args, _ = core_parser.parse_known_args()
 for path in args.proto_files_path:
     sys.path.append(os.path.abspath(os.path.expanduser(path)))
@@ -50,21 +52,27 @@ else:
     handler.setLevel(logging.WARNING)
 
 
-def get_config_pb2():
+def get_config_pb2(with_fallback: bool = args.use_shipped_fallback):
     """Retrieve prebuilt _pb2 file or attempt to compile it live."""
     # try to just import a precompiled module if we have been given it in our path
     # (perhaps someone already compiled it for us for whatever reason)
     try:
+        logger.debug("Trying precompiled protobuf modules...")
         return importlib.import_module('config_pb2')
     except ModuleNotFoundError:
         # no found precompiled config, try to compile the proto files in realtime
         # because it's possible someone put them on the path
         try:
-            logger.info("Invoking gRPC tool to compile config.proto...")
+            logger.debug("No precompiled protobuf modules found, invoking gRPC tool to compile config.proto...")
             return grpc.protos('config.proto')
         except (ModuleNotFoundError, TypeError):
             # (TypeError could be the windows bug https://github.com/protocolbuffers/protobuf/issues/14345)
+            if not with_fallback:
+                logger.exception("no viable set of protobuf modules could be found, please use -P or -S!")
+                raise RuntimeError("no viable set of protobuf modules could be found, please use -P or -S!")
+
             # that failed, import the snapshot (may be lagging what's in GP2040-CE)
             logger.warning("using the fallback .proto files! please supply your files with -P if you can!")
             sys.path.append(os.path.join(pathlib.Path(__file__).parent.resolve(), 'proto_snapshot'))
+            logger.debug("Invoking gRPC tool again to compile shipped config.proto...")
             return grpc.protos('config.proto')

gp2040ce_bintools/storage.py

@@ -17,10 +17,11 @@ from gp2040ce_bintools.rp2040 import get_bootsel_endpoints, read
 
 logger = logging.getLogger(__name__)
 
-BOARD_CONFIG_BINARY_LOCATION = 0x1F8000
+STORAGE_SIZE = 32768
+
+BOARD_CONFIG_BINARY_LOCATION = (2 * 1024 * 1024) - (STORAGE_SIZE * 2)   # 0x1F0000
 BOARD_CONFIG_BOOTSEL_ADDRESS = 0x10000000 + BOARD_CONFIG_BINARY_LOCATION
-STORAGE_SIZE = 16384
-USER_CONFIG_BINARY_LOCATION = 0x1FC000
+USER_CONFIG_BINARY_LOCATION = (2 * 1024 * 1024) - STORAGE_SIZE          # 0x1F8000
 USER_CONFIG_BOOTSEL_ADDRESS = 0x10000000 + USER_CONFIG_BINARY_LOCATION
 
 FOOTER_SIZE = 12
@@ -425,7 +426,7 @@ def visualize():
     group.add_argument('--usb', action='store_true', help="retrieve the config from a RP2040 board connected over USB "
                                                           "and in BOOTSEL mode")
     group.add_argument('--filename', help=".bin file of a GP2040-CE board's storage section, bytes "
-                                          "101FC000-10200000, or of a GP2040-CE's whole board dump "
+                                          "101F8000-10200000, or of a GP2040-CE's whole board dump "
                                           "if --whole-board is specified")
     args, _ = parser.parse_known_args()
 

pyproject.toml

@@ -74,6 +74,10 @@ ignore_errors = true
 [tool.pytest]
 python_files = ["*_tests.py", "tests.py", "test_*.py"]
 
+[tool.pytest.ini_options]
+log_cli = 0
+log_cli_level = "WARNING"
+
 [tool.setuptools]
 packages = [
     "gp2040ce_bintools",

requirements/requirements-dev.txt

@@ -4,22 +4,22 @@
 #
 #    pip-compile --extra=dev --output-file=requirements/requirements-dev.txt
 #
-aiohappyeyeballs==2.4.2
+aiohappyeyeballs==2.4.4
     # via aiohttp
-aiohttp==3.10.8
+aiohttp==3.11.10
     # via
     #   aiohttp-jinja2
     #   textual-dev
     #   textual-serve
 aiohttp-jinja2==1.6
     # via textual-serve
-aiosignal==1.3.1
+aiosignal==1.3.2
     # via aiohttp
-attrs==24.2.0
+attrs==24.3.0
     # via
     #   aiohttp
     #   reuse
-bandit==1.7.10
+bandit==1.8.0
     # via gp2040ce-binary-tools (pyproject.toml)
 binaryornot==0.4.4
     # via reuse
@@ -27,11 +27,11 @@ boolean-py==4.0
     # via
     #   license-expression
     #   reuse
-build==1.2.2
+build==1.2.2.post1
     # via pip-tools
 cachetools==5.5.0
     # via tox
-certifi==2024.8.30
+certifi==2024.12.14
     # via requests
 cffi==1.17.1
     # via cryptography
@@ -40,21 +40,22 @@ chardet==5.2.0
     #   binaryornot
     #   python-debian
     #   tox
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via
     #   pip-tools
+    #   reuse
     #   textual-dev
 colorama==0.4.6
     # via tox
-coverage[toml]==7.6.1
+coverage[toml]==7.6.9
     # via pytest-cov
-cryptography==43.0.1
+cryptography==44.0.0
     # via secretstorage
 decorator==5.1.1
     # via gp2040ce-binary-tools (pyproject.toml)
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
 docutils==0.21.2
     # via readme-renderer
@@ -89,20 +90,18 @@ flake8-mutable==1.2.0
     # via gp2040ce-binary-tools (pyproject.toml)
 flake8-pyproject==1.2.3
     # via gp2040ce-binary-tools (pyproject.toml)
-frozenlist==1.4.1
+frozenlist==1.5.0
     # via
     #   aiohttp
     #   aiosignal
-grpcio==1.66.2
+grpcio==1.68.1
     # via grpcio-tools
-grpcio-tools==1.66.2
+grpcio-tools==1.68.1
     # via gp2040ce-binary-tools (pyproject.toml)
 idna==3.10
     # via
     #   requests
     #   yarl
-importlib-metadata==8.5.0
-    # via twine
 iniconfig==2.0.0
     # via pytest
 isort==5.13.2
@@ -122,9 +121,9 @@ jinja2==3.1.4
     #   aiohttp-jinja2
     #   reuse
     #   textual-serve
-keyring==25.4.1
+keyring==25.5.0
     # via twine
-license-expression==30.3.1
+license-expression==30.4.0
     # via reuse
 linkify-it-py==2.0.3
     # via markdown-it-py
@@ -133,7 +132,7 @@ markdown-it-py[linkify,plugins]==3.0.0
     #   mdit-py-plugins
     #   rich
     #   textual
-markupsafe==2.1.5
+markupsafe==3.0.2
     # via jinja2
 mccabe==0.7.0
     # via flake8
@@ -151,24 +150,25 @@ multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-mypy==1.11.2
+mypy==1.13.0
     # via gp2040ce-binary-tools (pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
-nh3==0.2.18
+nh3==0.2.20
     # via readme-renderer
-packaging==24.1
+packaging==24.2
     # via
     #   build
     #   pyproject-api
     #   pytest
     #   setuptools-scm
     #   tox
+    #   twine
 pbr==6.1.0
     # via stevedore
 pip-tools==7.4.1
     # via gp2040ce-binary-tools (pyproject.toml)
-pkginfo==1.10.0
+pkginfo==1.12.0
     # via twine
 platformdirs==4.3.6
     # via
@@ -179,7 +179,11 @@ pluggy==1.5.0
     # via
     #   pytest
     #   tox
-protobuf==5.28.2
+propcache==0.2.1
+    # via
+    #   aiohttp
+    #   yarl
+protobuf==5.29.1
     # via grpcio-tools
 pycodestyle==2.12.1
     # via flake8
@@ -199,14 +203,14 @@ pyproject-hooks==1.2.0
     # via
     #   build
     #   pip-tools
-pytest==8.3.3
+pytest==8.3.4
     # via
     #   gp2040ce-binary-tools (pyproject.toml)
     #   pytest-asyncio
     #   pytest-cov
-pytest-asyncio==0.24.0
+pytest-asyncio==0.25.0
     # via gp2040ce-binary-tools (pyproject.toml)
-pytest-cov==5.0.0
+pytest-cov==6.0.0
     # via gp2040ce-binary-tools (pyproject.toml)
 python-debian==0.1.49
     # via reuse
@@ -222,11 +226,11 @@ requests==2.32.3
     #   twine
 requests-toolbelt==1.0.0
     # via twine
-reuse==4.0.3
+reuse==5.0.2
     # via gp2040ce-binary-tools (pyproject.toml)
 rfc3986==2.0.0
     # via twine
-rich==13.8.1
+rich==13.9.4
     # via
     #   bandit
     #   textual
@@ -238,22 +242,22 @@ setuptools-scm==8.1.0
     # via gp2040ce-binary-tools (pyproject.toml)
 snowballstemmer==2.2.0
     # via pydocstyle
-stevedore==5.3.0
+stevedore==5.4.0
     # via bandit
-textual==0.81.0
+textual==1.0.0
     # via
     #   gp2040ce-binary-tools (pyproject.toml)
     #   textual-dev
     #   textual-serve
-textual-dev==1.6.1
+textual-dev==1.7.0
     # via gp2040ce-binary-tools (pyproject.toml)
 textual-serve==1.1.1
     # via textual-dev
 tomlkit==0.13.2
     # via reuse
-tox==4.20.0
+tox==4.23.2
     # via gp2040ce-binary-tools (pyproject.toml)
-twine==5.1.1
+twine==6.0.1
     # via gp2040ce-binary-tools (pyproject.toml)
 typing-extensions==4.12.2
     # via
@@ -266,14 +270,12 @@ urllib3==2.2.3
     # via
     #   requests
     #   twine
-virtualenv==20.26.6
+virtualenv==20.28.0
     # via tox
-wheel==0.44.0
+wheel==0.45.1
     # via pip-tools
-yarl==1.13.1
+yarl==1.18.3
     # via aiohttp
-zipp==3.20.2
-    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/requirements.txt

@@ -4,9 +4,9 @@
 #
 #    pip-compile --output-file=requirements/requirements.txt
 #
-grpcio==1.66.2
+grpcio==1.68.1
     # via grpcio-tools
-grpcio-tools==1.66.2
+grpcio-tools==1.68.1
     # via gp2040ce-binary-tools (pyproject.toml)
 linkify-it-py==2.0.3
     # via markdown-it-py
@@ -21,15 +21,15 @@ mdurl==0.1.2
     # via markdown-it-py
 platformdirs==4.3.6
     # via textual
-protobuf==5.28.2
+protobuf==5.29.1
     # via grpcio-tools
 pygments==2.18.0
     # via rich
 pyusb==1.2.1
     # via gp2040ce-binary-tools (pyproject.toml)
-rich==13.8.1
+rich==13.9.4
     # via textual
-textual==0.81.0
+textual==1.0.0
     # via gp2040ce-binary-tools (pyproject.toml)
 typing-extensions==4.12.2
     # via textual

tests/conftest.py

@@ -42,7 +42,7 @@ def firmware_binary():
 
 @pytest.fixture
 def storage_dump():
-    """Read in a test storage dump file (101FC000-10200000) of a GP2040-CE board."""
+    """Read in a test storage dump file (101F8000-10200000) of a GP2040-CE board."""
     filename = os.path.join(HERE, 'test-files', 'test-storage-area.bin')
     with open(filename, 'rb') as file:
         content = file.read()

tests/test-files/pb2-files/nanopb_pb2.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: nanopb.proto
+# Protobuf Python Version: 5.27.2
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    5,
+    27,
+    2,
+    '',
+    'nanopb.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\xc4\x07\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x12\n\nmax_length\x18\x0e \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12&\n\x08int_size\x18\x07 \x01(\x0e\x32\x08.IntSize:\nIS_DEFAULT\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\x1a\n\x0bpacked_enum\x18\n \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0cskip_message\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\x18\n\tno_unions\x18\x08 \x01(\x08:\x05\x66\x61lse\x12\r\n\x05msgid\x18\t \x01(\r\x12\x1e\n\x0f\x61nonymous_oneof\x18\x0b \x01(\x08:\x05\x66\x61lse\x12\x15\n\x06proto3\x18\x0c \x01(\x08:\x05\x66\x61lse\x12#\n\x14proto3_singular_msgs\x18\x15 \x01(\x08:\x05\x66\x61lse\x12\x1d\n\x0e\x65num_to_string\x18\r \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0c\x66ixed_length\x18\x0f \x01(\x08:\x05\x66\x61lse\x12\x1a\n\x0b\x66ixed_count\x18\x10 \x01(\x08:\x05\x66\x61lse\x12\x1e\n\x0fsubmsg_callback\x18\x16 \x01(\x08:\x05\x66\x61lse\x12/\n\x0cmangle_names\x18\x11 \x01(\x0e\x32\x11.TypenameMangling:\x06M_NONE\x12(\n\x11\x63\x61llback_datatype\x18\x12 \x01(\t:\rpb_callback_t\x12\x34\n\x11\x63\x61llback_function\x18\x13 \x01(\t:\x19pb_default_field_callback\x12\x30\n\x0e\x64\x65scriptorsize\x18\x14 \x01(\x0e\x32\x0f.DescriptorSize:\x07\x44S_AUTO\x12\x1a\n\x0b\x64\x65\x66\x61ult_has\x18\x17 \x01(\x08:\x05\x66\x61lse\x12\x0f\n\x07include\x18\x18 \x03(\t\x12\x0f\n\x07\x65xclude\x18\x1a \x03(\t\x12\x0f\n\x07package\x18\x19 \x01(\t\x12\x41\n\rtype_override\x18\x1b \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.Type\x12\x19\n\x0bsort_by_tag\x18\x1c \x01(\x08:\x04true\x12.\n\rfallback_type\x18\x1d \x01(\x0e\x32\n.FieldType:\x0b\x46T_CALLBACK\x12\x1e\n\x0f\x64isallow_export\x18\x1e \x01(\x08:\x05\x66\x61lse*i\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03\x12\r\n\tFT_INLINE\x10\x05*D\n\x07IntSize\x12\x0e\n\nIS_DEFAULT\x10\x00\x12\x08\n\x04IS_8\x10\x08\x12\t\n\x05IS_16\x10\x10\x12\t\n\x05IS_32\x10 \x12\t\n\x05IS_64\x10@*Z\n\x10TypenameMangling\x12\n\n\x06M_NONE\x10\x00\x12\x13\n\x0fM_STRIP_PACKAGE\x10\x01\x12\r\n\tM_FLATTEN\x10\x02\x12\x16\n\x12M_PACKAGE_INITIALS\x10\x03*E\n\x0e\x44\x65scriptorSize\x12\x0b\n\x07\x44S_AUTO\x10\x00\x12\x08\n\x04\x44S_1\x10\x01\x12\x08\n\x04\x44S_2\x10\x02\x12\x08\n\x04\x44S_4\x10\x04\x12\x08\n\x04\x44S_8\x10\x08:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'nanopb_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\030fi.kapsi.koti.jpa.nanopb'
+  _globals['_FIELDTYPE']._serialized_start=1017
+  _globals['_FIELDTYPE']._serialized_end=1122
+  _globals['_INTSIZE']._serialized_start=1124
+  _globals['_INTSIZE']._serialized_end=1192
+  _globals['_TYPENAMEMANGLING']._serialized_start=1194
+  _globals['_TYPENAMEMANGLING']._serialized_end=1284
+  _globals['_DESCRIPTORSIZE']._serialized_start=1286
+  _globals['_DESCRIPTORSIZE']._serialized_end=1355
+  _globals['_NANOPBOPTIONS']._serialized_start=51
+  _globals['_NANOPBOPTIONS']._serialized_end=1015
+# @@protoc_insertion_point(module_scope)

tests/test_builder.py

@@ -25,13 +25,15 @@ logger = logging.getLogger(__name__)
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_concatenate_to_file(tmp_path):
@@ -55,7 +57,7 @@ def test_concatenate_board_config_to_file(tmp_path):
                                                    combined_filename=tmp_file)
     with open(tmp_file, 'rb') as file:
         content = file.read()
-    assert len(content) == (2 * 1024 * 1024) - (16 * 1024)
+    assert len(content) == (2 * 1024 * 1024) - (32 * 1024)
 
 
 def test_concatenate_both_configs_to_file(tmp_path):
@@ -178,19 +180,19 @@ def test_dont_always_find_version_string(firmware_binary):
 def test_padding_firmware(firmware_binary):
     """Test that firmware is padded to the expected size."""
     padded = builder.pad_binary_up_to_user_config(firmware_binary)
-    assert len(padded) == 2080768
+    assert len(padded) == 2064384
 
 
 def test_padding_firmware_can_truncate():
     """Test that firmware is padded to the expected size."""
     padded = builder.pad_binary_up_to_user_config(bytearray(b'\x00' * 4 * 1024 * 1024), or_truncate=True)
-    assert len(padded) == 2080768
+    assert len(padded) == 2064384
 
 
 def test_padding_firmware_to_board(firmware_binary):
     """Test that firmware is padded to the expected size."""
     padded = builder.pad_binary_up_to_board_config(firmware_binary)
-    assert len(padded) == 2080768 - (16 * 1024)
+    assert len(padded) == 2064384 - (32 * 1024)
 
 
 def test_firmware_plus_storage_section(firmware_binary, storage_dump):
@@ -224,7 +226,7 @@ def test_chunky_firmware_plus_user_config_binary(config_binary):
 def test_firmware_plus_board_config_binary(firmware_binary, config_binary):
     """Test that combining firmware and board config produces a valid combined binary."""
     almost_whole_board = builder.combine_firmware_and_config(firmware_binary, config_binary, None)
-    assert len(almost_whole_board) == (2 * 1024 * 1024) - (16 * 1024)
+    assert len(almost_whole_board) == (2 * 1024 * 1024) - (32 * 1024)
     # if this is valid, we should be able to find the storage and footer again
     storage = get_board_storage_section(almost_whole_board)
     footer_size, _, _ = get_config_footer(storage)

tests/test_commands.py

@@ -18,13 +18,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_version_flag():
@@ -49,7 +51,7 @@ def test_concatenate_invocation(tmpdir):
     with open(out_filename, 'rb') as out_file, open('tests/test-files/test-storage-area.bin', 'rb') as storage_file:
         out = out_file.read()
         storage = storage_file.read()
-    assert out[2080768:2097152] == storage
+    assert out[2064384:2097152] == storage
 
 
 def test_concatenate_invocation_json(tmpdir):
@@ -86,7 +88,7 @@ def test_debug_storage_dump_invocation():
                   '--filename', 'tests/test-files/test-storage-area.bin'],
                  capture_output=True, encoding='utf8')
     assert 'boardVersion: "v0.7.5"' in result.stdout
-    assert 'length of content to look for footer in: 16384' in result.stderr
+    assert 'length of content to look for footer in: 32768' in result.stderr
 
 
 def test_storage_dump_json_invocation():

tests/test_gui.py

@@ -21,13 +21,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 async def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     await test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 @pytest.mark.asyncio
@@ -119,7 +121,7 @@ async def test_simple_edit_via_input_field():
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Input#field-input')
         await pilot.wait_for_scheduled_animations()
-        await pilot.press('backspace', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', '5')
+        await pilot.press('end', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', '5')
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Button#confirm-button')
         assert pilot.app.config.displayOptions.deprecatedI2cSpeed == 5
@@ -199,7 +201,7 @@ async def test_simple_edit_via_input_field_string():
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Input#field-input')
         await pilot.wait_for_scheduled_animations()
-        await pilot.press('backspace', '-', 'h', 'i')
+        await pilot.press('end', 'backspace', '-', 'h', 'i')
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Button#confirm-button')
         assert pilot.app.config.boardVersion == 'v0.7.-hi'
@@ -229,7 +231,7 @@ async def test_add_node_to_repeated():
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Input#field-input')
         await pilot.wait_for_scheduled_animations()
-        await pilot.press('backspace', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', '5')
+        await pilot.press('end', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', 'backspace', '5')
         await pilot.wait_for_scheduled_animations()
         await pilot.click('Button#confirm-button')
 

tests/test_package.py

@@ -6,43 +6,69 @@ SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import sys
 
+import pytest
+from decorator import decorator
+
 from gp2040ce_bintools import get_config_pb2
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
 
-def test_get_config_pb2_compile():
-    """Without any precompiled files on the path, test we can read proto files and compile them."""
-    # append to path as -P would
+@decorator
+def with_pb2s(test, *args, **kwargs):
+    """Wrap a test with precompiled pb2 files on the path."""
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
+    sys.path.append(proto_path)
+
+    test(*args, **kwargs)
+
+    sys.path.pop()
+    del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
+
+
+@decorator
+def with_protos(test, *args, **kwargs):
+    """Wrap a test with .proto files on the path."""
     proto_path = os.path.join(HERE, 'test-files', 'proto-files')
     sys.path.append(proto_path)
 
-    # let grpc tools compile the proto files on demand and give us the module
-    config_pb2 = get_config_pb2()
-    _ = config_pb2.Config()
+    test(*args, **kwargs)
 
-    # clean up the path and unload config_pb2
-    sys.path.pop()
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
+
+
+@with_pb2s
+def test_get_config_pb2_precompiled():
+    """With precompiled files on the path, test we can read and use them."""
+    # get the module from the provided files
+    config_pb2 = get_config_pb2()
+    _ = config_pb2.Config()
 
 
 def test_get_config_pb2_exception():
+    """Test that we fail if no config .proto files are available."""
+    with pytest.raises(RuntimeError):
+        _ = get_config_pb2()
+
+
+def test_get_config_pb2_shipped_config_files():
     """Without any precompiled files or proto files on the path, test we DO NOT raise an exception."""
-    # this used to raise ModuleNotFoundError, but with our snapshot included now,
-    # we should always have a config to import
-    _ = get_config_pb2()
+    # use the shipped .proto files to generate the config
+    config_pb2 = get_config_pb2(with_fallback=True)
+    _ = config_pb2.Config()
+    del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
-def test_get_config_pb2_precompile():
-    """Test we can import precompiled protobuf files."""
-    proto_path = os.path.join(HERE, 'test-files')
-    sys.path.append(proto_path)
-
-    # let grpc tools import the proto files normally
+@with_protos
+def test_get_config_pb2_compile():
+    """Without any precompiled files on the path, test we can read proto files and compile them."""
+    # let grpc tools compile the proto files on demand and give us the module
     config_pb2 = get_config_pb2()
     _ = config_pb2.Config()
-
-    # clean up the path and unload config_pb2
-    sys.path.pop()
-    del sys.modules['config_pb2']

tests/test_rp2040.py

@@ -20,13 +20,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_get_bootsel_endpoints():

tests/test_storage.py

@@ -20,13 +20,15 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 @decorator
 def with_pb2s(test, *args, **kwargs):
     """Wrap a test with precompiled pb2 files on the path."""
-    proto_path = os.path.join(HERE, 'test-files')
+    proto_path = os.path.join(HERE, 'test-files', 'pb2-files')
     sys.path.append(proto_path)
 
     test(*args, **kwargs)
 
     sys.path.pop()
     del sys.modules['config_pb2']
+    del sys.modules['enums_pb2']
+    del sys.modules['nanopb_pb2']
 
 
 def test_config_footer(storage_dump):
@@ -95,7 +97,7 @@ def test_get_board_config_from_file_whole_board_dump():
     """Test that we can open a storage dump file and find its config."""
     filename = os.path.join(HERE, 'test-files', 'test-whole-board-with-board-config.bin')
     config = storage.get_config_from_file(filename, whole_board=True, board_config=True)
-    assert config.boardVersion == 'v0.7.6-15-g71f4512'
+    assert config.boardVersion == 'v0.7.8'
     assert config.addonOptions.bootselButtonOptions.enabled is False
 
 
@@ -165,7 +167,7 @@ def test_convert_binary_to_uf2_with_offsets(whole_board_with_board_config_dump):
     assert len(uf2) == 4194304                              # binary is 8192 256 byte chunks, UF2 is 512 b per chunk
     assert uf2[0:4] == b'\x55\x46\x32\x0a' == b'UF2\n'      # proper magic
     assert uf2[8:12] == bytearray(b'\x00\x20\x00\x00')      # family ID set
-    assert uf2[524:528] == bytearray(b'\x00\xc1\x1f\x10')   # address to write the second chunk
+    assert uf2[524:528] == bytearray(b'\x00\x81\x1f\x10')   # address to write the second chunk
 
 
 def test_convert_binary_to_uf2_to_binary(whole_board_with_board_config_dump):
@@ -213,7 +215,7 @@ def test_read_created_uf2(tmp_path, firmware_binary, config_binary):
     binary = storage.convert_uf2_to_binary(content)
     # the converted binary should be aligned properly and of the right size
     assert len(binary) == 2 * 1024 * 1024
-    assert binary[-16384-4:-16384] == storage.FOOTER_MAGIC
+    assert binary[-32768-4:-32768] == storage.FOOTER_MAGIC
     assert binary[-4:] == storage.FOOTER_MAGIC
     user_storage = storage.get_user_storage_section(binary)
     footer_size, _, _ = storage.get_config_footer(user_storage)
@@ -255,13 +257,13 @@ def test_serialize_modified_config_with_footer(storage_dump):
 def test_pad_config_to_storage(config_binary):
     """Test that we can properly pad a config section to the correct storage section size."""
     storage_section = storage.pad_config_to_storage_size(config_binary)
-    assert len(storage_section) == 16384
+    assert len(storage_section) == 32768
 
 
 def test_pad_config_to_storage_raises(config_binary):
     """Test that we raise an exception if the config is bigger than the storage section."""
     with pytest.raises(storage.ConfigLengthError):
-        _ = storage.pad_config_to_storage_size(config_binary * 5)
+        _ = storage.pad_config_to_storage_size(config_binary * 10)
 
 
 @with_pb2s
@@ -278,7 +280,7 @@ def test_get_board_config_from_usb(config_binary):
             config, _, _ = storage.get_board_config_from_usb()
 
     mock_get.assert_called_once()
-    mock_read.assert_called_with(mock_out, mock_in, 0x101F8000, 16384)
+    mock_read.assert_called_with(mock_out, mock_in, 0x101F0000, 32768)
     assert config == storage.get_config(config_binary)
 
 
@@ -296,7 +298,7 @@ def test_get_user_config_from_usb(config_binary):
             config, _, _ = storage.get_user_config_from_usb()
 
     mock_get.assert_called_once()
-    mock_read.assert_called_with(mock_out, mock_in, 0x101FC000, 16384)
+    mock_read.assert_called_with(mock_out, mock_in, 0x101F8000, 32768)
     assert config == storage.get_config(config_binary)