option to get the config out of a whole board dump

now you don't need to fiddle with specific byte ranges of a dump, you
can just dump the whole board if that's more convenient, and
visualize-storage will parse that

also more testing in general

README.md

@@ -27,13 +27,14 @@ As above, plus also `pip install -Ur requirements/requirements-dev.txt` to get l
 
 ### visualize-storage
 
-**visualize-storage** reads a dump of a GP2040-CE board's flash storage section, where the configuration lives,
-and prints it out for visual inspection or diffing with other tools. Usage is simple; just pass the tool your
-binary file to analyze along with the path to the Protobuf files.
+**visualize-storage** reads a dump of a GP2040-CE board's flash storage section, where the configuration lives, and
+prints it out for visual inspection or diffing with other tools. It can also find the storage section from a GP2040-CE
+whole board dump, if you have that instead. Usage is simple; just pass the tool your binary file to analyze along with
+the path to the Protobuf files.
 
-Because Protobuf relies on .proto files to convey the serialized structure, you must supply them
-from the main GP2040-CE project, e.g. pointing this tool at your clone of the core project. Something like
-this would suffice for a working invocation (note: you do not need to compile the files yourself):
+Because Protobuf relies on .proto files to convey the serialized structure, you must supply them from the main GP2040-CE
+project, e.g. pointing this tool at your clone of the core project. Something like this would suffice for a working
+invocation (note: you do not need to compile the files yourself):
 
 ```
 % visualize-storage -P ~/proj/GP2040-CE/proto -P ~/proj/GP2040-CE/lib/nanopb/generator/proto memory.bin
@@ -110,10 +111,16 @@ forcedSetupOptions {
 }
 ```
 
-### Dumping the storage section
+### Dumping the GP2040-CE board
 
-The storage section of a GP2040-CE board is a reserved 8 KB starting at `0x101FE000`. To dump your board's storage:
+These tools require a dump of your GP2040-CE board, either the storage section or the whole board, depending on the
+context. The storage section of a GP2040-CE board is a reserved 8 KB starting at `0x101FE000`. To dump your board's storage:
 
 ```
 % picotool save -r 101FE000 10200000 memory.bin
 ```
+
+And to dump your whole board:
+```
+% picotool save -a whole-board.bin
+```

gp2040ce_bintools/storage.py

@@ -6,6 +6,9 @@ from gp2040ce_bintools import core_parser, get_config_pb2
 
 logger = logging.getLogger(__name__)
 
+STORAGE_LOCATION = 0x1FE000
+STORAGE_SIZE = 8192
+
 FOOTER_SIZE = 12
 FOOTER_MAGIC = b'\x65\xe3\xf1\xd2'
 
@@ -43,10 +46,12 @@ def get_config_footer(content: bytes) -> tuple[int, int, str]:
         ValueError: if the provided bytes are not a config footer
     """
     # last 12 bytes are the footer
+    logger.debug("length of content to look for footer in: %s", len(content))
     if len(content) < FOOTER_SIZE:
         raise ValueError("provided content is not large enough to have a config footer!")
 
     footer = content[-FOOTER_SIZE:]
+    logger.debug("suspected footer magic: %s", footer[-4:])
     if footer[-4:] != FOOTER_MAGIC:
         raise ValueError("content's magic is not as expected!")
 
@@ -62,6 +67,39 @@ def get_config_footer(content: bytes) -> tuple[int, int, str]:
     return config_size, config_crc, config_magic
 
 
+def get_config_from_file(filename: str, whole_board: bool = False) -> dict:
+    """Read the specified file (memory dump or whole board dump) and get back its config section.
+
+    Args:
+        filename: the filename of the file to open and read
+    Returns:
+        the parsed configuration
+    """
+    with open(filename, 'rb') as dump:
+        content = dump.read()
+
+    if whole_board:
+        return get_config(get_storage_section(content))
+    else:
+        return get_config(content)
+
+
+def get_storage_section(content: bytes) -> bytes:
+    """Pull out what should be the GP2040-CE storage section from a whole board dump.
+
+    Args:
+        content: bytes of a GP2040-CE whole board dump
+    Returns:
+        the presumed storage section from the binary
+    """
+    # a whole board must be at least as big as the known fences
+    logger.debug("length of content to look for storage in: %s", len(content))
+    if len(content) < STORAGE_LOCATION + STORAGE_SIZE:
+        raise ValueError("provided content is not large enough to have a storage section!")
+
+    logger.debug("returning bytes from %s to %s", hex(STORAGE_LOCATION), hex(STORAGE_LOCATION + STORAGE_SIZE))
+    return content[STORAGE_LOCATION:(STORAGE_LOCATION + STORAGE_SIZE)]
+
 ############
 # COMMANDS #
 ############
@@ -74,10 +112,8 @@ def visualize():
                     "its contents.",
         parents=[core_parser],
     )
-    parser.add_argument('filename', help=".bin file of a GP2040-CE board's storage section, bytes 101FE000-10200000")
+    parser.add_argument('--whole-board', action='store_true', help="indicate the binary file is a whole board dump")
+    parser.add_argument('filename', help=".bin file of a GP2040-CE board's storage section, bytes 101FE000-10200000, "
+                                         "or of a GP2040-CE's whole board dump if --whole-board is specified")
     args, _ = parser.parse_known_args()
-    with open(args.filename, 'rb') as dump:
-        content = dump.read()
-
-    config = get_config(content)
-    print(config)
+    print(get_config_from_file(args.filename, whole_board=args.whole_board))

pyproject.toml

@@ -28,6 +28,9 @@ extend-ignore = "T101"
 max-complexity = 10
 max-line-length = 120
 
+[tool.isort]
+line_length = 120
+
 [tool.mypy]
 ignore_missing_imports = true
 

tests/conftest.py

@@ -14,3 +14,13 @@ def storage_dump():
         content = file.read()
 
     yield content
+
+
+@pytest.fixture
+def whole_board_dump():
+    """Read in a test whole board dump file of a GP2040-CE board."""
+    filename = os.path.join(HERE, 'test-files', 'test-whole-board.bin')
+    with open(filename, 'rb') as file:
+        content = file.read()
+
+    yield content

tests/test_commands.py

@@ -0,0 +1,17 @@
+"""Test our tools themselves to make sure they adhere to certain flags."""
+from subprocess import run
+
+from gp2040ce_bintools import __version__
+
+
+def test_version_flag():
+    """Test that tools report the version."""
+    result = run(['visualize-storage', '-v'], capture_output=True, encoding='utf8')
+    assert __version__ in result.stdout
+
+
+def test_help_flag():
+    """Test that tools report the usage information."""
+    result = run(['visualize-storage', '-h'], capture_output=True, encoding='utf8')
+    assert 'usage: visualize-storage' in result.stdout
+    assert 'Read the configuration section from a dump of a GP2040-CE board' in result.stdout

tests/test_storage.py

@@ -5,7 +5,7 @@ import sys
 import pytest
 from decorator import decorator
 
-from gp2040ce_bintools.storage import get_config, get_config_footer
+from gp2040ce_bintools.storage import get_config, get_config_footer, get_config_from_file, get_storage_section
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
@@ -33,13 +33,19 @@ def test_config_footer(storage_dump):
 def test_config_footer_way_too_small(storage_dump):
     """Test that a config footer isn't detected if the size is way too small."""
     with pytest.raises(ValueError):
-        size, crc, magic = get_config_footer(storage_dump[-11:])
+        _, _, _ = get_config_footer(storage_dump[-11:])
 
 
 def test_config_footer_too_small(storage_dump):
     """Test that a config footer isn't detected if the size is smaller than that found in the header."""
     with pytest.raises(ValueError):
-        size, crc, magic = get_config_footer(storage_dump[-1000:])
+        _, _, _ = get_config_footer(storage_dump[-1000:])
+
+
+def test_whole_board_too_small(whole_board_dump):
+    """Test that a storage section isn't detected if the size is too small to contain where it should be."""
+    with pytest.raises(ValueError):
+        _, _, _ = get_storage_section(whole_board_dump[-100000:])
 
 
 def test_config_footer_bad_magic(storage_dump):
@@ -47,7 +53,7 @@ def test_config_footer_bad_magic(storage_dump):
     unmagical = bytearray(storage_dump)
     unmagical[-1] = 0
     with pytest.raises(ValueError):
-        size, crc, magic = get_config_footer(unmagical)
+        _, _, _ = get_config_footer(unmagical)
 
 
 def test_config_fails_without_pb2s(storage_dump):
@@ -56,9 +62,35 @@ def test_config_fails_without_pb2s(storage_dump):
         _ = get_config(storage_dump)
 
 
+@with_pb2s
+def test_get_config_from_file_storage_dump():
+    """Test that we can open a storage dump file and find its config."""
+    filename = os.path.join(HERE, 'test-files', 'test-storage-area.bin')
+    config = get_config_from_file(filename)
+    assert config.boardVersion == 'v0.7.2'
+    assert config.addonOptions.bootselButtonOptions.enabled is False
+
+
+@with_pb2s
+def test_get_config_from_file_whole_board_dump():
+    """Test that we can open a storage dump file and find its config."""
+    filename = os.path.join(HERE, 'test-files', 'test-whole-board.bin')
+    config = get_config_from_file(filename, whole_board=True)
+    assert config.boardVersion == 'v0.7.2'
+    assert config.addonOptions.bootselButtonOptions.enabled is False
+
+
 @with_pb2s
 def test_config_parses(storage_dump):
     """Test that we need the config_pb2 to exist/be compiled for reading the config to work."""
     config = get_config(storage_dump)
     assert config.boardVersion == 'v0.7.2'
     assert config.hotkeyOptions.hotkeyF1Up.dpadMask == 1
+
+
+@with_pb2s
+def test_config_from_whole_board_parses(whole_board_dump):
+    """Test that we can read in a whole board and still find the config section."""
+    config = get_config(get_storage_section(whole_board_dump))
+    assert config.boardVersion == 'v0.7.2'
+    assert config.hotkeyOptions.hotkeyF1Up.dpadMask == 1

tox.ini

@@ -37,7 +37,7 @@ commands =
 # report on coverage runs from above
 skip_install = true
 commands =
-    coverage report --fail-under=80 --show-missing
+    coverage report --fail-under=90 --show-missing
 
 [testenv:bandit]
 commands =