--usb flag to visualize-storage direct off the board

README.md

@@ -66,17 +66,17 @@ Sample usage:
 
 ### visualize-storage
 
-**visualize-storage** reads a dump of a GP2040-CE board's flash storage section, where the configuration lives, and
+**visualize-storage** reads a GP2040-CE board's configuration, either over USB or from a dump of the board's flash
-prints it out for visual inspection or diffing with other tools. It can also find the storage section from a GP2040-CE
+storage section, and prints it out for visual inspection or diffing with other tools. It can also find the storage
-whole board dump, if you have that instead. Usage is simple; just pass the tool your binary file to analyze along with
+section from a GP2040-CE whole board dump, if you have that instead. Usage is simple; just connect your board in BOOTSEL
-the path to the Protobuf files.
+mode or pass the tool your binary file to analyze along with the path to the Protobuf files.
 
 Because Protobuf relies on .proto files to convey the serialized structure, you must supply them from the main GP2040-CE
 project, e.g. pointing this tool at your clone of the core project. Something like this would suffice for a working
 invocation (note: you do not need to compile the files yourself):
 
 ```
-% visualize-storage -P ~/proj/GP2040-CE/proto -P ~/proj/GP2040-CE/lib/nanopb/generator/proto memory.bin
+% visualize-storage -P ~/proj/GP2040-CE/proto -P ~/proj/GP2040-CE/lib/nanopb/generator/proto --filename memory.bin
 ```
 
 (In the future we will look into publishing complete packages that include the compiled `_pb2.py` files, so that you
@@ -85,7 +85,7 @@ don't need to provide them yourself.)
 Sample output:
 
 ```
-% visualize-storage -P ~/proj/GP2040-CE/proto -P ~/proj/GP2040-CE/lib/nanopb/generator/proto ~/proj/GP2040-CE/demo-memory.bin
+% visualize-storage -P ~/proj/GP2040-CE/proto -P ~/proj/GP2040-CE/lib/nanopb/generator/proto --usb
 boardVersion: "v0.7.2"
 gamepadOptions {
   inputMode: INPUT_MODE_HID
@@ -152,8 +152,9 @@ forcedSetupOptions {
 
 ### Dumping the GP2040-CE board
 
-These tools require a dump of your GP2040-CE board, either the storage section or the whole board, depending on the
+Some of these tools require a dump of your GP2040-CE board, either the storage section or the whole board, depending on
-context. The storage section of a GP2040-CE board is a reserved 8 KB starting at `0x101FE000`. To dump your board's storage:
+the context. The storage section of a GP2040-CE board is a reserved 8 KB starting at `0x101FE000`. To dump your board's
+storage:
 
 ```
 % picotool save -r 101FE000 10200000 memory.bin

gp2040ce_bintools/builder.py

@@ -6,7 +6,7 @@ import logging
 from google.protobuf.message import Message
 
 from gp2040ce_bintools import core_parser
-from gp2040ce_bintools.storage import (STORAGE_LOCATION, STORAGE_SIZE, pad_config_to_storage_size,
+from gp2040ce_bintools.storage import (STORAGE_BINARY_LOCATION, STORAGE_SIZE, pad_config_to_storage_size,
                                        serialize_config_with_footer)
 
 logger = logging.getLogger(__name__)
@@ -57,11 +57,11 @@ def pad_firmware_up_to_storage(firmware: bytes) -> bytearray:
     Raises:
         FirmwareLengthError: if the firmware is larger than the storage location
     """
-    bytes_to_pad = STORAGE_LOCATION - len(firmware)
+    bytes_to_pad = STORAGE_BINARY_LOCATION - len(firmware)
     logger.debug("firmware is length %s, padding %s bytes", len(firmware), bytes_to_pad)
     if bytes_to_pad < 0:
         raise FirmwareLengthError(f"provided firmware binary is larger than the start of "
-                                  f"storage at {STORAGE_LOCATION}!")
+                                  f"storage at {STORAGE_BINARY_LOCATION}!")
 
     return bytearray(firmware) + bytearray(b'\x00' * bytes_to_pad)
 
@@ -79,12 +79,13 @@ def replace_config_in_binary(board_binary: bytearray, config_binary: bytearray)
     Returns:
         the resulting correctly-offset binary suitable for a GP2040-CE board
     """
-    if len(board_binary) < STORAGE_LOCATION + STORAGE_SIZE:
+    if len(board_binary) < STORAGE_BINARY_LOCATION + STORAGE_SIZE:
         # this is functionally the same, since this doesn't sanity check the firmware
         return combine_firmware_and_config(board_binary, config_binary)
     else:
         new_binary = bytearray(copy.copy(board_binary))
-        new_binary[STORAGE_LOCATION:(STORAGE_LOCATION + STORAGE_SIZE)] = pad_config_to_storage_size(config_binary)
+        new_config = pad_config_to_storage_size(config_binary)
+        new_binary[STORAGE_BINARY_LOCATION:(STORAGE_BINARY_LOCATION + STORAGE_SIZE)] = new_config
         return new_binary
 
 

gp2040ce_bintools/storage.py

@@ -7,10 +7,12 @@ from google.protobuf.json_format import MessageToJson
 from google.protobuf.message import Message
 
 from gp2040ce_bintools import core_parser, get_config_pb2
+from gp2040ce_bintools.pico import get_bootsel_endpoints, read
 
 logger = logging.getLogger(__name__)
 
-STORAGE_LOCATION = 0x1FE000
+STORAGE_BINARY_LOCATION = 0x1FE000
+STORAGE_MEMORY_ADDRESS = 0x101FE000
 STORAGE_SIZE = 8192
 
 FOOTER_SIZE = 12
@@ -126,11 +128,12 @@ def get_storage_section(content: bytes) -> bytes:
     """
     # a whole board must be at least as big as the known fences
     logger.debug("length of content to look for storage in: %s", len(content))
-    if len(content) < STORAGE_LOCATION + STORAGE_SIZE:
+    if len(content) < STORAGE_BINARY_LOCATION + STORAGE_SIZE:
         raise ConfigLengthError("provided content is not large enough to have a storage section!")
 
-    logger.debug("returning bytes from %s to %s", hex(STORAGE_LOCATION), hex(STORAGE_LOCATION + STORAGE_SIZE))
+    logger.debug("returning bytes from %s to %s", hex(STORAGE_BINARY_LOCATION),
-    return content[STORAGE_LOCATION:(STORAGE_LOCATION + STORAGE_SIZE)]
+                 hex(STORAGE_BINARY_LOCATION + STORAGE_SIZE))
+    return content[STORAGE_BINARY_LOCATION:(STORAGE_BINARY_LOCATION + STORAGE_SIZE)]
 
 
 def pad_config_to_storage_size(config: bytes) -> bytearray:
@@ -174,12 +177,23 @@ def visualize():
                     "its contents.",
         parents=[core_parser],
     )
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--usb', action='store_true', help="retrieve the config from a Pico board connected over USB "
+                                                          "and in BOOTSEL mode")
+    group.add_argument('--filename', help=".bin file of a GP2040-CE board's storage section, bytes "
+                                          "101FE000-10200000, or of a GP2040-CE's whole board dump "
+                                          "if --whole-board is specified")
     parser.add_argument('--whole-board', action='store_true', help="indicate the binary file is a whole board dump")
     parser.add_argument('--json', action='store_true', help="print the config out as a JSON document")
-    parser.add_argument('filename', help=".bin file of a GP2040-CE board's storage section, bytes 101FE000-10200000, "
-                                         "or of a GP2040-CE's whole board dump if --whole-board is specified")
     args, _ = parser.parse_known_args()
-    config = get_config_from_file(args.filename, whole_board=args.whole_board)
+
+    if args.usb:
+        endpoint_out, endpoint_in = get_bootsel_endpoints()
+        storage = read(endpoint_out, endpoint_in, STORAGE_MEMORY_ADDRESS, STORAGE_SIZE)
+        config = get_config(bytes(storage))
+    else:
+        config = get_config_from_file(args.filename, whole_board=args.whole_board)
+
     if args.json:
         print(MessageToJson(config))
     else:

tests/test_commands.py

@@ -1,6 +1,7 @@
 """Test our tools themselves to make sure they adhere to certain flags."""
 import json
 import os
+import unittest.mock as mock
 from subprocess import run
 
 from gp2040ce_bintools import __version__
@@ -32,7 +33,8 @@ def test_concatenate_invocation(tmpdir):
 
 def test_storage_dump_invocation():
     """Test that a normal invocation against a dump works."""
-    result = run(['visualize-storage', '-P', 'tests/test-files/proto-files', 'tests/test-files/test-storage-area.bin'],
+    result = run(['visualize-storage', '-P', 'tests/test-files/proto-files',
+                  '--filename', 'tests/test-files/test-storage-area.bin'],
                  capture_output=True, encoding='utf8')
     assert 'boardVersion: "v0.7.2"' in result.stdout
 
@@ -40,7 +42,7 @@ def test_storage_dump_invocation():
 def test_debug_storage_dump_invocation():
     """Test that a normal invocation against a dump works."""
     result = run(['visualize-storage', '-d', '-P', 'tests/test-files/proto-files',
-                  'tests/test-files/test-storage-area.bin'],
+                  '--filename', 'tests/test-files/test-storage-area.bin'],
                  capture_output=True, encoding='utf8')
     assert 'boardVersion: "v0.7.2"' in result.stdout
     assert 'length of content to look for footer in: 8192' in result.stderr
@@ -49,7 +51,16 @@ def test_debug_storage_dump_invocation():
 def test_storage_dump_json_invocation():
     """Test that a normal invocation against a dump works."""
     result = run(['visualize-storage', '-P', 'tests/test-files/proto-files', '--json',
-                  'tests/test-files/test-storage-area.bin'],
+                  '--filename', 'tests/test-files/test-storage-area.bin'],
                  capture_output=True, encoding='utf8')
     to_dict = json.loads(result.stdout)
     assert to_dict['boardVersion'] == 'v0.7.2'
+
+
+def test_visualize_usb_invocation(storage_dump):
+    """Test that a normal invocation against a dump works."""
+    with mock.patch('gp2040ce_bintools.pico.get_bootsel_endpoints', return_value=(mock.MagicMock(), mock.MagicMock())):
+        with mock.patch('gp2040ce_bintools.pico.read', return_value=storage_dump):
+            result = run(['visualize-storage', '-P', 'tests/test-files/proto-files', '--usb'],
+                         capture_output=True, encoding='utf8')
+    assert 'boardVersion: "v0.7.2"' in result.stdout