From 0d54d3b805f94535c8423813e7740203e0459194 Mon Sep 17 00:00:00 2001 From: "Brian S. Stephan" Date: Sat, 6 Jan 2024 13:42:10 -0600 Subject: [PATCH] concatenate flags to combine board and user configs Signed-off-by: Brian S. Stephan --- README.md | 9 ++++++- gp2040ce_bintools/builder.py | 52 +++++++++++++++++++++++++----------- tests/test_builder.py | 36 +++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 792ed62..67cc27f 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,10 @@ A quick demonstration of the editor is available [on asciinema.org](https://asci `concatenate` combines a GP2040-CE firmware .bin file (such as from a fresh build) with: -* a GP2040-CE user config, in the form of +* a GP2040-CE board config, in the form of + * a config section .bin (with footer) (optionally padded) (`--binary-board-config-filename`) or + * a JSON file representing the config (`--json-board-config-filename`) +* and/or a GP2040-CE user config, in the form of * a config section .bin (with footer) (optionally padded) (`--binary-user-config-filename`) or * a JSON file representing the config (`--json-user-config-filename`) @@ -64,6 +67,10 @@ flashed with a particular configuration, for instances such as producing a binar configuration (specific customizations, etc.), or keeping documented backups of what you're testing with during development. +The `--...-board-config-filename` flags allow for shipping a default configuration as part of the binary, replacing +the need for generating these board configurations at compile time. This allows for more custom builds and less +dependency on the build jobs, and is a feature in progress in the core firmware. + The produced binary can be written to a file with `--new-binary-filename FILENAME` or straight to a RP2040 in BOOTSEL mode with `--usb`. diff --git a/gp2040ce_bintools/builder.py b/gp2040ce_bintools/builder.py index 9a02815..eecf777 100644 --- a/gp2040ce_bintools/builder.py +++ b/gp2040ce_bintools/builder.py @@ -56,7 +56,10 @@ def combine_firmware_and_config(firmware_binary: bytearray, board_config_binary: return combined -def concatenate_firmware_and_storage_files(firmware_filename: str, binary_user_config_filename: Optional[str] = None, +def concatenate_firmware_and_storage_files(firmware_filename: str, + binary_board_config_filename: Optional[str] = None, + json_board_config_filename: Optional[str] = None, + binary_user_config_filename: Optional[str] = None, json_user_config_filename: Optional[str] = None, combined_filename: str = '', usb: bool = False, replace_extra: bool = False) -> None: @@ -64,25 +67,36 @@ def concatenate_firmware_and_storage_files(firmware_filename: str, binary_user_c Args: firmware_filename: filename of the firmware binary to read + binary_board_config_filename: filename of the board config section to read, in binary format + json_board_config_filename: filename of the board config section to read, in JSON format binary_user_config_filename: filename of the user config section to read, in binary format json_user_config_filename: filename of the user config section to read, in JSON format combined_filename: filename of where to write the combine binary replace_extra: if larger than normal firmware files should have their overage replaced """ new_binary = None - if binary_user_config_filename: - with open(firmware_filename, 'rb') as firmware, open(binary_user_config_filename, 'rb') as storage: - new_binary = combine_firmware_and_config(bytearray(firmware.read()), None, bytearray(storage.read()), - replace_extra=replace_extra) - elif json_user_config_filename: - with open(firmware_filename, 'rb') as firmware, open(json_user_config_filename, 'r') as json_file: - config = get_config_from_json(json_file.read()) - serialized_config = serialize_config_with_footer(config) - new_binary = combine_firmware_and_config(bytearray(firmware.read()), None, serialized_config, - replace_extra=replace_extra) + board_config_binary = None + user_config_binary = None - if not new_binary: - raise ValueError("no means to create a binary was provided") + if binary_board_config_filename: + with open(binary_board_config_filename, 'rb') as storage: + board_config_binary = storage.read() + elif json_board_config_filename: + with open(json_board_config_filename, 'r') as json_file: + config = get_config_from_json(json_file.read()) + board_config_binary = serialize_config_with_footer(config) + + if binary_user_config_filename: + with open(binary_user_config_filename, 'rb') as storage: + user_config_binary = storage.read() + elif json_user_config_filename: + with open(json_user_config_filename, 'r') as json_file: + config = get_config_from_json(json_file.read()) + user_config_binary = serialize_config_with_footer(config) + + with open(firmware_filename, 'rb') as firmware: + new_binary = combine_firmware_and_config(bytearray(firmware.read()), board_config_binary, user_config_binary, + replace_extra=replace_extra) if combined_filename: with open(combined_filename, 'wb') as combined: @@ -234,15 +248,19 @@ def write_new_config_to_usb(config: Message, endpoint_out: object, endpoint_in: def concatenate(): """Combine a built firmware .bin and a storage .bin.""" parser = argparse.ArgumentParser( - description="Combine a compiled GP2040-CE firmware-only .bin and an existing storage area or config .bin " - "into one file suitable for flashing onto a board.", + description="Combine a compiled GP2040-CE firmware-only .bin and existing user and/or board storage area(s) " + "or config .bin(s) into one file suitable for flashing onto a board.", parents=[core_parser], ) parser.add_argument('--replace-extra', action='store_true', help="if the firmware file is larger than the location of storage, perhaps because it's " "actually a full board dump, overwrite its config section with the config binary") parser.add_argument('firmware_filename', help=".bin file of a GP2040-CE firmware, probably from a build") - user_config_group = parser.add_mutually_exclusive_group(required=True) + board_config_group = parser.add_mutually_exclusive_group(required=False) + board_config_group.add_argument('--binary-board-config-filename', + help=".bin file of a GP2040-CE board config w/footer") + board_config_group.add_argument('--json-board-config-filename', help=".json file of a GP2040-CE board config") + user_config_group = parser.add_mutually_exclusive_group(required=False) user_config_group.add_argument('--binary-user-config-filename', help=".bin file of a GP2040-CE user config w/footer") user_config_group.add_argument('--json-user-config-filename', help=".json file of a GP2040-CE user config") @@ -252,6 +270,8 @@ def concatenate(): args, _ = parser.parse_known_args() concatenate_firmware_and_storage_files(args.firmware_filename, + binary_board_config_filename=args.binary_board_config_filename, + json_board_config_filename=args.json_board_config_filename, binary_user_config_filename=args.binary_user_config_filename, json_user_config_filename=args.json_user_config_filename, combined_filename=args.new_binary_filename, usb=args.usb, diff --git a/tests/test_builder.py b/tests/test_builder.py index 97434cd..32d6fc4 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -38,12 +38,43 @@ def test_concatenate_to_file(tmp_path): tmp_file = os.path.join(tmp_path, 'concat.bin') firmware_file = os.path.join(HERE, 'test-files', 'test-firmware.bin') config_file = os.path.join(HERE, 'test-files', 'test-config.bin') - concatenate_firmware_and_storage_files(firmware_file, config_file, combined_filename=tmp_file) + concatenate_firmware_and_storage_files(firmware_file, binary_user_config_filename=config_file, + combined_filename=tmp_file) with open(tmp_file, 'rb') as file: content = file.read() assert len(content) == 2 * 1024 * 1024 +def test_concatenate_board_config_to_file(tmp_path): + """Test that we write a file with firmware + binary board config as expected.""" + tmp_file = os.path.join(tmp_path, 'concat.bin') + firmware_file = os.path.join(HERE, 'test-files', 'test-firmware.bin') + config_file = os.path.join(HERE, 'test-files', 'test-config.bin') + concatenate_firmware_and_storage_files(firmware_file, binary_board_config_filename=config_file, + combined_filename=tmp_file) + with open(tmp_file, 'rb') as file: + content = file.read() + assert len(content) == (2 * 1024 * 1024) - (16 * 1024) + + +def test_concatenate_both_configs_to_file(tmp_path): + """Test that we write a file with firmware + binary board + binary user config as expected.""" + tmp_file = os.path.join(tmp_path, 'concat.bin') + firmware_file = os.path.join(HERE, 'test-files', 'test-firmware.bin') + config_file = os.path.join(HERE, 'test-files', 'test-config.bin') + concatenate_firmware_and_storage_files(firmware_file, binary_board_config_filename=config_file, + binary_user_config_filename=config_file, combined_filename=tmp_file) + with open(tmp_file, 'rb') as file: + content = file.read() + assert len(content) == 2 * 1024 * 1024 + storage = get_board_storage_section(content) + footer_size, _, _ = get_config_footer(storage) + assert footer_size == 3309 + storage = get_user_storage_section(content) + footer_size, _, _ = get_config_footer(storage) + assert footer_size == 3309 + + @with_pb2s def test_concatenate_user_json_to_file(tmp_path): """Test that we write a file with firmware + JSON user config as expected.""" @@ -72,7 +103,8 @@ def test_concatenate_to_usb(tmp_path): end_out, end_in = mock.MagicMock(), mock.MagicMock() with mock.patch('gp2040ce_bintools.builder.get_bootsel_endpoints', return_value=(end_out, end_in)): with mock.patch('gp2040ce_bintools.builder.write') as mock_write: - concatenate_firmware_and_storage_files(firmware_file, config_file, usb=True) + concatenate_firmware_and_storage_files(firmware_file, binary_user_config_filename=config_file, + usb=True) assert mock_write.call_args.args[2] == 0x10000000 assert len(mock_write.call_args.args[3]) == 2 * 1024 * 1024