CHANGELOG updates for v0.8.3

Signed-off-by: Brian S. Stephan <bss@incorporeal.org>

CHANGELOG.md

@@ -3,6 +3,14 @@
 Included is a summary of changes to the project. For full details, especially on behind-the-scenes code changes and
 development tools, see the commit history.
 
+## v0.8.3
+
+### Improvements
+
+* `summarize-gp2040ce` can now understand the segmented UF2 files written in v0.8.1.
+* `concatenate` has an added `--backup` flag, which will move an existing output file aside before writing the new
+  output.
+
 ## v0.8.2
 
 ### Bugfixes

gp2040ce_bintools/builder.py

@@ -6,6 +6,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 import argparse
 import copy
 import logging
+import os
 import re
 from typing import Optional
 
@@ -55,13 +56,14 @@ def combine_firmware_and_config(firmware_binary: bytearray, board_config_binary:
     return combined
 
 
-def concatenate_firmware_and_storage_files(firmware_filename: str,
+def concatenate_firmware_and_storage_files(firmware_filename: str,      # noqa: C901
                                            binary_board_config_filename: Optional[str] = None,
                                            json_board_config_filename: Optional[str] = None,
                                            binary_user_config_filename: Optional[str] = None,
                                            json_user_config_filename: Optional[str] = None,
                                            combined_filename: str = '', usb: bool = False,
-                                           replace_extra: bool = False) -> None:
+                                           replace_extra: bool = False,
+                                           backup: bool = False) -> None:
     """Open the provided binary files and combine them into one combined GP2040-CE with config file.
 
     Args:
@@ -72,6 +74,7 @@ def concatenate_firmware_and_storage_files(firmware_filename: str,
         json_user_config_filename: filename of the user config section to read, in JSON format
         combined_filename: filename of where to write the combine binary
         replace_extra: if larger than normal firmware files should have their overage replaced
+        backup: if the output filename exists, move it to foo.ext.old before writing foo.ext
     """
     new_binary = bytearray([])
     board_config_binary = bytearray([])
@@ -112,6 +115,8 @@ def concatenate_firmware_and_storage_files(firmware_filename: str,
         new_binary = storage.convert_binary_to_uf2(binary_list)
 
     if combined_filename:
+        if backup and os.path.exists(combined_filename):
+            os.rename(combined_filename, f'{combined_filename}.old')
         with open(combined_filename, 'wb') as combined:
             combined.write(new_binary)
 
@@ -304,6 +309,8 @@ def concatenate():
     output_group = parser.add_mutually_exclusive_group(required=True)
     output_group.add_argument('--usb', action='store_true', help="write the resulting firmware + storage to USB")
     output_group.add_argument('--new-filename', help="output .bin or .uf2 file of the resulting firmware + storage")
+    parser.add_argument('--backup', action='store_true', default=False,
+                        help="if the output file exists, move it to .old before writing")
 
     args, _ = parser.parse_known_args()
     concatenate_firmware_and_storage_files(args.firmware_filename,
@@ -312,7 +319,7 @@ def concatenate():
                                            binary_user_config_filename=args.binary_user_config_filename,
                                            json_user_config_filename=args.json_user_config_filename,
                                            combined_filename=args.new_filename, usb=args.usb,
-                                           replace_extra=args.replace_extra)
+                                           replace_extra=args.replace_extra, backup=args.backup)
 
 
 def dump_gp2040ce():

gp2040ce_bintools/storage.py

@@ -114,9 +114,13 @@ def convert_uf2_to_binary(uf2: bytearray) -> bytearray:
         if block_count != len(uf2) // 512:
             raise ValueError(f"inconsistent block count in reading UF2, got {block_count}, expected {len(uf2) // 512}!")
 
-        # the UF2 is indexed, which we could convert to binary with padding 0s, but we don't yet
-        if old_uf2_addr and (uf2_addr != old_uf2_addr + 256):
-            raise ValueError("segmented UF2 files are not yet supported!")
+        if old_uf2_addr and (uf2_addr >= old_uf2_addr + bytes_):
+            # the new binary content is not immediately after what we wrote, it's further ahead, so pad
+            # the difference
+            binary += bytearray(b'\x00' * (uf2_addr - (old_uf2_addr + bytes_)))
+        elif old_uf2_addr and (uf2_addr < old_uf2_addr + bytes_):
+            # this is seeking backwards which we don't see yet
+            raise NotImplementedError("going backwards in binary files is not yet supported")
 
         binary += content[0:bytes_]
         old_uf2_addr = uf2_addr

tests/test_builder.py

@@ -137,6 +137,31 @@ def test_concatenate_to_uf2_board_only(tmp_path, firmware_binary, config_binary)
                             math.ceil(STORAGE_SIZE/256) * 512)
 
 
+def test_concatenate_with_backup(tmp_path, firmware_binary, config_binary):
+    """Test that we write a UF2 file as expected."""
+    tmp_file = os.path.join(tmp_path, 'concat.uf2')
+    firmware_file = os.path.join(HERE, 'test-files', 'test-firmware.bin')
+    config_file = os.path.join(HERE, 'test-files', 'test-config.bin')
+    # create the file we are going to try to overwrite and want backed up
+    builder.concatenate_firmware_and_storage_files(firmware_file, binary_board_config_filename=config_file,
+                                                   combined_filename=tmp_file)
+    # second file, expecting an overwrite of the target with a backup made
+    builder.concatenate_firmware_and_storage_files(firmware_file, binary_board_config_filename=config_file,
+                                                   binary_user_config_filename=config_file,
+                                                   combined_filename=tmp_file,
+                                                   backup=True)
+    # size of the file should be 2x the padded firmware + 2x the board config space + 2x the user config space
+    with open(tmp_file, 'rb') as file:
+        content = file.read()
+    assert len(content) == (math.ceil(len(firmware_binary)/256) * 512 +
+                            math.ceil(STORAGE_SIZE/256) * 512 * 2)
+    # size of the backup file should be 2x the padded firmware + 2x the board config space
+    with open(f'{tmp_file}.old', 'rb') as file:
+        content = file.read()
+    assert len(content) == (math.ceil(len(firmware_binary)/256) * 512 +
+                            math.ceil(STORAGE_SIZE/256) * 512)
+
+
 def test_find_version_string(firmware_binary):
     """Test that we can find a version string in a binary."""
     assert builder.find_version_string_in_binary(firmware_binary) == 'v0.7.5'

tests/test_storage.py

@@ -12,6 +12,7 @@ import pytest
 from decorator import decorator
 
 import gp2040ce_bintools.storage as storage
+from gp2040ce_bintools.builder import concatenate_firmware_and_storage_files
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 
@@ -187,6 +188,36 @@ def test_malformed_uf2(whole_board_with_board_config_dump):
         storage.convert_uf2_to_binary(uf2 + uf2)
 
 
+def test_read_created_uf2(tmp_path, firmware_binary, config_binary):
+    """Test that we read a UF2 with disjoint segments."""
+    tmp_file = os.path.join(tmp_path, 'concat.uf2')
+    firmware_file = os.path.join(HERE, 'test-files', 'test-firmware.bin')
+    config_file = os.path.join(HERE, 'test-files', 'test-config.bin')
+    concatenate_firmware_and_storage_files(firmware_file, binary_board_config_filename=config_file,
+                                           binary_user_config_filename=config_file,
+                                           combined_filename=tmp_file)
+    with open(tmp_file, 'rb') as file:
+        content = file.read()
+    assert len(content) == (math.ceil(len(firmware_binary)/256) * 512 +
+                            math.ceil(storage.STORAGE_SIZE/256) * 512 * 2)
+
+    binary = storage.convert_uf2_to_binary(content)
+    # the converted binary should be aligned properly and of the right size
+    assert len(binary) == 2 * 1024 * 1024
+    assert binary[-16384-4:-16384] == storage.FOOTER_MAGIC
+    assert binary[-4:] == storage.FOOTER_MAGIC
+    user_storage = storage.get_user_storage_section(binary)
+    footer_size, _, _ = storage.get_config_footer(user_storage)
+    assert footer_size == 3309
+
+
+def test_cant_read_out_of_order_uf2():
+    """Test that we currently raise an exception at out of order UF2s until we fix it."""
+    uf2 = storage.convert_binary_to_uf2([(0x1000, b'\x11'), (0, b'\x11')])
+    with pytest.raises(NotImplementedError):
+        storage.convert_uf2_to_binary(uf2)
+
+
 @with_pb2s
 def test_serialize_config_with_footer(storage_dump, config_binary):
     """Test that reserializing a read in config matches the original.