From 24617bf9208d9334bd873614f56d1a03a134dfc0 Mon Sep 17 00:00:00 2001 From: "Brian S. Stephan" Date: Tue, 20 Jun 2023 12:50:32 -0500 Subject: [PATCH] visualize-storage tool --- read GP2040-CE config this also comes with a lot of project scaffolding for (IMO) a well-organized python project. this should get the ball rolling for other devs --- .gitattributes | 1 + README.md | 37 +++++++++++++++++++++- gp2040ce_bintools/__init__.py | 33 ++++++++++++++++++++ gp2040ce_bintools/storage.py | 51 +++++++++++++++++++++++++++++++ pyproject.toml | 13 ++++++-- requirements/requirements-dev.txt | 47 ++++++++++++++++++++++++---- requirements/requirements.txt | 11 +++++-- 7 files changed, 180 insertions(+), 13 deletions(-) create mode 100644 .gitattributes create mode 100644 gp2040ce_bintools/__init__.py create mode 100644 gp2040ce_bintools/storage.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..176a458 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/README.md b/README.md index 387e754..4783f79 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,38 @@ -# OpenStickCommunity Binary Tools +# GP2040-CE Binary Tools Tools for working with GP2040-CE binary dumps. + +## Installation + +``` +% git clone [URL to this repository] +% cd gp2040ce-binary-tools +% pip install -e . +``` + +At some point we may publish packages to e.g. pypi. + +### Development Installation + +As above, plus also `pip install -Ur requirements/requirements-dev.txt` to get linters and whatnot. + +## Tools + +### visualize-storage + +**visualize-storage** reads a dump of a GP2040-CE board's flash storage section, where the configuration lives, +and prints it out for visual inspection or diffing with other tools. Usage is simple; just pass the tool your +binary file to analyze along with the path to the Protobuf files. + +Because Protobuf relies on .proto files to convey the serialized structure, you must supply them +from the main GP2040-CE project, e.g. pointing this tool at your clone of the core project. Something like +this would suffice for a working invocation (note: you do not need to compile the files yourself): + +``` +% visualize-storage --proto-files-path=~/proj/GP2040-CE/proto \ +--proto-files-path=~/proj/GP2040-CE/lib/nanopb/generator/proto \ +memory.bin +``` + +In the future we will look into publishing complete packages that include the compiled `_pb2.py` files, so that you +don't need to provide them yourself. diff --git a/gp2040ce_bintools/__init__.py b/gp2040ce_bintools/__init__.py new file mode 100644 index 0000000..78204ba --- /dev/null +++ b/gp2040ce_bintools/__init__.py @@ -0,0 +1,33 @@ +"""Initialize the package and get dependencies.""" +import argparse +import importlib +import logging +import os +import pathlib +import sys + +import grpc + +logger = logging.getLogger(__name__) + +core_parser = argparse.ArgumentParser(add_help=False) +core_parser.add_argument('--proto-files-path', type=pathlib.Path, default=list(), action='append', + help="path to .proto files to read, including dependencies; you will likely need " + "to supply this twice, once for GP2040-CE's .proto files and once for nanopb's") +args, _ = core_parser.parse_known_args() +for path in args.proto_files_path: + sys.path.append(os.path.abspath(os.path.expanduser(path))) + + +def get_config_pb2(): + """Retrieve prebuilt _pb2 file or attempt to compile it live.""" + try: + return importlib.import_module('config_pb2') + except ModuleNotFoundError: + if args.proto_files_path: + # compile the proto files in realtime, leave them in this package + package_path = pathlib.Path(__file__).parent + logger.info("Generating Protobuf Python files into %s...", package_path) + return grpc.protos('config.proto') + + raise diff --git a/gp2040ce_bintools/storage.py b/gp2040ce_bintools/storage.py new file mode 100644 index 0000000..f2ba9bd --- /dev/null +++ b/gp2040ce_bintools/storage.py @@ -0,0 +1,51 @@ +"""Interact with the protobuf config from a picotool flash dump of a GP2040-CE board. + +This is more manual than I'd like at the moment, but this demonstrates/documents the means +to display the config on a board. It requires a checkout that matches the version of the +firmware on the board (so that the protobuf messages match the flash, though protobuf +tolerates differences well, so it'll probably work, just be misnamed or incomplete). + +Generating the Python proto code: +[env setup] +% protoc ../../proto/* -I../../proto/ -I../../lib/nanopb/generator/proto --python_out=build + +Usage: +[env setup] +% picotool save -r 101FE000 101FFFF4 build/memory.bin # 101FE000 = storage start, 101FFFF4 storage end - footer +% export PYTHONPATH=../../lib/nanopb/generator/proto:build +% python visualize.py +""" +import argparse +import pprint + +from gp2040ce_bintools import core_parser, get_config_pb2 + + +def get_config(filename): + """Load the protobuf section of an flash and display the contents.""" + with open(filename, 'rb') as dump: + # read off the unused space + while True: + byte = dump.read(1) + if byte != b'\x00': + break + content = byte + dump.read() + + config_pb2 = get_config_pb2() + config = config_pb2.Config() + config.ParseFromString(content) + return config + + +def visualize(): + """Pretty print the contents of GP2040-CE's storage.""" + parser = argparse.ArgumentParser( + prog="visualize-storage", + description="Read a the configuration storage section from a GP2040-CE board dump and print out its contents.", + parents=[core_parser], + ) + parser.add_argument('filename', help=".bin file of a GP2040-CE board's storage section, bytes 101FE000-101FFFF4") + args, _ = parser.parse_known_args() + + config = get_config(args.filename) + pprint.pprint(config) diff --git a/pyproject.toml b/pyproject.toml index 42d2ac4..66761d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "openstickcommunity-binary-tools" +name = "gp2040ce-binary-tools" version = "0" description = "Tools for working with GP2040-CE binary dumps." readme = "README.md" @@ -8,7 +8,14 @@ authors = [ {name = "Brian S. Stephan", email = "bss@incorporeal.org"}, ] requires-python = ">=3.8" -dependencies = ["argparse", "protobuf"] +dependencies = ["grpcio-tools"] [project.optional-dependencies] -dev = ["flake8", "pip-tools"] +dev = ["flake8", "flake8-blind-except", "flake8-builtins", "flake8-docstrings", "flake8-executable", "flake8-fixme", + "flake8-isort", "flake8-logging-format", "flake8-mutable", "flake8-pyproject", "pip-tools"] + +[project.scripts] +visualize-storage = "gp2040ce_bintools.storage:visualize" + +[tool.flake8] +max-line-length = 120 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 43f5971..e578681 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -4,30 +4,65 @@ # # pip-compile --extra=dev --output-file=requirements/requirements-dev.txt pyproject.toml # -argparse==1.4.0 - # via openstickcommunity-binary-tools (pyproject.toml) build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools flake8==6.0.0 - # via openstickcommunity-binary-tools (pyproject.toml) + # via + # flake8-builtins + # flake8-docstrings + # flake8-executable + # flake8-isort + # flake8-mutable + # flake8-pyproject + # gp2040ce-binary-tools (pyproject.toml) +flake8-blind-except==0.2.1 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-builtins==2.1.0 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-docstrings==1.7.0 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-executable==2.1.3 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-fixme==1.1.1 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-isort==6.0.0 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-logging-format==0.9.0 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-mutable==1.2.0 + # via gp2040ce-binary-tools (pyproject.toml) +flake8-pyproject==1.2.3 + # via gp2040ce-binary-tools (pyproject.toml) +grpcio==1.54.2 + # via grpcio-tools +grpcio-tools==1.54.2 + # via gp2040ce-binary-tools (pyproject.toml) +isort==5.12.0 + # via flake8-isort mccabe==0.7.0 # via flake8 packaging==23.1 # via build pip-tools==6.13.0 - # via openstickcommunity-binary-tools (pyproject.toml) + # via gp2040ce-binary-tools (pyproject.toml) protobuf==4.23.3 - # via openstickcommunity-binary-tools (pyproject.toml) + # via grpcio-tools pycodestyle==2.10.0 # via flake8 +pydocstyle==6.3.0 + # via flake8-docstrings pyflakes==3.0.1 # via flake8 pyproject-hooks==1.0.0 # via build +snowballstemmer==2.2.0 + # via pydocstyle tomli==2.0.1 - # via build + # via + # build + # flake8-pyproject wheel==0.40.0 # via pip-tools diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 924da88..4fd21e2 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,7 +4,12 @@ # # pip-compile --output-file=requirements/requirements.txt pyproject.toml # -argparse==1.4.0 - # via openstickcommunity-binary-tools (pyproject.toml) +grpcio==1.54.2 + # via grpcio-tools +grpcio-tools==1.54.2 + # via gp2040ce-binary-tools (pyproject.toml) protobuf==4.23.3 - # via openstickcommunity-binary-tools (pyproject.toml) + # via grpcio-tools + +# The following packages are considered to be unsafe in a requirements file: +# setuptools