fengqi/lv_micropython
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tools/mpremote: Add hashing ability and use for recursive copy.

Browse Source 
Changes in this commit:
- Adds transport API `fs_hashfile` to compute the hash of a file with given
  algorithm.
- Adds commands `mpremote <...>sum file` to compute and print hashes of
  various algorithms.
- Adds shortcut `mpremote sha256sum file`.
- Uses the hash computation to improve speed of recursive file copy to
  avoid copying a file where the target is identical.

For recursive copy, if possible it will use the board's support (e.g.
built-in hashlib or hashlib from micropython-lib), but will fall back to
downloading the file and using the local implementation.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Jim Mussared 2023-06-10 00:46:08 +10:00 committed by Damien George
parent db59e55fe7
commit 6f8157d880
4 changed files with 46 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tools/mpremote/README.md
View File

@ -20,7 +20,7 @@ The full list of supported commands are:
mpremote exec <string> -- execute the string
mpremote run <file> -- run the given local script
mpremote fs <command> <args...> -- execute filesystem commands on the device
command may be: cat, ls, cp, rm, mkdir, rmdir
command may be: cat, ls, cp, rm, mkdir, rmdir, sha256sum
use ":" as a prefix to specify a file on the device
mpremote repl -- enter REPL
options:
@ -78,6 +78,7 @@ Examples:
mpremote cp :main.py .
mpremote cp main.py :
mpremote cp -r dir/ :
mpremote sha256sum :main.py
mpremote mip install aioble
mpremote mip install github:org/repo@branch
mpremote mip install gitlab:org/repo@branch

21
tools/mpremote/mpremote/commands.py
View File

@ -1,3 +1,4 @@
import hashlib
import os
import sys
import tempfile
@ -127,7 +128,7 @@ def _remote_path_basename(a):
return a.rsplit("/", 1)[-1]
def do_filesystem_cp(state, src, dest, multiple):
def do_filesystem_cp(state, src, dest, multiple, check_hash=False):
if dest.startswith(":"):
dest_exists = state.transport.fs_exists(dest[1:])
dest_isdir = dest_exists and state.transport.fs_isdir(dest[1:])
@ -159,6 +160,19 @@ def do_filesystem_cp(state, src, dest, multiple):
if dest_isdir:
dest = ":" + _remote_path_join(dest[1:], filename)
# Skip copy if the destination file is identical.
if check_hash:
try:
remote_hash = state.transport.fs_hashfile(dest[1:], "sha256")
source_hash = hashlib.sha256(data).digest()
# remote_hash will be None if the device doesn't support
# hashlib.sha256 (and therefore won't match).
if remote_hash == source_hash:
print("Up to date:", dest[1:])
return
except OSError:
pass
# Write to remote.
state.transport.fs_writefile(dest[1:], data, progress_callback=show_progress_bar)
else:
@ -274,7 +288,7 @@ def do_filesystem_recursive_cp(state, src, dest, multiple):
else:
dest_path_joined = os.path.join(dest, *dest_path_split)
do_filesystem_cp(state, src_path_joined, dest_path_joined, multiple=False)
do_filesystem_cp(state, src_path_joined, dest_path_joined, multiple=False, check_hash=True)
def do_filesystem(state, args):
@ -333,6 +347,9 @@ def do_filesystem(state, args):
state.transport.fs_rmdir(path)
elif command == "touch":
state.transport.fs_touchfile(path)
elif command.endswith("sum") and command[-4].isdigit():
digest = state.transport.fs_hashfile(path, command[:-3])
print(digest.hex())
elif command == "cp":
if args.recursive:
do_filesystem_recursive_cp(state, path, cp_dest, len(paths) > 1)

11
tools/mpremote/mpremote/main.py
View File

@ -190,7 +190,9 @@ def argparse_filesystem():
"enable verbose output (defaults to True for all commands except cat)",
)
cmd_parser.add_argument(
"command", nargs=1, help="filesystem command (e.g. cat, cp, ls, rm, rmdir, touch)"
"command",
nargs=1,
help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch)",
)
cmd_parser.add_argument("path", nargs="+", help="local and remote paths")
return cmd_parser
@ -308,12 +310,13 @@ _BUILTIN_COMMAND_EXPANSIONS = {
},
# Filesystem shortcuts (use `cp` instead of `fs cp`).
"cat": "fs cat",
"ls": "fs ls",
"cp": "fs cp",
"rm": "fs rm",
"touch": "fs touch",
"ls": "fs ls",
"mkdir": "fs mkdir",
"rm": "fs rm",
"rmdir": "fs rmdir",
"sha256sum": "fs sha256sum",
"touch": "fs touch",
# Disk used/free.
"df": [
"exec",

19
tools/mpremote/mpremote/transport.py
View File

@ -24,7 +24,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import ast, os, sys
import ast, hashlib, os, sys
from collections import namedtuple
@ -174,3 +174,20 @@ class Transport:
self.exec("f=open('%s','a')\nf.close()" % path)
except TransportError as e:
raise _convert_filesystem_error(e, path) from None
def fs_hashfile(self, path, algo, chunk_size=256):
try:
self.exec("import hashlib\nh = hashlib.{algo}()".format(algo=algo))
except TransportError:
# hashlib (or hashlib.{algo}) not available on device. Do the hash locally.
data = self.fs_readfile(path, chunk_size=chunk_size)
return getattr(hashlib, algo)(data).digest()
try:
self.exec(
"buf = memoryview(bytearray({chunk_size}))\nwith open('{path}', 'rb') as f:\n while True:\n n = f.readinto(buf)\n if n == 0:\n break\n h.update(buf if n == {chunk_size} else buf[:n])\n".format(
chunk_size=chunk_size, path=path
)
)
return self.eval("h.digest()")
except TransportExecError as e:
raise _convert_filesystem_error(e, path) from None