From d5481142933f7b331460d8df7c03a50cbc9286af Mon Sep 17 00:00:00 2001 From: jdjjm <66574578+jdjjm@users.noreply.github.com> Date: Thu, 18 Jun 2020 08:20:55 +1000 Subject: [PATCH 1/5] Only upload file to device if remote is different Gets a hash of the remote file and compares it to a hash of the local file. If they are different it uploads. If not it skips over. --- scripts/microupload.py | 54 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/scripts/microupload.py b/scripts/microupload.py index 0ee90bd8..9d6abc8b 100644 --- a/scripts/microupload.py +++ b/scripts/microupload.py @@ -22,6 +22,7 @@ -X --exclude=PATH Path to exclude, may be repeated. -C --chdir=PATH Change current directory to path. -v --verbose Verbose output. + -d --different Only Upload if remote file is different to local file. """ import time @@ -31,21 +32,55 @@ from typing import List, Iterable, TypeVar, Sequence, Set from docopt import docopt -from ampy.pyboard import Pyboard +from ampy.pyboard import Pyboard, PyboardError from ampy.files import Files, DirectoryExistsError +from hashlib import sha1 +import textwrap __all__ = [] -verbose = False +verbose = True T = TypeVar('T') +def get_hash(self, filename): + command = """ + import sys + from uhashlib import sha1 + hash = sha1() + with open('{0}', 'rb') as infile: + while True: + result = infile.read({1}) + hash.update(result) + if result == b'': + break + len = sys.stdout.write(hash.digest()) + """.format( + filename, 32 + ) + self._pyboard.enter_raw_repl() + try: + out = self._pyboard.exec_(textwrap.dedent(command)) + except PyboardError as ex: + # Check if this is an OSError #2, i.e. file doesn't exist and + # rethrow it as something more descriptive. + if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1: + raise RuntimeError("No such file: {0}".format(filename)) + else: + raise ex + self._pyboard.exit_raw_repl() + return out + +Files.get_hash = get_hash def main(args: List[str]) -> None: global verbose + start_time = time.time() + opts = docopt(__doc__, argv=args) verbose = opts['--verbose'] root = opts['PATH'] + different = opts['--different'] chdir = opts['--chdir'] if chdir: @@ -64,7 +99,6 @@ def main(args: List[str]) -> None: for x in list_files(root, opts['--exclude'])] else: to_upload = [rel_root] - created_cache = set() for path in progress('Uploading files', to_upload): local_path = os.path.abspath(path) @@ -76,10 +110,22 @@ def main(args: List[str]) -> None: if remote_dir: make_dirs(files, remote_dir, created_cache) with open(local_path, 'rb') as fd: - files.put(remote_path, fd.read()) + if different: + raw = fd.read() + local_file_hash = sha1(raw).digest() + + remote_file_hash = files.get_hash(remote_path) + if remote_file_hash == local_file_hash: + print("File Identical", file=sys.stderr, flush=True) + else: + print("Files different...", file=sys.stderr, flush=True) + files.put(remote_path, raw) + else: + files.put(remote_path, fd.read()) print('Soft reboot', file=sys.stderr, flush=True) soft_reset(board) + print("--- %s seconds ---" % (time.time() - start_time), file=sys.stderr, flush=True) def make_dirs(files: Files, path: str, From 157275dc61124f5a1c8bf16d959444992ebde617 Mon Sep 17 00:00:00 2001 From: jdjjm <66574578+jdjjm@users.noreply.github.com> Date: Mon, 22 Jun 2020 12:58:56 +1000 Subject: [PATCH 2/5] Fixed Windows path issue and unicode issue Fixed windows path issue when finding the remote path. Used ubinascii to get hexdigest as string when finding hash to fix bug when hashing certain hex values. --- scripts/microupload.py | 72 +++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/scripts/microupload.py b/scripts/microupload.py index 9d6abc8b..b943de8e 100644 --- a/scripts/microupload.py +++ b/scripts/microupload.py @@ -47,14 +47,16 @@ def get_hash(self, filename): command = """ import sys from uhashlib import sha1 + import ubinascii hash = sha1() with open('{0}', 'rb') as infile: - while True: - result = infile.read({1}) - hash.update(result) - if result == b'': - break - len = sys.stdout.write(hash.digest()) + while True: + result = infile.read({1}) + hash.update(result) + if result == b'': + break + len = sys.stdout.write(ubinascii.hexlify(hash.digest())) + infile.close() """.format( filename, 32 ) @@ -65,13 +67,42 @@ def get_hash(self, filename): # Check if this is an OSError #2, i.e. file doesn't exist and # rethrow it as something more descriptive. if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1: - raise RuntimeError("No such file: {0}".format(filename)) + self._pyboard.exit_raw_repl() + return "" else: raise ex self._pyboard.exit_raw_repl() - return out + return out.decode('utf-8') + +def get_size(self, filename): + print(filename) + command = """ + import sys + import os + + len = sys.stdout.write(str(os.stat('{0}')[6]).encode('utf-8')) + """.format( + filename + ) + self._pyboard.enter_raw_repl() + try: + out = self._pyboard.exec_(textwrap.dedent(command)) + print("out", out, file=sys.stderr, flush=True) + except PyboardError as ex: + print("except", file=sys.stderr, flush=True) + # Check if this is an OSError #2, i.e. file doesn't exist and + # rethrow it as something more descriptive. + if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1: + print("fnf", file=sys.stderr, flush=True) + self._pyboard.exit_raw_repl() + return "-1" + else: + raise ex + self._pyboard.exit_raw_repl() + return out.decode('utf-8') Files.get_hash = get_hash +Files.get_size = get_size def main(args: List[str]) -> None: global verbose @@ -102,7 +133,7 @@ def main(args: List[str]) -> None: created_cache = set() for path in progress('Uploading files', to_upload): local_path = os.path.abspath(path) - remote_path = os.path.normpath(path) + remote_path = os.path.normpath(path).replace(os.path.sep, '/') if verbose: print('\n{} -> {}'.format(local_path, remote_path), file=sys.stderr, flush=True) @@ -112,13 +143,24 @@ def main(args: List[str]) -> None: with open(local_path, 'rb') as fd: if different: raw = fd.read() - local_file_hash = sha1(raw).digest() - - remote_file_hash = files.get_hash(remote_path) - if remote_file_hash == local_file_hash: - print("File Identical", file=sys.stderr, flush=True) + local_file_size = str(os.stat(local_path)[6]) + local_file_hash = sha1(raw).hexdigest() + + remote_file_size = files.get_size(remote_path) + print("Size", remote_file_size, local_file_size, file=sys.stderr, flush=True) + if local_file_size == remote_file_size: + print("Size same", file=sys.stderr, flush=True) + remote_file_hash = files.get_hash(remote_path) + if remote_file_hash == local_file_hash: + print("File Identical", file=sys.stderr, flush=True) + else: + print("HASH", local_file_hash, remote_file_hash, file=sys.stderr, flush=True) + print("Files different...", file=sys.stderr, flush=True) + wait_for_board() + files.put(remote_path, raw) else: - print("Files different...", file=sys.stderr, flush=True) + print("Sizes different... Uploading", file=sys.stderr, flush=True) + wait_for_board() files.put(remote_path, raw) else: files.put(remote_path, fd.read()) From 1aeeb5a08de97831808cf033285c8ae57a98b91f Mon Sep 17 00:00:00 2001 From: jdjjm <66574578+jdjjm@users.noreply.github.com> Date: Mon, 22 Jun 2020 13:03:21 +1000 Subject: [PATCH 3/5] Removed print commands Removed print commands used for development --- scripts/microupload.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/microupload.py b/scripts/microupload.py index b943de8e..488517a7 100644 --- a/scripts/microupload.py +++ b/scripts/microupload.py @@ -87,7 +87,6 @@ def get_size(self, filename): self._pyboard.enter_raw_repl() try: out = self._pyboard.exec_(textwrap.dedent(command)) - print("out", out, file=sys.stderr, flush=True) except PyboardError as ex: print("except", file=sys.stderr, flush=True) # Check if this is an OSError #2, i.e. file doesn't exist and @@ -147,19 +146,14 @@ def main(args: List[str]) -> None: local_file_hash = sha1(raw).hexdigest() remote_file_size = files.get_size(remote_path) - print("Size", remote_file_size, local_file_size, file=sys.stderr, flush=True) if local_file_size == remote_file_size: - print("Size same", file=sys.stderr, flush=True) remote_file_hash = files.get_hash(remote_path) if remote_file_hash == local_file_hash: - print("File Identical", file=sys.stderr, flush=True) + print("File Identical... Skipping upload", file=sys.stderr, flush=True) else: - print("HASH", local_file_hash, remote_file_hash, file=sys.stderr, flush=True) - print("Files different...", file=sys.stderr, flush=True) wait_for_board() files.put(remote_path, raw) else: - print("Sizes different... Uploading", file=sys.stderr, flush=True) wait_for_board() files.put(remote_path, raw) else: From 8f6fc8713b8bda41c8438e7bd2d5b219c29aa984 Mon Sep 17 00:00:00 2001 From: jdjjm <66574578+jdjjm@users.noreply.github.com> Date: Mon, 22 Jun 2020 13:07:28 +1000 Subject: [PATCH 4/5] More Formatting --- scripts/microupload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/microupload.py b/scripts/microupload.py index 488517a7..db196dd4 100644 --- a/scripts/microupload.py +++ b/scripts/microupload.py @@ -75,7 +75,6 @@ def get_hash(self, filename): return out.decode('utf-8') def get_size(self, filename): - print(filename) command = """ import sys import os @@ -105,6 +104,7 @@ def get_size(self, filename): def main(args: List[str]) -> None: global verbose + start_time = time.time() opts = docopt(__doc__, argv=args) @@ -139,6 +139,7 @@ def main(args: List[str]) -> None: remote_dir = os.path.dirname(path) if remote_dir: make_dirs(files, remote_dir, created_cache) + with open(local_path, 'rb') as fd: if different: raw = fd.read() From ce1dd4787451b11a39c754bf1af39801ff5c4d9b Mon Sep 17 00:00:00 2001 From: jdjjm <66574578+jdjjm@users.noreply.github.com> Date: Tue, 23 Jun 2020 14:52:24 +1000 Subject: [PATCH 5/5] Removed Debug Print statments Removed some debug print statements I forgot to remove. --- scripts/microupload.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/microupload.py b/scripts/microupload.py index db196dd4..1cbf87bc 100644 --- a/scripts/microupload.py +++ b/scripts/microupload.py @@ -87,11 +87,9 @@ def get_size(self, filename): try: out = self._pyboard.exec_(textwrap.dedent(command)) except PyboardError as ex: - print("except", file=sys.stderr, flush=True) # Check if this is an OSError #2, i.e. file doesn't exist and # rethrow it as something more descriptive. if ex.args[2].decode("utf-8").find("OSError: [Errno 2] ENOENT") != -1: - print("fnf", file=sys.stderr, flush=True) self._pyboard.exit_raw_repl() return "-1" else: