Fix progress status, check for already processed face detection

Author: Mr-Pi

main_ops.py

@@ -156,15 +156,17 @@ def create_by_location(entry, dest_dir, db_meta, db_hash_datename):
     return '{} has location data'.format(basename), None, None
 
 
-def detect_faces(entry, dest_dir, db_hash_datename):
+def detect_faces(entry, dest_dir, db_hash_face, db_hash_datename):
     source = entry[0]
     sha512 = entry[1]
 
     basename = os.path.basename(source)
     hashed_path = os.path.join(dest_dir, 'hashed/raw', sha512)
     date_basename = db_ops.get(db_hash_datename, sha512)
 
-    summary = face_ops.detect_faces(source)
+    summary = db_ops.get(db_hash_face, sha512)
+    if not isinstance(summary, list):
+        summary = face_ops.detect_faces(source)
 
     ret_str = ''
     if len(summary) >= 1:

mt_ops.py

@@ -2,34 +2,38 @@
 from basic_ops import *
 
 
-def handler(handler_function, handler_queue, extra_args, progress_status, db):
+def handler(handler_function, handler_queue, extra_args, progress_max, progress_current, db):
     process_id = multiprocessing.current_process().name
     while True:
         entry = handler_queue.get()
         try:
             log_output, k, v = handler_function(entry, *extra_args)
             if k and db:
-                if v:
+                if v != None:
                     db.set(k, pickle.dumps(v))
                 else:
                     db.append(k, b'')
         except Exception:
             print('Failed to handle {}'.format(entry))
             traceback.print_exc(file=sys.stdout)
             os._exit(10)
-        if progress_status:
-            progress_status['current'] += 1
-            stdout('{:6.2f}% Process {}: {}'.format(progress_status['current']/progress_status['sum'], process_id, log_output))
+        if progress_max:
+            progress_current.value += 1
+            stdout('{:6.2f}% Process {}: {}'.format(progress_current.value/progress_max, process_id, log_output))
         else:
             stdout('Process {}: {}'.format(process_id, log_output))
         handler_queue.task_done()
 
 
-def iter_threaded(iter_funtion, handler_function, db=None, handler_args=(), num_threads=8, size_queue=10, iter_args=(), progress_status=None):
+def iter_threaded(iter_funtion, handler_function, db=None, handler_args=(), num_threads=8, size_queue=10, iter_args=(), progress_max=None):
     processes = []
+    progress_current = None
+    if progress_max:
+        progress_max /= 100
+        progress_current = multiprocessing.Value('i', 0)
     handler_queue = multiprocessing.JoinableQueue(size_queue)
     for i in range(0, num_threads):
-        process = multiprocessing.Process(name = '{}'.format(i), target=handler, args=(handler_function, handler_queue, handler_args, progress_status, db))
+        process = multiprocessing.Process(name = '{}'.format(i), target=handler, args=(handler_function, handler_queue, handler_args, progress_max, progress_current, db))
         process.start()
         processes.append(process)
     for request in iter_funtion(*iter_args):
@@ -39,6 +43,6 @@ def iter_threaded(iter_funtion, handler_function, db=None, handler_args=(), num_
     for process in processes:
         process.terminate()
         process.join()
-    if progress_status:
+    if progress_max:
         sys.stdout.write('\r100.00%')
     print('\n')

pic_sort.py

@@ -66,35 +66,29 @@ def main():
     entries = db.source_hash.dbsize()
 
     print_bold('copy/move all files')
-    progress_status = {'current': 0, 'sum': entries/100}
-    iter_threaded(db_ops.iter_db, copy_move_file, progress_status = progress_status, db=db.hash_meta,
+    iter_threaded(db_ops.iter_db, copy_move_file, progress_max = entries, db=db.hash_meta,
             iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, move_file, ))
 
     print_bold('parse meta data')
-    progress_status = {'current': 0, 'sum': entries/100}
-    iter_threaded(db_ops.iter_db, get_meta_data, progress_status = progress_status, db=db.hash_meta,
+    iter_threaded(db_ops.iter_db, get_meta_data, progress_max = entries, db=db.hash_meta,
             iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, args.max_diff, ))
 
     print_bold('create date links')
-    progress_status = {'current': 0, 'sum': entries/100}
-    iter_threaded(db_ops.iter_db, create_date_link, progress_status = progress_status, db=db.hash_datename,
+    iter_threaded(db_ops.iter_db, create_date_link, progress_max = entries, db=db.hash_datename,
             iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, db.hash_meta, ))
 
     print_bold('create by links')
-    progress_status = {'current': 0, 'sum': entries/100}
-    iter_threaded(db_ops.iter_db, create_by_link, progress_status = progress_status,
+    iter_threaded(db_ops.iter_db, create_by_link, progress_max = entries,
             iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, db.hash_meta, db.hash_datename, ))
 
     print_bold('create location links')
-    progress_status = {'current': 0, 'sum': entries/100}
-    iter_threaded(db_ops.iter_db, create_by_location, progress_status = progress_status,
+    iter_threaded(db_ops.iter_db, create_by_location, progress_max = entries,
             iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, db.hash_meta, db.hash_datename, ))
 
     if not args.skip_faces:
         print_bold('detect faces')
-        progress_status = {'current': 0, 'sum': entries/100}
-        iter_threaded(db_ops.iter_db, detect_faces, progress_status = progress_status, db=db.hash_face,
-                iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, db.hash_datename, ))
+        iter_threaded(db_ops.iter_db, detect_faces, progress_max = entries, db=db.hash_face,
+                iter_args=(db.source_hash,), num_threads = args.threads, size_queue = args.queue_size, handler_args = (dest_dir, db.hash_face, db.hash_datename, ))
 
     print('\n   finished processed {} files\n'.format(entries))