Merge remote-tracking branch 'origin/dev' into main

picframe/controller.py

@@ -94,6 +94,9 @@ def refresh_show_text(self):
             if pic is not None:
                 self.__viewer.reset_name_tm(pic, self.paused, side, self.__model.get_current_pics()[1] is not None)
 
+    def purge_files(self):
+        self.__model.purge_files()
+
     @property
     def subdirectory(self):
         return self.__model.subdirectory

picframe/html/index.html

@@ -38,6 +38,7 @@
                 "location_filter": {type:"text", fn:"setter", val:""},
                 "tags_filter": {type:"text", fn:"setter", val:""},
                 "delete": {type:"action", fn:"delete={}", val:false},
+                "purge_files": {type:"action", fn:"purge_files={}", val:false},
                 "stop": {type:"action", fn:"stop={}", val:false},
                 };
         </script>

picframe/image_cache.py

@@ -22,7 +22,9 @@ class ImageCache:
                      'IPTC Object Name': 'title'}
 
 
-    def __init__(self, picture_dir, db_file, geo_reverse, required_db_schema_version=1, portrait_pairs=False):
+    def __init__(self, picture_dir, db_file, geo_reverse, portrait_pairs=False):
+        # TODO these class methods will crash if Model attempts to instantiate this using a
+        # different version from the latest one - should this argument be taken out?
         self.__modified_folders = []
         self.__modified_files = []
         self.__logger = logging.getLogger("image_cache.ImageCache")
@@ -32,11 +34,13 @@ def __init__(self, picture_dir, db_file, geo_reverse, required_db_schema_version
         self.__geo_reverse = geo_reverse
         self.__portrait_pairs = portrait_pairs #TODO have a function to turn this on and off?
         self.__db = self.__create_open_db(self.__db_file)
-        self.__update_schema(required_db_schema_version)
+        # NB this is where the required schema is set
+        self.__update_schema(2)
 
         self.__keep_looping = True
         self.__pause_looping = False
         self.__shutdown_completed = False
+        self.__purge_files = False
 
         t = threading.Thread(target=self.__loop)
         t.start()
@@ -62,6 +66,8 @@ def stop(self):
         while not self.__shutdown_completed:
             time.sleep(0.05) # make function blocking to ensure staged shutdown
 
+    def purge_files(self):
+        self.__purge_files = True
 
     def update_cache(self):
         """Update the cache database with new and/or modified files
@@ -120,7 +126,7 @@ def query_cache(self, where_clause, sort_clause = 'fname ASC'):
                 for i in range(len(full_list)):
                     if full_list[i][0] != -1:
                         newlist.append(full_list[i])
-                    elif pair_list: #OK @rec - this is tidier and qicker!
+                    elif pair_list:
                         elem = pair_list.pop(0)
                         if pair_list:
                             elem += pair_list.pop(0)
@@ -270,11 +276,30 @@ def __update_schema(self, required_db_schema_version):
 
         # Here, we need to update the db schema as necessary
         if schema_version < required_db_schema_version:
-            pass
-
-        # Finally, update the schema version stamp
-        self.__db.execute('DELETE FROM db_info')
-        self.__db.execute('INSERT INTO db_info VALUES(?)', (required_db_schema_version,))
+            if schema_version <= 1:
+                self.__db.execute("DROP VIEW all_data") # remake all_data for all updates
+                self.__db.execute("ALTER TABLE folder ADD COLUMN missing INTEGER DEFAULT 0 NOT NULL")
+                self.__db.execute("""
+                    CREATE VIEW IF NOT EXISTS all_data
+                    AS
+                    SELECT
+                        folder.name || "/" || file.basename || "." || file.extension AS fname,
+                        file.last_modified,
+                        meta.*,
+                        meta.height > meta.width as is_portrait,
+                        location.description as location
+                    FROM file
+                        INNER JOIN folder
+                            ON folder.folder_id = file.folder_id
+                        LEFT JOIN meta
+                            ON file.file_id = meta.file_id
+                        LEFT JOIN location
+                            ON location.latitude = meta.latitude AND location.longitude = meta.longitude
+                    WHERE folder.missing = 0
+                    """)
+            # Finally, update the schema version stamp
+            self.__db.execute('DELETE FROM db_info')
+            self.__db.execute('INSERT INTO db_info VALUES(?)', (required_db_schema_version,))
 
 
     def __get_modified_folders(self):
@@ -283,7 +308,7 @@ def __get_modified_folders(self):
         for dir in [d[0] for d in os.walk(self.__picture_dir)]:
             mod_tm = int(os.stat(dir).st_mtime)
             found = self.__db.execute(sql_select, (dir,)).fetchone()
-            if not found or found['last_modified'] < mod_tm:
+            if not found or found['last_modified'] < mod_tm or found['missing'] == 1:
                 out_of_date_folders.append((dir, mod_tm))
         return out_of_date_folders
 
@@ -306,7 +331,7 @@ def __get_modified_files(self, modified_folders):
 
     def __insert_file(self, file):
         file_insert = "INSERT OR REPLACE INTO file(folder_id, basename, extension, last_modified) VALUES((SELECT folder_id from folder where name = ?), ?, ?, ?)"
-        folder_insert = "INSERT OR IGNORE INTO folder(name) VALUES(?)"
+        folder_insert = "INSERT OR REPLACE INTO folder(name, missing) VALUES(?, 0)"
         mod_tm =  os.path.getmtime(file)
         dir, file_only = os.path.split(file)
         base, extension = os.path.splitext(file_only)
@@ -344,22 +369,26 @@ def __purge_missing_files_and_folders(self):
             if not os.path.exists(row['name']):
                 folder_id_list.append([row['folder_id']])
 
-        # Delete any non-existent folders from the db. Note, this will automatically
+        # Flag or delete any non-existent folders from the db. Note, deleting will automatically
         # remove orphaned records from the 'file' and 'meta' tables
         if len(folder_id_list):
-            self.__db.executemany('DELETE FROM folder WHERE folder_id = ?', folder_id_list)
+            if self.__purge_files:
+                self.__db.executemany('DELETE FROM folder WHERE folder_id = ?', folder_id_list)
+            else:
+                self.__db.executemany('UPDATE folder SET missing = 1 WHERE folder_id = ?', folder_id_list)
 
         # Find files in the db that are no longer on disk
-        file_id_list = []
-        for row in self.__db.execute('SELECT file_id, fname from all_data'):
-            if not os.path.exists(row['fname']):
-                file_id_list.append([row['file_id']])
-
-        # Delete any non-existent files from the db. Note, this will automatically
-        # remove matching records from the 'meta' table as well.
-        if len(file_id_list):
-            self.__db.executemany('DELETE FROM file WHERE file_id = ?', file_id_list)
-
+        if self.__purge_files:
+            file_id_list = []
+            for row in self.__db.execute('SELECT file_id, fname from all_data'):
+                if not os.path.exists(row['fname']):
+                    file_id_list.append([row['file_id']])
+
+            # Delete any non-existent files from the db. Note, this will automatically
+            # remove matching records from the 'meta' table as well.
+            if len(file_id_list):
+                self.__db.executemany('DELETE FROM file WHERE file_id = ?', file_id_list)
+            self.__purge_files = False
 
     def __get_exif_info(self, file_path_name):
         exifs = get_image_meta.GetImageMeta(file_path_name)

picframe/interface_http.py

@@ -29,7 +29,8 @@ def heif_to_jpg(fname):
         image.save("/dev/shm/temp.jpg") # default 75% quality
         return "/dev/shm/temp.jpg"
     except:
-        self.__logger.warning("Failed attempt to convert %s \n** Have you installed pyheif? **", fname)
+        logger = logging.getLogger("interface_http.heif_to_jpg")
+        logger.warning("Failed attempt to convert %s \n** Have you installed pyheif? **", fname)
         return "" # this will not render as a page and will generate error TODO serve specific page with explicit error
 
 class RequestHandler(BaseHTTPRequestHandler):
@@ -80,22 +81,23 @@ def do_GET(self):
                         for subkey in self.server._setters:
                             message[subkey] = getattr(self.server._controller, subkey)
                     elif key in dir(self.server._controller):
-                        if key in self.server._setters: # can info back from controller
+                        if key in self.server._setters: # can get info back from controller TODO 
                             message[key] = getattr(self.server._controller, key)
-                        lwr_val = value.lower()
-                        if lwr_val in ("true", "on", "yes"): # this only works for simple values *not* json style kwargs
-                            value = True
-                        elif lwr_val in ("false", "off", "no"):
-                            value = False
-                        try:
-                            if key in self.server._setters:
-                                setattr(self.server._controller, key, value)
-                            else:
-                                value = value.replace("\'", "\"") # only " permitted in json
-                                # value must be json kwargs
-                                getattr(self.server._controller, key)(**json.loads(value))
-                        except Exception as e:
-                            message['ERROR'] = 'Excepton:{}>{};'.format(key, e)
+                        if value != "": # parse_qsl can return empty string for value when just querying
+                            lwr_val = value.lower()
+                            if lwr_val in ("true", "on", "yes"): # this only works for simple values *not* json style kwargs
+                                value = True
+                            elif lwr_val in ("false", "off", "no"):
+                                value = False
+                            try:
+                                if key in self.server._setters:
+                                    setattr(self.server._controller, key, value)
+                                else:
+                                    value = value.replace("\'", "\"") # only " permitted in json
+                                    # value must be json kwargs
+                                    getattr(self.server._controller, key)(**json.loads(value))
+                            except Exception as e:
+                                message['ERROR'] = 'Excepton:{}>{};'.format(key, e)
 
                     self.wfile.write(bytes(json.dumps(message), "utf8"))
                     self.connection.close()
@@ -142,9 +144,10 @@ def __init__(self, controller, html_path, pic_dir, no_files_img, port=9000):
         self._pic_dir = os.path.expanduser(pic_dir)
         self._no_files_img = os.path.expanduser(no_files_img)
         self._html_path = os.path.expanduser(html_path)
-        self._setters = ["paused", "subdirectory", "date_from", "date_to",
-                         "display_is_on", "shuffle", "fade_time", "time_delay",
-                         "brightness", "location_filter"] #TODO can this be done with dir() and getattr() to avoid hard coding?
+        # TODO check below works with all decorated methods.. seems to work
+        controller_class = controller.__class__
+        self._setters = [method for method in dir(controller_class)
+                            if 'setter' in dir(getattr(controller_class, method))]
         self.__keep_looping = True
         self.__shutdown_completed = False
         t = threading.Thread(target=self.__loop)

picframe/interface_mqtt.py

@@ -115,6 +115,7 @@ def on_connect(self, client, userdata, flags, rc):
         self.__setup_switch(client, switch_topic_head, "_back", "mdi:skip-previous", available_topic)
         self.__setup_switch(client, switch_topic_head, "_next", "mdi:skip-next", available_topic)
 
+        client.subscribe(self.__device_id + "/purge_files", qos=0) # close down without killing!
         client.subscribe(self.__device_id + "/stop", qos=0) # close down without killing!
 
     def __setup_sensor(self, client, sensor_topic_head, topic, icon, available_topic, has_attributes=False):
@@ -307,6 +308,10 @@ def on_message(self, client, userdata, message):
             self.__logger.info("Recieved tags filter: %s", msg)
             self.__controller.tags_filter = msg
 
+        # set the flag to purge files from database
+        elif message.topic == self.__device_id + "/purge_files":
+            self.__controller.purge_files()
+
         # stop loops and end program
         elif message.topic == self.__device_id + "/stop":
             self.__controller.stop()

picframe/model.py

@@ -125,7 +125,6 @@ def __init__(self, configfile = DEFAULT_CONFIGFILE):
         self.__logger.debug('creating an instance of Model')
         self.__config = DEFAULT_CONFIG
         self.__last_file_change = 0.0
-        self.__required_db_schema_version = 1
         configfile = os.path.expanduser(configfile)
         self.__logger.info("Open config file: %s:",configfile)
         with open(configfile, 'r') as stream:
@@ -157,7 +156,6 @@ def __init__(self, configfile = DEFAULT_CONFIGFILE):
         self.__image_cache = image_cache.ImageCache(self.__pic_dir,
                                                     os.path.expanduser(model_config['db_file']),
                                                     self.__geo_reverse,
-                                                    self.__required_db_schema_version,
                                                     model_config['portrait_pairs'])
         self.__deleted_pictures = model_config['deleted_pictures']
         self.__no_files_img = os.path.expanduser(model_config['no_files_img'])
@@ -242,6 +240,9 @@ def pause_looping(self, val):
     def stop_image_chache(self):
         self.__image_cache.stop()
 
+    def purge_files(self):
+        self.__image_cache.purge_files()
+
     def get_directory_list(self):
         _, root = os.path.split(self.__pic_dir)
         actual_dir = root

setup.py

@@ -33,7 +33,7 @@
       install_requires=[
         'Pillow',
         'ExifRead',
-        'pi3d>=2.44',
+        'pi3d>=2.45',
         'PyYAML',
         'paho-mqtt',
         'IPTCInfo3',