diff options
| -rw-r--r-- | modules/cache.py | 17 | ||||
| -rw-r--r-- | modules/extensions.py | 11 | ||||
| -rw-r--r-- | modules/extra_networks.py | 5 | ||||
| -rw-r--r-- | modules/infotext_utils.py | 4 | ||||
| -rw-r--r-- | modules/launch_utils.py | 7 | ||||
| -rw-r--r-- | modules/postprocessing.py | 7 | ||||
| -rw-r--r-- | modules/shared_init.py | 4 | ||||
| -rw-r--r-- | modules/ui_extra_networks.py | 20 | ||||
| -rw-r--r-- | modules/ui_gradio_extensions.py | 8 | ||||
| -rw-r--r-- | modules/ui_loadsave.py | 5 | ||||
| -rw-r--r-- | modules/util.py | 76 |
11 files changed, 123 insertions, 41 deletions
diff --git a/modules/cache.py b/modules/cache.py index 2d37e7b9..a9822a0e 100644 --- a/modules/cache.py +++ b/modules/cache.py @@ -62,16 +62,15 @@ def cache(subsection): if cache_data is None:
with cache_lock:
if cache_data is None:
- if not os.path.isfile(cache_filename):
+ try:
+ with open(cache_filename, "r", encoding="utf8") as file:
+ cache_data = json.load(file)
+ except FileNotFoundError:
+ cache_data = {}
+ except Exception:
+ os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json"))
+ print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache')
cache_data = {}
- else:
- try:
- with open(cache_filename, "r", encoding="utf8") as file:
- cache_data = json.load(file)
- except Exception:
- os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json"))
- print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache')
- cache_data = {}
s = cache_data.get(subsection, {})
cache_data[subsection] = s
diff --git a/modules/extensions.py b/modules/extensions.py index 1899cd52..99e7ee60 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -32,11 +32,12 @@ class ExtensionMetadata: self.config = configparser.ConfigParser()
filepath = os.path.join(path, self.filename)
- if os.path.isfile(filepath):
- try:
- self.config.read(filepath)
- except Exception:
- errors.report(f"Error reading {self.filename} for extension {canonical_name}.", exc_info=True)
+ # `self.config.read()` will quietly swallow OSErrors (which FileNotFoundError is),
+ # so no need to check whether the file exists beforehand.
+ try:
+ self.config.read(filepath)
+ except Exception:
+ errors.report(f"Error reading {self.filename} for extension {canonical_name}.", exc_info=True)
self.canonical_name = self.config.get("Extension", "Name", fallback=canonical_name)
self.canonical_name = canonical_name.lower().strip()
diff --git a/modules/extra_networks.py b/modules/extra_networks.py index b9533677..04249dff 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -206,7 +206,7 @@ def parse_prompts(prompts): return res, extra_data
-def get_user_metadata(filename):
+def get_user_metadata(filename, lister=None):
if filename is None:
return {}
@@ -215,7 +215,8 @@ def get_user_metadata(filename): metadata = {}
try:
- if os.path.isfile(metadata_filename):
+ exists = lister.exists(metadata_filename) if lister else os.path.exists(metadata_filename)
+ if exists:
with open(metadata_filename, "r", encoding="utf8") as file:
metadata = json.load(file)
except Exception as e:
diff --git a/modules/infotext_utils.py b/modules/infotext_utils.py index e582ee47..6978a0bf 100644 --- a/modules/infotext_utils.py +++ b/modules/infotext_utils.py @@ -453,9 +453,11 @@ def connect_paste(button, paste_fields, input_comp, override_settings_component, def paste_func(prompt):
if not prompt and not shared.cmd_opts.hide_ui_dir_config:
filename = os.path.join(data_path, "params.txt")
- if os.path.exists(filename):
+ try:
with open(filename, "r", encoding="utf8") as file:
prompt = file.read()
+ except OSError:
+ pass
params = parse_generation_parameters(prompt)
script_callbacks.infotext_pasted_callback(prompt, params)
diff --git a/modules/launch_utils.py b/modules/launch_utils.py index e2ad412a..7ebdf0b4 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -245,9 +245,10 @@ def list_extensions(settings_file): settings = {}
try:
- if os.path.isfile(settings_file):
- with open(settings_file, "r", encoding="utf8") as file:
- settings = json.load(file)
+ with open(settings_file, "r", encoding="utf8") as file:
+ settings = json.load(file)
+ except FileNotFoundError:
+ pass
except Exception:
errors.report(f'\nCould not load settings\nThe config file "{settings_file}" is likely corrupted\nIt has been moved to the "tmp/config.json"\nReverting config to default\n\n''', exc_info=True)
os.replace(settings_file, os.path.join(script_path, "tmp", "config.json"))
diff --git a/modules/postprocessing.py b/modules/postprocessing.py index 7850328f..7449b0dc 100644 --- a/modules/postprocessing.py +++ b/modules/postprocessing.py @@ -97,11 +97,12 @@ def run_postprocessing(extras_mode, image, image_folder, input_dir, output_dir, if pp.caption:
caption_filename = os.path.splitext(fullfn)[0] + ".txt"
- if os.path.isfile(caption_filename):
+ existing_caption = ""
+ try:
with open(caption_filename, encoding="utf8") as file:
existing_caption = file.read().strip()
- else:
- existing_caption = ""
+ except FileNotFoundError:
+ pass
action = shared.opts.postprocessing_existing_caption_action
if action == 'Prepend' and existing_caption:
diff --git a/modules/shared_init.py b/modules/shared_init.py index d3fb687e..586be342 100644 --- a/modules/shared_init.py +++ b/modules/shared_init.py @@ -18,8 +18,10 @@ def initialize(): shared.options_templates = shared_options.options_templates
shared.opts = options.Options(shared_options.options_templates, shared_options.restricted_opts)
shared.restricted_opts = shared_options.restricted_opts
- if os.path.exists(shared.config_filename):
+ try:
shared.opts.load(shared.config_filename)
+ except FileNotFoundError:
+ pass
from modules import devices
devices.device, devices.device_interrogate, devices.device_gfpgan, devices.device_esrgan, devices.device_codeformer = \
diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index e1c679ec..62db36f5 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -3,7 +3,7 @@ import os.path import urllib.parse
from pathlib import Path
-from modules import shared, ui_extra_networks_user_metadata, errors, extra_networks
+from modules import shared, ui_extra_networks_user_metadata, errors, extra_networks, util
from modules.images import read_info_from_image, save_image_with_geninfo
import gradio as gr
import json
@@ -107,13 +107,14 @@ class ExtraNetworksPage: self.allow_negative_prompt = False
self.metadata = {}
self.items = {}
+ self.lister = util.MassFileLister()
def refresh(self):
pass
def read_user_metadata(self, item):
filename = item.get("filename", None)
- metadata = extra_networks.get_user_metadata(filename)
+ metadata = extra_networks.get_user_metadata(filename, lister=self.lister)
desc = metadata.get("description", None)
if desc is not None:
@@ -123,7 +124,7 @@ class ExtraNetworksPage: def link_preview(self, filename):
quoted_filename = urllib.parse.quote(filename.replace('\\', '/'))
- mtime = os.path.getmtime(filename)
+ mtime, _ = self.lister.mctime(filename)
return f"./sd_extra_networks/thumb?filename={quoted_filename}&mtime={mtime}"
def search_terms_from_path(self, filename, possible_directories=None):
@@ -137,6 +138,8 @@ class ExtraNetworksPage: return ""
def create_html(self, tabname):
+ self.lister.reset()
+
items_html = ''
self.metadata = {}
@@ -282,10 +285,10 @@ class ExtraNetworksPage: List of default keys used for sorting in the UI.
"""
pth = Path(path)
- stat = pth.stat()
+ mtime, ctime = self.lister.mctime(path)
return {
- "date_created": int(stat.st_ctime or 0),
- "date_modified": int(stat.st_mtime or 0),
+ "date_created": int(mtime),
+ "date_modified": int(ctime),
"name": pth.name.lower(),
"path": str(pth.parent).lower(),
}
@@ -298,7 +301,7 @@ class ExtraNetworksPage: potential_files = sum([[path + "." + ext, path + ".preview." + ext] for ext in allowed_preview_extensions()], [])
for file in potential_files:
- if os.path.isfile(file):
+ if self.lister.exists(file):
return self.link_preview(file)
return None
@@ -308,6 +311,9 @@ class ExtraNetworksPage: Find and read a description file for a given path (without extension).
"""
for file in [f"{path}.txt", f"{path}.description.txt"]:
+ if not self.lister.exists(file):
+ continue
+
try:
with open(file, "r", encoding="utf-8", errors="replace") as f:
return f.read()
diff --git a/modules/ui_gradio_extensions.py b/modules/ui_gradio_extensions.py index a86c368e..f5278d22 100644 --- a/modules/ui_gradio_extensions.py +++ b/modules/ui_gradio_extensions.py @@ -35,13 +35,11 @@ def css_html(): return f'<link rel="stylesheet" property="stylesheet" href="{webpath(fn)}">'
for cssfile in scripts.list_files_with_name("style.css"):
- if not os.path.isfile(cssfile):
- continue
-
head += stylesheet(cssfile)
- if os.path.exists(os.path.join(data_path, "user.css")):
- head += stylesheet(os.path.join(data_path, "user.css"))
+ user_css = os.path.join(data_path, "user.css")
+ if os.path.exists(user_css):
+ head += stylesheet(user_css)
return head
diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index 693ff75c..2555cdb6 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -26,8 +26,9 @@ class UiLoadsave: self.ui_defaults_review = None
try:
- if os.path.exists(self.filename):
- self.ui_settings = self.read_from_file()
+ self.ui_settings = self.read_from_file()
+ except FileNotFoundError:
+ pass
except Exception as e:
self.error_loading = True
errors.display(e, "loading settings")
diff --git a/modules/util.py b/modules/util.py index 4861bcb0..ee373e92 100644 --- a/modules/util.py +++ b/modules/util.py @@ -21,11 +21,11 @@ def html_path(filename): def html(filename):
path = html_path(filename)
- if os.path.exists(path):
+ try:
with open(path, encoding="utf8") as file:
return file.read()
-
- return ""
+ except OSError:
+ return ""
def walk_files(path, allowed_extensions=None):
@@ -66,3 +66,73 @@ def truncate_path(target_path, base_path=cwd): except ValueError:
pass
return abs_target
+
+
+class MassFileListerCachedDir:
+ """A class that caches file metadata for a specific directory."""
+
+ def __init__(self, dirname):
+ self.files = None
+ self.files_cased = None
+ self.dirname = dirname
+
+ stats = ((x.name, x.stat(follow_symlinks=False)) for x in os.scandir(self.dirname))
+ files = [(n, s.st_mtime, s.st_ctime) for n, s in stats]
+ self.files = {x[0].lower(): x for x in files}
+ self.files_cased = {x[0]: x for x in files}
+
+
+class MassFileLister:
+ """A class that provides a way to check for the existence and mtime/ctile of files without doing more than one stat call per file."""
+
+ def __init__(self):
+ self.cached_dirs = {}
+
+ def find(self, path):
+ """
+ Find the metadata for a file at the given path.
+
+ Returns:
+ tuple or None: A tuple of (name, mtime, ctime) if the file exists, or None if it does not.
+ """
+
+ dirname, filename = os.path.split(path)
+
+ cached_dir = self.cached_dirs.get(dirname)
+ if cached_dir is None:
+ cached_dir = MassFileListerCachedDir(dirname)
+ self.cached_dirs[dirname] = cached_dir
+
+ stats = cached_dir.files_cased.get(filename)
+ if stats is not None:
+ return stats
+
+ stats = cached_dir.files.get(filename.lower())
+ if stats is None:
+ return None
+
+ try:
+ os_stats = os.stat(path, follow_symlinks=False)
+ return filename, os_stats.st_mtime, os_stats.st_ctime
+ except Exception:
+ return None
+
+ def exists(self, path):
+ """Check if a file exists at the given path."""
+
+ return self.find(path) is not None
+
+ def mctime(self, path):
+ """
+ Get the modification and creation times for a file at the given path.
+
+ Returns:
+ tuple: A tuple of (mtime, ctime) if the file exists, or (0, 0) if it does not.
+ """
+
+ stats = self.find(path)
+ return (0, 0) if stats is None else stats[1:3]
+
+ def reset(self):
+ """Clear the cache of all directories."""
+ self.cached_dirs.clear()
|