From 3c570421d3a2eb24528b5f5bb615dcb0c7717e4a Mon Sep 17 00:00:00 2001 From: wfjsw Date: Tue, 18 Jul 2023 19:00:16 +0800 Subject: move start timer --- modules/launch_utils.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 03552bc2..ea995eda 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -10,9 +10,6 @@ from functools import lru_cache from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir -from modules import timer - -timer.startup_timer.record("start") args, _ = cmd_args.parser.parse_known_args() -- cgit v1.2.1 From 118529a6dc9481499ebcd589a9b42f98b58bca8a Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Fri, 21 Jul 2023 21:49:33 +0800 Subject: typo fix --- modules/launch_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 03552bc2..8e11bf42 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -233,7 +233,7 @@ def run_extensions_installers(settings_file): re_requirement = re.compile(r"\s*([-_a-zA-Z0-9]+)\s*(?:==\s*([-+_.a-zA-Z0-9]+))?\s*") -def requrements_met(requirements_file): +def requirements_met(requirements_file): """ Does a simple parse of a requirements.txt file to determine if all rerqirements in it are already installed. Returns True if so, False if not installed or parsing fails. @@ -293,7 +293,7 @@ def prepare_environment(): try: # the existance of this file is a signal to webui.sh/bat that webui needs to be restarted when it stops execution os.remove(os.path.join(script_path, "tmp", "restart")) - os.environ.setdefault('SD_WEBUI_RESTARTING ', '1') + os.environ.setdefault('SD_WEBUI_RESTARTING', '1') except OSError: pass @@ -354,7 +354,7 @@ def prepare_environment(): if not os.path.isfile(requirements_file): requirements_file = os.path.join(script_path, requirements_file) - if not requrements_met(requirements_file): + if not requirements_met(requirements_file): run_pip(f"install -r \"{requirements_file}\"", "requirements") run_extensions_installers(settings_file=args.ui_settings_file) -- cgit v1.2.1 From 16eddc622e6d091f51d22269742afddc9b6d0f4b Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Fri, 21 Jul 2023 22:00:03 +0800 Subject: prepend the pythonpath instead of overriding it --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 609a181e..231ebc5f 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -190,7 +190,7 @@ def run_extension_installer(extension_dir): try: env = os.environ.copy() - env['PYTHONPATH'] = os.path.abspath(".") + env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env['PYTHONPATH']}" print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env)) except Exception as e: -- cgit v1.2.1 From 90eb731ff1d73fdc5872ff9682d5c88c9737ba38 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 22 Jul 2023 12:21:05 +0300 Subject: start timer early anyway --- modules/launch_utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index aaa671ab..18b444d4 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -10,6 +10,7 @@ from functools import lru_cache from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir +from modules import timer # noqa:F401 args, _ = cmd_args.parser.parse_known_args() -- cgit v1.2.1 From 7afe7375e104d85542f7572ca0f8559bb4e3a7fe Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Sat, 22 Jul 2023 17:46:50 +0800 Subject: display a progressbar for extension installer --- modules/launch_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 18b444d4..b827debe 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -7,6 +7,7 @@ import importlib.util import platform import json from functools import lru_cache +from tqdm.auto import tqdm from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir @@ -224,7 +225,10 @@ def run_extensions_installers(settings_file): if not os.path.isdir(extensions_dir): return - for dirname_extension in list_extensions(settings_file): + pbar_extensions = tqdm(list_extensions(settings_file), + bar_format="{desc}: |{bar}|{percentage:3.0f}% [{n_fmt}/{total_fmt} {elapsed}<{remaining}]") + for dirname_extension in pbar_extensions: + pbar_extensions.set_description("Installing %s" % dirname_extension) run_extension_installer(os.path.join(extensions_dir, dirname_extension)) -- cgit v1.2.1 From b2f0040da7df65a5deac34f59263cf518d970dcd Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Sat, 22 Jul 2023 17:51:15 +0800 Subject: fix tqdm not found on new instance --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index b827debe..43256072 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -7,7 +7,6 @@ import importlib.util import platform import json from functools import lru_cache -from tqdm.auto import tqdm from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir @@ -225,6 +224,7 @@ def run_extensions_installers(settings_file): if not os.path.isdir(extensions_dir): return + from tqdm.auto import tqdm pbar_extensions = tqdm(list_extensions(settings_file), bar_format="{desc}: |{bar}|{percentage:3.0f}% [{n_fmt}/{total_fmt} {elapsed}<{remaining}]") for dirname_extension in pbar_extensions: -- cgit v1.2.1 From 2a7e34fe79c82ac264919be8965b8d51afa32ac1 Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Sat, 22 Jul 2023 18:09:00 +0800 Subject: fix https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/11921#issuecomment-1646547908 --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 18b444d4..0c863e4a 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -194,7 +194,7 @@ def run_extension_installer(extension_dir): try: env = os.environ.copy() - env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env['PYTHONPATH']}" + env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env.get('PYTHONPATH')}" print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env)) except Exception as e: -- cgit v1.2.1 From 3c26734d603560f57f7a145eed6cbdd50d20379c Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Sat, 22 Jul 2023 18:33:59 +0800 Subject: nop --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 0c863e4a..c61edb36 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -194,7 +194,7 @@ def run_extension_installer(extension_dir): try: env = os.environ.copy() - env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env.get('PYTHONPATH')}" + env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env.get('PYTHONPATH', '')}" print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env)) except Exception as e: -- cgit v1.2.1 From c76a30af41e50932847230631d26bfa9635ebd62 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 22 Jul 2023 13:49:29 +0300 Subject: more info for startup timings --- modules/launch_utils.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 18b444d4..0178e1b0 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -10,7 +10,7 @@ from functools import lru_cache from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir -from modules import timer # noqa:F401 +from modules.timer import startup_timer args, _ = cmd_args.parser.parse_known_args() @@ -224,8 +224,10 @@ def run_extensions_installers(settings_file): if not os.path.isdir(extensions_dir): return - for dirname_extension in list_extensions(settings_file): - run_extension_installer(os.path.join(extensions_dir, dirname_extension)) + with startup_timer.subcategory("run extensions installers"): + for dirname_extension in list_extensions(settings_file): + run_extension_installer(os.path.join(extensions_dir, dirname_extension)) + startup_timer.record(dirname_extension) re_requirement = re.compile(r"\s*([-_a-zA-Z0-9]+)\s*(?:==\s*([-+_.a-zA-Z0-9]+))?\s*") @@ -298,8 +300,11 @@ def prepare_environment(): if not args.skip_python_version_check: check_python_version() + startup_timer.record("checks") + commit = commit_hash() tag = git_tag() + startup_timer.record("git version info") print(f"Python {sys.version}") print(f"Version: {tag}") @@ -307,21 +312,27 @@ def prepare_environment(): if args.reinstall_torch or not is_installed("torch") or not is_installed("torchvision"): run(f'"{python}" -m {torch_command}', "Installing torch and torchvision", "Couldn't install torch", live=True) + startup_timer.record("install torch") if not args.skip_torch_cuda_test and not check_run_python("import torch; assert torch.cuda.is_available()"): raise RuntimeError( 'Torch is not able to use GPU; ' 'add --skip-torch-cuda-test to COMMANDLINE_ARGS variable to disable this check' ) + startup_timer.record("torch GPU test") + if not is_installed("gfpgan"): run_pip(f"install {gfpgan_package}", "gfpgan") + startup_timer.record("install gfpgan") if not is_installed("clip"): run_pip(f"install {clip_package}", "clip") + startup_timer.record("install clip") if not is_installed("open_clip"): run_pip(f"install {openclip_package}", "open_clip") + startup_timer.record("install open_clip") if (not is_installed("xformers") or args.reinstall_xformers) and args.xformers: if platform.system() == "Windows": @@ -335,8 +346,11 @@ def prepare_environment(): elif platform.system() == "Linux": run_pip(f"install -U -I --no-deps {xformers_package}", "xformers") + startup_timer.record("install xformers") + if not is_installed("ngrok") and args.ngrok: run_pip("install ngrok", "ngrok") + startup_timer.record("install ngrok") os.makedirs(os.path.join(script_path, dir_repos), exist_ok=True) @@ -346,22 +360,28 @@ def prepare_environment(): git_clone(codeformer_repo, repo_dir('CodeFormer'), "CodeFormer", codeformer_commit_hash) git_clone(blip_repo, repo_dir('BLIP'), "BLIP", blip_commit_hash) + startup_timer.record("clone repositores") + if not is_installed("lpips"): run_pip(f"install -r \"{os.path.join(repo_dir('CodeFormer'), 'requirements.txt')}\"", "requirements for CodeFormer") + startup_timer.record("install CodeFormer requirements") if not os.path.isfile(requirements_file): requirements_file = os.path.join(script_path, requirements_file) if not requirements_met(requirements_file): run_pip(f"install -r \"{requirements_file}\"", "requirements") + startup_timer.record("install requirements") run_extensions_installers(settings_file=args.ui_settings_file) if args.update_check: version_check(commit) + startup_timer.record("check version") if args.update_all_extensions: git_pull_recursive(extensions_dir) + startup_timer.record("update extensions") if "--exit" in sys.argv: print("Exiting because of --exit argument") -- cgit v1.2.1 From a8d4213317c6970aa3ca8cbeeaacb07b936b591c Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 22 Jul 2023 17:08:45 +0300 Subject: add --log-startup option to print detailed startup progress --- modules/launch_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index c9e4344b..f77b577a 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -226,8 +226,11 @@ def run_extensions_installers(settings_file): with startup_timer.subcategory("run extensions installers"): for dirname_extension in list_extensions(settings_file): - run_extension_installer(os.path.join(extensions_dir, dirname_extension)) - startup_timer.record(dirname_extension) + path = os.path.join(extensions_dir, dirname_extension) + + if os.path.isdir(path): + run_extension_installer(path) + startup_timer.record(dirname_extension) re_requirement = re.compile(r"\s*([-_a-zA-Z0-9]+)\s*(?:==\s*([-+_.a-zA-Z0-9]+))?\s*") -- cgit v1.2.1 From 2f1d5b6b04fd38b1fca1b0193b800533398d91ca Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Tue, 1 Aug 2023 11:20:59 +0800 Subject: attempt to fix workspace status when doing git clone --- modules/launch_utils.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index f77b577a..c7bb9370 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -139,6 +139,12 @@ def check_run_python(code: str) -> bool: return result.returncode == 0 +def git_fix_workspace(dir): + run(f'"{git}" -C "{dir}" fetch --refetch --no-auto-gc', f"Fetching all contents for {name}", f"Couldn't fetch {name}", live=True) + run(f'"{git}" -C "{dir}" gc --aggressive --prune=now', f"Pruning {name}", f"Couldn't prune {name}", live=True) + return + + def git_clone(url, dir, name, commithash=None): # TODO clone into temporary dir and move if successful @@ -151,7 +157,23 @@ def git_clone(url, dir, name, commithash=None): return run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") - run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + + if commithash is not None: + try: + run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + except RuntimeError: + print(f"Unable to checkout {name} with hash {commithash}, attempting autofix...") + git_fix_workspace(dir) + run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + else: + try: + run(f'"{git}" -C "{dir}" reset --hard FETCH_HEAD', f"Checking out latest commit for {name}...", f"Couldn't checkout latest commit for {name}", live=True) + except RuntimeError: + print(f"Unable to checkout {name}, attempting autofix...") + git_fix_workspace(dir) + run(f'"{git}" -C "{dir}" reset --hard FETCH_HEAD', f"Checking out latest commit for {name}...", f"Couldn't checkout latest commit for {name}", live=True) + + return run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}", live=True) -- cgit v1.2.1 From 955542a6540e3c2c27b39dc515c0ee3f8044b57b Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Tue, 1 Aug 2023 11:24:54 +0800 Subject: also check on rev-parse --- modules/launch_utils.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index c7bb9370..87c577e0 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -152,27 +152,25 @@ def git_clone(url, dir, name, commithash=None): if commithash is None: return - current_hash = run(f'"{git}" -C "{dir}" rev-parse HEAD', None, f"Couldn't determine {name}'s hash: {commithash}", live=False).strip() - if current_hash == commithash: - return + try: + current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() + if current_hash == commithash: + return + except RuntimeError: + print(f"Unable to determine {name}'s hash, attempting autofix...") + git_fix_workspace(dir) + current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() + if current_hash == commithash: + return run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") - if commithash is not None: - try: - run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) - except RuntimeError: - print(f"Unable to checkout {name} with hash {commithash}, attempting autofix...") - git_fix_workspace(dir) - run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) - else: - try: - run(f'"{git}" -C "{dir}" reset --hard FETCH_HEAD', f"Checking out latest commit for {name}...", f"Couldn't checkout latest commit for {name}", live=True) - except RuntimeError: - print(f"Unable to checkout {name}, attempting autofix...") - git_fix_workspace(dir) - run(f'"{git}" -C "{dir}" reset --hard FETCH_HEAD', f"Checking out latest commit for {name}...", f"Couldn't checkout latest commit for {name}", live=True) - + try: + run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + except RuntimeError: + print(f"Unable to checkout {name} with hash {commithash}, attempting autofix...") + git_fix_workspace(dir) + run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From c46525b70b54e4f6eaa8326d20777ecbad959a20 Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Tue, 1 Aug 2023 11:26:17 +0800 Subject: fix exception --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 87c577e0..4be25990 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -156,7 +156,7 @@ def git_clone(url, dir, name, commithash=None): current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() if current_hash == commithash: return - except RuntimeError: + except Exception: print(f"Unable to determine {name}'s hash, attempting autofix...") git_fix_workspace(dir) current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() -- cgit v1.2.1 From 8b036d8a8253996f2a9c977bea63babbe59eb348 Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Tue, 1 Aug 2023 11:26:59 +0800 Subject: fix --- modules/launch_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 4be25990..7225af08 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -139,7 +139,7 @@ def check_run_python(code: str) -> bool: return result.returncode == 0 -def git_fix_workspace(dir): +def git_fix_workspace(dir, name): run(f'"{git}" -C "{dir}" fetch --refetch --no-auto-gc', f"Fetching all contents for {name}", f"Couldn't fetch {name}", live=True) run(f'"{git}" -C "{dir}" gc --aggressive --prune=now', f"Pruning {name}", f"Couldn't prune {name}", live=True) return @@ -158,7 +158,7 @@ def git_clone(url, dir, name, commithash=None): return except Exception: print(f"Unable to determine {name}'s hash, attempting autofix...") - git_fix_workspace(dir) + git_fix_workspace(dir, name) current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() if current_hash == commithash: return @@ -169,7 +169,7 @@ def git_clone(url, dir, name, commithash=None): run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) except RuntimeError: print(f"Unable to checkout {name} with hash {commithash}, attempting autofix...") - git_fix_workspace(dir) + git_fix_workspace(dir, name) run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From 0ea20a0d526a531f3d329b62625900a4a18f364e Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Mon, 7 Aug 2023 08:38:18 +0300 Subject: rework #12230 to not have duplicate code --- modules/launch_utils.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 7225af08..5be30a18 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -145,6 +145,21 @@ def git_fix_workspace(dir, name): return +def run_git(dir, name, command, desc=None, errdesc=None, custom_env=None, live: bool = default_command_live, autofix=True): + try: + return run(f'"{git}" -C "{dir}" {command}', desc=desc, errdesc=errdesc, custom_env=custom_env, live=live) + except RuntimeError: + pass + + if not autofix: + return None + + print(f"{errdesc}, attempting autofix...") + git_fix_workspace(dir, name) + + return run(f'"{git}" -C "{dir}" {command}', desc=desc, errdesc=errdesc, custom_env=custom_env, live=live) + + def git_clone(url, dir, name, commithash=None): # TODO clone into temporary dir and move if successful @@ -152,25 +167,13 @@ def git_clone(url, dir, name, commithash=None): if commithash is None: return - try: - current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() - if current_hash == commithash: - return - except Exception: - print(f"Unable to determine {name}'s hash, attempting autofix...") - git_fix_workspace(dir, name) - current_hash = subprocess.check_output([git, "-C", dir, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip() - if current_hash == commithash: - return + current_hash = run_git(dir, name, 'rev-parse HEAD', None, f"Couldn't determine {name}'s hash: {commithash}", live=False).strip() + if current_hash == commithash: + return - run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") + run_git('fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") - try: - run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) - except RuntimeError: - print(f"Unable to checkout {name} with hash {commithash}, attempting autofix...") - git_fix_workspace(dir, name) - run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + run_git('checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From a2a97e57f069d24c9616d1867a6df7f532f92a04 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Wed, 9 Aug 2023 17:08:36 +0300 Subject: simplify --- modules/launch_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index c0847e14..98ac5bbd 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -7,6 +7,7 @@ import importlib.util import platform import json from functools import lru_cache +import tqdm from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir @@ -248,11 +249,10 @@ def run_extensions_installers(settings_file): return with startup_timer.subcategory("run extensions installers"): - from tqdm.auto import tqdm - pbar_extensions = tqdm(list_extensions(settings_file), - bar_format="{desc}: |{bar}|{percentage:3.0f}% [{n_fmt}/{total_fmt} {elapsed}<{remaining}]") - for dirname_extension in pbar_extensions: - pbar_extensions.set_description("Installing %s" % dirname_extension) + progress_bar = tqdm.tqdm(list_extensions(settings_file)) + for dirname_extension in progress_bar: + progress_bar.set_description(f"Installing {dirname_extension}") + path = os.path.join(extensions_dir, dirname_extension) if os.path.isdir(path): -- cgit v1.2.1 From 8b7b99f8d582edf6422e52f41b82f08cdb80568d Mon Sep 17 00:00:00 2001 From: catboxanon <122327233+catboxanon@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:18:03 -0400 Subject: fix: Only import tqdm when needed --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 98ac5bbd..7143f144 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -7,7 +7,6 @@ import importlib.util import platform import json from functools import lru_cache -import tqdm from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir @@ -249,6 +248,7 @@ def run_extensions_installers(settings_file): return with startup_timer.subcategory("run extensions installers"): + import tqdm progress_bar = tqdm.tqdm(list_extensions(settings_file)) for dirname_extension in progress_bar: progress_bar.set_description(f"Installing {dirname_extension}") -- cgit v1.2.1 From edfae9e78af23bdd6161c55c7ec88533de8925f8 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Wed, 9 Aug 2023 20:49:33 +0300 Subject: add --loglevel commandline argument for logging remove the progressbar for extension installation in favor of logging output --- modules/launch_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 7143f144..90c00dd2 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -1,4 +1,5 @@ # this scripts installs necessary requirements and launches main program in webui.py +import logging import re import subprocess import os @@ -11,8 +12,10 @@ from functools import lru_cache from modules import cmd_args, errors from modules.paths_internal import script_path, extensions_dir from modules.timer import startup_timer +from modules import logging_config args, _ = cmd_args.parser.parse_known_args() +logging_config.setup_logging(args.loglevel) python = sys.executable git = os.environ.get('GIT', "git") @@ -248,10 +251,8 @@ def run_extensions_installers(settings_file): return with startup_timer.subcategory("run extensions installers"): - import tqdm - progress_bar = tqdm.tqdm(list_extensions(settings_file)) - for dirname_extension in progress_bar: - progress_bar.set_description(f"Installing {dirname_extension}") + for dirname_extension in list_extensions(settings_file): + logging.debug(f"Installing {dirname_extension}") path = os.path.join(extensions_dir, dirname_extension) -- cgit v1.2.1 From 5a705c246880c26ec4fc940dae0da5ecdd2bff50 Mon Sep 17 00:00:00 2001 From: Jabasukuriputo Wang Date: Thu, 10 Aug 2023 09:18:10 -0500 Subject: rm dir on failed clone, disable autofix for fetch --- modules/launch_utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 90c00dd2..953f968b 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -3,6 +3,7 @@ import logging import re import subprocess import os +import shutil import sys import importlib.util import platform @@ -152,10 +153,8 @@ def run_git(dir, name, command, desc=None, errdesc=None, custom_env=None, live: try: return run(f'"{git}" -C "{dir}" {command}', desc=desc, errdesc=errdesc, custom_env=custom_env, live=live) except RuntimeError: - pass - - if not autofix: - return None + if not autofix: + raise print(f"{errdesc}, attempting autofix...") git_fix_workspace(dir, name) @@ -174,13 +173,17 @@ def git_clone(url, dir, name, commithash=None): if current_hash == commithash: return - run_git('fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") + run_git('fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) run_git('checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return - run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}", live=True) + try: + run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}", live=True) + except RuntimeError: + shutil.rmtree(dir, ignore_errors=True) + raise if commithash is not None: run(f'"{git}" -C "{dir}" checkout {commithash}', None, "Couldn't checkout {name}'s hash: {commithash}") -- cgit v1.2.1 From e0906096c57c25c7ecaa583f20af84b1699b8736 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Thu, 10 Aug 2023 17:22:08 +0300 Subject: remove unnecessary GFPGAN_PACKAGE (we install GFPGAN from the requirements file) --- modules/launch_utils.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 90c00dd2..bc5f38cc 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -305,7 +305,6 @@ def prepare_environment(): requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt") xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.20') - gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "https://github.com/TencentARC/GFPGAN/archive/8d2447a2d918f8eba5a4a01463fd48e45126a379.zip") clip_package = os.environ.get('CLIP_PACKAGE', "https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip") openclip_package = os.environ.get('OPENCLIP_PACKAGE', "https://github.com/mlfoundations/open_clip/archive/bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b.zip") @@ -352,11 +351,6 @@ def prepare_environment(): ) startup_timer.record("torch GPU test") - - if not is_installed("gfpgan"): - run_pip(f"install {gfpgan_package}", "gfpgan") - startup_timer.record("install gfpgan") - if not is_installed("clip"): run_pip(f"install {clip_package}", "clip") startup_timer.record("install clip") -- cgit v1.2.1 From f57bc1a21ba3979061752bd7b66e881bd25cc64f Mon Sep 17 00:00:00 2001 From: w-e-w <40751091+w-e-w@users.noreply.github.com> Date: Sat, 12 Aug 2023 12:06:31 +0900 Subject: disable extensions installer with arg --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 2782872e..65eb684f 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -243,7 +243,7 @@ def list_extensions(settings_file): disabled_extensions = set(settings.get('disabled_extensions', [])) disable_all_extensions = settings.get('disable_all_extensions', 'none') - if disable_all_extensions != 'none': + if disable_all_extensions != 'none' or args.disable_extra_extensions or args.disable_all_extensions: return [] return [x for x in os.listdir(extensions_dir) if x not in disabled_extensions] -- cgit v1.2.1 From 1ae9dacb4b036a6cb4b5fb9b9ff030962f43908e Mon Sep 17 00:00:00 2001 From: catboxanon <122327233+catboxanon@users.noreply.github.com> Date: Sun, 13 Aug 2023 07:57:29 -0400 Subject: Add DPM-Solver++(3M) SDE --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 65eb684f..e30fbac8 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -319,7 +319,7 @@ def prepare_environment(): stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") stable_diffusion_xl_commit_hash = os.environ.get('STABLE_DIFFUSION_XL_COMMIT_HASH', "5c10deee76adad0032b412294130090932317a87") - k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "c9fe758757e022f05ca5a53fa8fac28889e4f1cf") + k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "ab527a9a6d347f364e3d185ba6d714e22d80cb3c") codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af") blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9") -- cgit v1.2.1 From 3163d1269af7f9fd95382e58bb1581fd741b5119 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sun, 13 Aug 2023 16:51:21 +0300 Subject: fix for the broken run_git calls --- modules/launch_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index e30fbac8..4fc254a2 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -173,9 +173,9 @@ def git_clone(url, dir, name, commithash=None): if current_hash == commithash: return - run_git('fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) + run_git(dir, name, 'fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) - run_git('checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + run_git(dir, name, 'checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From 9058620cec2788495d295f4e68ef2932d6d700e6 Mon Sep 17 00:00:00 2001 From: brkirch Date: Sat, 12 Aug 2023 04:44:16 -0400 Subject: `git checkout` with commit hash --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 4fc254a2..e77baa52 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -175,7 +175,7 @@ def git_clone(url, dir, name, commithash=None): run_git(dir, name, 'fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) - run_git(dir, name, 'checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + run_git(dir, name, f'checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From f4dbb0c820344798e3481d4104618b95594a3d10 Mon Sep 17 00:00:00 2001 From: brkirch Date: Thu, 20 Jul 2023 01:44:45 -0400 Subject: Change the repositories origin URLs when necessary --- modules/launch_utils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index e77baa52..9eda7c9d 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -173,6 +173,9 @@ def git_clone(url, dir, name, commithash=None): if current_hash == commithash: return + if run_git(dir, name, 'config --get remote.origin.url', None, f"Couldn't determine {name}'s origin URL", live=False).strip() != url: + run_git(dir, name, f'remote set-url origin "{url}"', None, f"Failed to set {name}'s origin URL", live=False) + run_git(dir, name, 'fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) run_git(dir, name, f'checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) -- cgit v1.2.1 From 09ff5b5416e9e989cf2ddb2bab9129e27ed23f14 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Mon, 14 Aug 2023 01:03:49 +0900 Subject: Fix typo in launch_utils.py existance -> existence --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index e1c9cfbe..8d2256ee 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -291,7 +291,7 @@ def prepare_environment(): blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9") try: - # the existance of this file is a signal to webui.sh/bat that webui needs to be restarted when it stops execution + # the existence of this file is a signal to webui.sh/bat that webui needs to be restarted when it stops execution os.remove(os.path.join(script_path, "tmp", "restart")) os.environ.setdefault('SD_WEBUI_RESTARTING', '1') except OSError: -- cgit v1.2.1 From 16781ba09abe1494993f819b91ea0b88c48903b7 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sun, 13 Aug 2023 20:15:20 +0300 Subject: fix 2 for git code botched by previous PRs --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 4fc254a2..e77baa52 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -175,7 +175,7 @@ def git_clone(url, dir, name, commithash=None): run_git(dir, name, 'fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}", autofix=False) - run_git(dir, name, 'checkout', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) + run_git(dir, name, f'checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}", live=True) return -- cgit v1.2.1 From bc63339df3d489de9d1b10c936c87be32bd68790 Mon Sep 17 00:00:00 2001 From: brkirch Date: Wed, 26 Jul 2023 23:05:16 -0400 Subject: Update hash for SD XL Repo --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 449a8755..444f16fa 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -321,7 +321,7 @@ def prepare_environment(): blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git') stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf") - stable_diffusion_xl_commit_hash = os.environ.get('STABLE_DIFFUSION_XL_COMMIT_HASH', "5c10deee76adad0032b412294130090932317a87") + stable_diffusion_xl_commit_hash = os.environ.get('STABLE_DIFFUSION_XL_COMMIT_HASH', "45c443b316737a4ab6e40413d7794a7f5657c19f") k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "ab527a9a6d347f364e3d185ba6d714e22d80cb3c") codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af") blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9") -- cgit v1.2.1 From e1a29266b29455cfd9bb6be207aba5abbd177b37 Mon Sep 17 00:00:00 2001 From: w-e-w <40751091+w-e-w@users.noreply.github.com> Date: Thu, 17 Aug 2023 00:24:24 +0900 Subject: return empty list if extensions_dir not exist --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 444f16fa..7e4d5a61 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -246,7 +246,7 @@ def list_extensions(settings_file): disabled_extensions = set(settings.get('disabled_extensions', [])) disable_all_extensions = settings.get('disable_all_extensions', 'none') - if disable_all_extensions != 'none' or args.disable_extra_extensions or args.disable_all_extensions: + if disable_all_extensions != 'none' or args.disable_extra_extensions or args.disable_all_extensions or not os.path.isdir(extensions_dir): return [] return [x for x in os.listdir(extensions_dir) if x not in disabled_extensions] -- cgit v1.2.1 From 956e1d8d90ed369986c03eded4fb583fa00960e8 Mon Sep 17 00:00:00 2001 From: XDOneDude <106700244+XDOneDude@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:25:59 -0400 Subject: xformers update --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 7e4d5a61..c54e199f 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -310,7 +310,7 @@ def prepare_environment(): torch_command = os.environ.get('TORCH_COMMAND', f"pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url {torch_index_url}") requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt") - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.20') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.21') clip_package = os.environ.get('CLIP_PACKAGE', "https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip") openclip_package = os.environ.get('OPENCLIP_PACKAGE', "https://github.com/mlfoundations/open_clip/archive/bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b.zip") -- cgit v1.2.1 From f084e6bbd0833040f0dc77318475b18da60ae200 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 19 Aug 2023 08:22:12 +0300 Subject: revert xformers back to 0.0.20 --- modules/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/launch_utils.py') diff --git a/modules/launch_utils.py b/modules/launch_utils.py index c54e199f..7e4d5a61 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -310,7 +310,7 @@ def prepare_environment(): torch_command = os.environ.get('TORCH_COMMAND', f"pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url {torch_index_url}") requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt") - xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.21') + xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.20') clip_package = os.environ.get('CLIP_PACKAGE', "https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip") openclip_package = os.environ.get('OPENCLIP_PACKAGE', "https://github.com/mlfoundations/open_clip/archive/bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b.zip") -- cgit v1.2.1