From ff1609f91ea0e9a90ba7b6ecc6d794c39c1f8c8f Mon Sep 17 00:00:00 2001 From: Ritesh Gangnani Date: Sun, 5 Nov 2023 19:13:49 +0530 Subject: Add SSD-1B as a supported model --- modules/sd_hijack.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 592f0055..d19f853e 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -180,6 +180,17 @@ class StableDiffusionModelHijack: except Exception as e: errors.display(e, "applying cross attention optimization") undo_optimizations() + + def conv_ssd(self, m): + delattr(m.model.diffusion_model.middle_block, '1') + delattr(m.model.diffusion_model.middle_block, '2') + for i in ['9','8','7','6','5','4']: + delattr(m.model.diffusion_model.input_blocks[7][1].transformer_blocks,i) + delattr(m.model.diffusion_model.input_blocks[8][1].transformer_blocks,i) + delattr(m.model.diffusion_model.output_blocks[0][1].transformer_blocks,i) + delattr(m.model.diffusion_model.output_blocks[1][1].transformer_blocks,i) + delattr(m.model.diffusion_model.output_blocks[4][1].transformer_blocks,'1') + delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks,'1') def hijack(self, m): conditioner = getattr(m, 'conditioner', None) -- cgit v1.2.1 From 44db35fb1ad5d07837e890a0fd3c00addfb0402c Mon Sep 17 00:00:00 2001 From: Ritesh Gangnani Date: Sun, 5 Nov 2023 19:15:38 +0530 Subject: Added memory clearance after deletion --- modules/sd_hijack.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index d19f853e..059ffe8f 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -1,3 +1,5 @@ +import gc + import torch from torch.nn.functional import silu from types import MethodType @@ -190,7 +192,9 @@ class StableDiffusionModelHijack: delattr(m.model.diffusion_model.output_blocks[0][1].transformer_blocks,i) delattr(m.model.diffusion_model.output_blocks[1][1].transformer_blocks,i) delattr(m.model.diffusion_model.output_blocks[4][1].transformer_blocks,'1') - delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks,'1') + delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks,'1') + torch.cuda.empty_cache() + gc.collect() def hijack(self, m): conditioner = getattr(m, 'conditioner', None) -- cgit v1.2.1 From 44c5097375ae4cf40300c09473bb46cf6c5d6cb7 Mon Sep 17 00:00:00 2001 From: Ritesh Gangnani Date: Sun, 5 Nov 2023 20:31:57 +0530 Subject: Use devices.torch_gc() instead of empty_cache() --- modules/sd_hijack.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 059ffe8f..0a7e5135 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -1,5 +1,3 @@ -import gc - import torch from torch.nn.functional import silu from types import MethodType @@ -193,8 +191,7 @@ class StableDiffusionModelHijack: delattr(m.model.diffusion_model.output_blocks[1][1].transformer_blocks,i) delattr(m.model.diffusion_model.output_blocks[4][1].transformer_blocks,'1') delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks,'1') - torch.cuda.empty_cache() - gc.collect() + devices.torch_gc() def hijack(self, m): conditioner = getattr(m, 'conditioner', None) -- cgit v1.2.1 From 80d639a440929e9effe4620ce74333de283e7efc Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sun, 5 Nov 2023 19:32:21 +0300 Subject: linter --- modules/sd_hijack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 4fff418d..c6d17764 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -183,7 +183,7 @@ class StableDiffusionModelHijack: except Exception as e: errors.display(e, "applying cross attention optimization") undo_optimizations() - + def conv_ssd(self, m): delattr(m.model.diffusion_model.middle_block, '1') delattr(m.model.diffusion_model.middle_block, '2') -- cgit v1.2.1 From 6ad666e4794a57dd65790dd6a259d5d4330d45ed Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sun, 5 Nov 2023 19:46:20 +0300 Subject: more changes for #13865: fix formatting, rename the function, add comment and add a readme entry --- modules/sd_hijack.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index c6d17764..fba23c38 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -184,17 +184,19 @@ class StableDiffusionModelHijack: errors.display(e, "applying cross attention optimization") undo_optimizations() - def conv_ssd(self, m): - delattr(m.model.diffusion_model.middle_block, '1') - delattr(m.model.diffusion_model.middle_block, '2') - for i in ['9','8','7','6','5','4']: - delattr(m.model.diffusion_model.input_blocks[7][1].transformer_blocks,i) - delattr(m.model.diffusion_model.input_blocks[8][1].transformer_blocks,i) - delattr(m.model.diffusion_model.output_blocks[0][1].transformer_blocks,i) - delattr(m.model.diffusion_model.output_blocks[1][1].transformer_blocks,i) - delattr(m.model.diffusion_model.output_blocks[4][1].transformer_blocks,'1') - delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks,'1') - devices.torch_gc() + def convert_sdxl_to_ssd(self, m): + """Converts an SDXL model to a Segmind Stable Diffusion model (see https://huggingface.co/segmind/SSD-1B)""" + + delattr(m.model.diffusion_model.middle_block, '1') + delattr(m.model.diffusion_model.middle_block, '2') + for i in ['9', '8', '7', '6', '5', '4']: + delattr(m.model.diffusion_model.input_blocks[7][1].transformer_blocks, i) + delattr(m.model.diffusion_model.input_blocks[8][1].transformer_blocks, i) + delattr(m.model.diffusion_model.output_blocks[0][1].transformer_blocks, i) + delattr(m.model.diffusion_model.output_blocks[1][1].transformer_blocks, i) + delattr(m.model.diffusion_model.output_blocks[4][1].transformer_blocks, '1') + delattr(m.model.diffusion_model.output_blocks[5][1].transformer_blocks, '1') + devices.torch_gc() def hijack(self, m): conditioner = getattr(m, 'conditioner', None) -- cgit v1.2.1 From 9c1c0da026cb7ef091a0f3fa24b14ae8634f6de5 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Mon, 6 Nov 2023 11:17:36 +0300 Subject: fix exception related to the pix2pix --- modules/sd_hijack.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index fba23c38..0157e19f 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -256,8 +256,12 @@ class StableDiffusionModelHijack: self.layers = flatten(m) + import modules.models.diffusion.ddpm_edit + if isinstance(m, ldm.models.diffusion.ddpm.LatentDiffusion): sd_unet.original_forward = ldm_original_forward + elif isinstance(m, modules.models.diffusion.ddpm_edit.LatentDiffusion): + sd_unet.original_forward = ldm_original_forward elif isinstance(m, sgm.models.diffusion.DiffusionEngine): sd_unet.original_forward = sgm_original_forward else: -- cgit v1.2.1 From 293f44e6c1de7bbf744a4236db81ac4559bdb82a Mon Sep 17 00:00:00 2001 From: MrCheeze Date: Fri, 1 Dec 2023 22:56:08 -0500 Subject: Fix bug where is_using_v_parameterization_for_sd2 fails because the sd_hijack is only partially undone --- modules/sd_hijack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 0157e19f..3d340fc9 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -38,9 +38,6 @@ ldm.models.diffusion.ddpm.print = shared.ldm_print optimizers = [] current_optimizer: sd_hijack_optimizations.SdOptimization = None -ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) -sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) - def list_optimizers(): new_optimizers = script_callbacks.list_optimizers_callback() @@ -258,6 +255,9 @@ class StableDiffusionModelHijack: import modules.models.diffusion.ddpm_edit + ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) + sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) + if isinstance(m, ldm.models.diffusion.ddpm.LatentDiffusion): sd_unet.original_forward = ldm_original_forward elif isinstance(m, modules.models.diffusion.ddpm_edit.LatentDiffusion): @@ -303,6 +303,9 @@ class StableDiffusionModelHijack: self.layers = None self.clip = None + patches.undo(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward") + patches.undo(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward") + sd_unet.original_forward = None -- cgit v1.2.1 From a5f61aa8c5933d8e5a0e0aa841138eeaccd86d62 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 2 Dec 2023 18:03:34 +0300 Subject: potential fix for #14172 --- modules/sd_hijack.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 3d340fc9..14fe62c7 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -38,6 +38,10 @@ ldm.models.diffusion.ddpm.print = shared.ldm_print optimizers = [] current_optimizer: sd_hijack_optimizations.SdOptimization = None +ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) +sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) + + def list_optimizers(): new_optimizers = script_callbacks.list_optimizers_callback() @@ -255,9 +259,6 @@ class StableDiffusionModelHijack: import modules.models.diffusion.ddpm_edit - ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) - sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) - if isinstance(m, ldm.models.diffusion.ddpm.LatentDiffusion): sd_unet.original_forward = ldm_original_forward elif isinstance(m, modules.models.diffusion.ddpm_edit.LatentDiffusion): @@ -303,11 +304,6 @@ class StableDiffusionModelHijack: self.layers = None self.clip = None - patches.undo(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward") - patches.undo(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward") - - sd_unet.original_forward = None - def apply_circular(self, enable): if self.circular_enabled == enable: -- cgit v1.2.1 From ac02216e540cd581f9169c6c791e55721e3117b0 Mon Sep 17 00:00:00 2001 From: AUTOMATIC1111 <16777216c@gmail.com> Date: Sat, 2 Dec 2023 19:35:47 +0300 Subject: alternate implementation for unet forward replacement that does not depend on hijack being applied --- modules/sd_hijack.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'modules/sd_hijack.py') diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 14fe62c7..e139d996 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -38,8 +38,11 @@ ldm.models.diffusion.ddpm.print = shared.ldm_print optimizers = [] current_optimizer: sd_hijack_optimizations.SdOptimization = None -ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) -sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sd_unet.UNetModel_forward) +ldm_patched_forward = sd_unet.create_unet_forward(ldm.modules.diffusionmodules.openaimodel.UNetModel.forward) +ldm_original_forward = patches.patch(__file__, ldm.modules.diffusionmodules.openaimodel.UNetModel, "forward", ldm_patched_forward) + +sgm_patched_forward = sd_unet.create_unet_forward(sgm.modules.diffusionmodules.openaimodel.UNetModel.forward) +sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sgm_patched_forward) def list_optimizers(): -- cgit v1.2.1