diff options
| author | Leon Feng <523684+leon0707@users.noreply.github.com> | 2023-07-18 04:24:14 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-07-18 04:24:14 -0400 |
| commit | a3730bd9becd2f1f5d209885b694b0dec178d110 (patch) | |
| tree | 8ac9948d89606f7519df786f07f6ddb93c3d2720 /extensions-builtin/Lora/network_ia3.py | |
| parent | d6668347c8b85b11b696ac56777cc396e34ee1f9 (diff) | |
| parent | 871b8687a82bb2ca907d8a49c87aed7635b8fc33 (diff) | |
Merge branch 'dev' into fix-11805
Diffstat (limited to 'extensions-builtin/Lora/network_ia3.py')
| -rw-r--r-- | extensions-builtin/Lora/network_ia3.py | 30 |
1 files changed, 30 insertions, 0 deletions
diff --git a/extensions-builtin/Lora/network_ia3.py b/extensions-builtin/Lora/network_ia3.py new file mode 100644 index 00000000..7edc4249 --- /dev/null +++ b/extensions-builtin/Lora/network_ia3.py @@ -0,0 +1,30 @@ +import network
+
+
+class ModuleTypeIa3(network.ModuleType):
+ def create_module(self, net: network.Network, weights: network.NetworkWeights):
+ if all(x in weights.w for x in ["weight"]):
+ return NetworkModuleIa3(net, weights)
+
+ return None
+
+
+class NetworkModuleIa3(network.NetworkModule):
+ def __init__(self, net: network.Network, weights: network.NetworkWeights):
+ super().__init__(net, weights)
+
+ self.w = weights.w["weight"]
+ self.on_input = weights.w["on_input"].item()
+
+ def calc_updown(self, orig_weight):
+ w = self.w.to(orig_weight.device, dtype=orig_weight.dtype)
+
+ output_shape = [w.size(0), orig_weight.size(1)]
+ if self.on_input:
+ output_shape.reverse()
+ else:
+ w = w.reshape(-1, 1)
+
+ updown = orig_weight * w
+
+ return self.finalize_updown(updown, orig_weight, output_shape)
|