From 255633016c915afa1cb66a247667bba354deedf9 Mon Sep 17 00:00:00 2001 From: FengWen Date: Thu, 5 Sep 2024 16:10:39 +0800 Subject: [PATCH 01/11] Add SDXL instantid --- bizyair_extras/__init__.py | 1 + bizyair_extras/nodes_comfyui_instantid.py | 162 ++++++++++++++++++ .../commands/processors/prompt_processor.py | 5 +- src/bizyair/configs/models.json | 4 +- src/bizyair/data_types.py | 2 + src/bizyair/nodes_base.py | 19 +- 6 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 bizyair_extras/nodes_comfyui_instantid.py diff --git a/bizyair_extras/__init__.py b/bizyair_extras/__init__.py index 0c496ee2..5d72a370 100644 --- a/bizyair_extras/__init__.py +++ b/bizyair_extras/__init__.py @@ -1,3 +1,4 @@ +from .nodes_comfyui_instantid import * from .nodes_ipadapter_plus.nodes_ipadapter_plus import * from .nodes_kolors_mz import * from .nodes_testing_utils import * diff --git a/bizyair_extras/nodes_comfyui_instantid.py b/bizyair_extras/nodes_comfyui_instantid.py new file mode 100644 index 00000000..30b5f8fa --- /dev/null +++ b/bizyair_extras/nodes_comfyui_instantid.py @@ -0,0 +1,162 @@ +from bizyair import BizyAirBaseNode, data_types +from bizyair.path_utils import path_manager as folder_paths + + +class InstantIDModelLoader(BizyAirBaseNode): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "instantid_file": (folder_paths.get_filename_list("instantid"),) + } + } + + RETURN_TYPES = (data_types.INSTANTID,) + FUNCTION = "default_function" + CATEGORY = "InstantID" + NODE_DISPLAY_NAME = "Load InstantID Model" + + +class ApplyInstantID(BizyAirBaseNode): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "instantid": (data_types.INSTANTID,), + "insightface": (data_types.FACEANALYSIS,), + "control_net": (data_types.CONTROL_NET,), + "image": ("IMAGE",), + "model": (data_types.MODEL,), + "positive": (data_types.CONDITIONING,), + "negative": (data_types.CONDITIONING,), + "weight": ( + "FLOAT", + { + "default": 0.8, + "min": 0.0, + "max": 5.0, + "step": 0.01, + }, + ), + "start_at": ( + "FLOAT", + { + "default": 0.0, + "min": 0.0, + "max": 1.0, + "step": 0.001, + }, + ), + "end_at": ( + "FLOAT", + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.001, + }, + ), + }, + "optional": { + "image_kps": ("IMAGE",), + "mask": ("MASK",), + }, + } + + RETURN_TYPES = ( + data_types.MODEL, + data_types.CONDITIONING, + data_types.CONDITIONING, + ) + RETURN_NAMES = ( + "MODEL", + "positive", + "negative", + ) + # FUNCTION = "apply_instantid" use default_function + CATEGORY = "InstantID" + NODE_DISPLAY_NAME = "Apply InstantID" + + +class ApplyInstantIDAdvanced(ApplyInstantID): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "instantid": (data_types.INSTANTID,), + "insightface": (data_types.FACEANALYSIS,), + "control_net": (data_types.CONTROL_NET,), + "image": ("IMAGE",), + "model": (data_types.MODEL,), + "positive": (data_types.CONDITIONING,), + "negative": (data_types.CONDITIONING,), + "ip_weight": ( + "FLOAT", + { + "default": 0.8, + "min": 0.0, + "max": 3.0, + "step": 0.01, + }, + ), + "cn_strength": ( + "FLOAT", + { + "default": 0.8, + "min": 0.0, + "max": 10.0, + "step": 0.01, + }, + ), + "start_at": ( + "FLOAT", + { + "default": 0.0, + "min": 0.0, + "max": 1.0, + "step": 0.001, + }, + ), + "end_at": ( + "FLOAT", + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.001, + }, + ), + "noise": ( + "FLOAT", + { + "default": 0.0, + "min": 0.0, + "max": 1.0, + "step": 0.1, + }, + ), + "combine_embeds": ( + ["average", "norm average", "concat"], + {"default": "average"}, + ), + }, + "optional": { + "image_kps": ("IMAGE",), + "mask": ("MASK",), + }, + } + + +class InstantIDFaceAnalysis(BizyAirBaseNode): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "provider": (["CUDA"],), + }, + } + + RETURN_TYPES = (data_types.FACEANALYSIS,) + # FUNCTION = "load_insight_face" + CATEGORY = "InstantID" + NODE_DISPLAY_NAME = "InstantID Face Analysis" diff --git a/src/bizyair/commands/processors/prompt_processor.py b/src/bizyair/commands/processors/prompt_processor.py index 25fab861..dc401cf4 100644 --- a/src/bizyair/commands/processors/prompt_processor.py +++ b/src/bizyair/commands/processors/prompt_processor.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List from bizyair.common import client -from bizyair.common.env_var import BIZYAIR_DEBUG +from bizyair.common.env_var import BIZYAIR_DEBUG, BIZYAIR_DEV_REQUEST_URL from bizyair.path_utils import ( convert_prompt_label_path_to_real_path, guess_url_from_node, @@ -34,6 +34,9 @@ class NodeUsageState: class SearchServiceRouter(Processor): def process(self, prompt: Dict[str, Dict[str, Any]], last_node_ids: List[str]): + if BIZYAIR_DEV_REQUEST_URL: + return BIZYAIR_DEV_REQUEST_URL + # TODO Improve distribution logic queue = deque(last_node_ids) visited = {key: True for key in last_node_ids} diff --git a/src/bizyair/configs/models.json b/src/bizyair/configs/models.json index a83faa85..772b2df5 100644 --- a/src/bizyair/configs/models.json +++ b/src/bizyair/configs/models.json @@ -68,7 +68,9 @@ "clip":[ "clip_l.safetensors", "t5xxl_fp16.safetensors", - "t5xxl_fp8_e4m3fn.safetensors" + "t5xxl_fp8_e4m3fn.safetensors", + "ViT-L-14-TEXT-detail-improved-hiT-GmP-TE-only-HF.safetensors" + ], "upscale_models":[ "4x_NMKD-Siax_200k.pth" diff --git a/src/bizyair/data_types.py b/src/bizyair/data_types.py index 0ad68615..2a886095 100644 --- a/src/bizyair/data_types.py +++ b/src/bizyair/data_types.py @@ -6,6 +6,8 @@ CONDITIONING = "BIZYAIR_CONDITIONING" CONTROL_NET = "BIZYAIR_CONTROL_NET" UPSCALE_MODEL = "BIZYAIR_UPSCALE_MODEL" +INSTANTID = "BIZYAIR_INSTANTID" +FACEANALYSIS = "BIZYAIR_FACEANALYSIS" def is_model_datatype(datatype): diff --git a/src/bizyair/nodes_base.py b/src/bizyair/nodes_base.py index 80876f2c..5f36a1ac 100644 --- a/src/bizyair/nodes_base.py +++ b/src/bizyair/nodes_base.py @@ -120,16 +120,11 @@ def assigned_id(self): def default_function(self, **kwargs): class_type = type(self).__name__ - return tuple( - BizyAirNodeIO( - self.assigned_id, - { - self.assigned_id: create_node_data( - class_type=class_type, - inputs=kwargs, - outputs={"slot_index": slot_index}, - ) - }, + outs = [] + for slot_index in range(len(self.RETURN_TYPES)): + node = BizyAirNodeIO(node_id=self.assigned_id, nodes={}) + node.add_node_data( + class_type=class_type, inputs=kwargs, outputs={"slot_index": slot_index} ) - for slot_index in range(len(self.RETURN_TYPES)) - ) + outs.append(node) + return tuple(outs) From 0097327dafed92106a2b8d1eb5e9597bac9ce150 Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 00:07:18 +0800 Subject: [PATCH 02/11] add Image_Encode --- bizyair_extras/__init__.py | 2 + bizyair_extras/nodes_controlnet.py | 19 ++++ bizyair_extras/nodes_image_utils.py | 25 +++++ bizyair_extras/nodes_testing_utils.py | 135 +++++++++++++++++++++----- src/bizyair/image_utils.py | 15 +-- 5 files changed, 167 insertions(+), 29 deletions(-) create mode 100644 bizyair_extras/nodes_controlnet.py create mode 100644 bizyair_extras/nodes_image_utils.py diff --git a/bizyair_extras/__init__.py b/bizyair_extras/__init__.py index 5d72a370..8ac4ed4f 100644 --- a/bizyair_extras/__init__.py +++ b/bizyair_extras/__init__.py @@ -1,4 +1,6 @@ from .nodes_comfyui_instantid import * +from .nodes_controlnet import * +from .nodes_image_utils import * from .nodes_ipadapter_plus.nodes_ipadapter_plus import * from .nodes_kolors_mz import * from .nodes_testing_utils import * diff --git a/bizyair_extras/nodes_controlnet.py b/bizyair_extras/nodes_controlnet.py new file mode 100644 index 00000000..06a141ee --- /dev/null +++ b/bizyair_extras/nodes_controlnet.py @@ -0,0 +1,19 @@ +from comfy.cldm.control_types import UNION_CONTROLNET_TYPES + +from bizyair import BizyAirBaseNode, data_types + + +class SetUnionControlNetType(BizyAirBaseNode): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "control_net": (data_types.CONTROL_NET,), + "type": (["auto"] + list(UNION_CONTROLNET_TYPES.keys()),), + } + } + + CATEGORY = "conditioning/controlnet" + RETURN_TYPES = (data_types.CONTROL_NET,) + + # FUNCTION = "set_controlnet_type" diff --git a/bizyair_extras/nodes_image_utils.py b/bizyair_extras/nodes_image_utils.py new file mode 100644 index 00000000..5d1c514c --- /dev/null +++ b/bizyair_extras/nodes_image_utils.py @@ -0,0 +1,25 @@ +from bizyair import BizyAirBaseNode +from bizyair.image_utils import encode_data + + +class Image_Encode(BizyAirBaseNode): + @classmethod + def INPUT_TYPES(s): + return { + "required": {"image": ("IMAGE",)}, + "optional": { + "lossless": ( + "BOOLEAN", + {"default": False, "label_on": "yes", "label_off": "no"}, + ), + }, + } + + CATEGORY = "image_utils" + FUNCTION = "apply" + RETURN_TYPES = ("IMAGE",) + NODE_DISPLAY_NAME = "Image Encode" + + def apply(self, image, lossless=False): + out = encode_data(image, lossless=lossless) + return (out,) diff --git a/bizyair_extras/nodes_testing_utils.py b/bizyair_extras/nodes_testing_utils.py index 80818122..38115938 100644 --- a/bizyair_extras/nodes_testing_utils.py +++ b/bizyair_extras/nodes_testing_utils.py @@ -3,47 +3,138 @@ """ +import os +import random + +import folder_paths import numpy as np -import torch -import torch.nn.functional as F from PIL import Image from bizyair import NODE_CLASS_MAPPINGS +from bizyair.image_utils import decode_data, encode_data class ImagesTest: # https://docs.comfy.org/essentials/custom_node_images_and_masks#images + def __init__(self): + self.output_dir = folder_paths.get_temp_directory() + self.type = "temp" + self.prefix_append = "_temp_" + "".join( + random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5) + ) + @classmethod def INPUT_TYPES(s): return { - "required": {}, - "optional": {"images_a": ("*", {}), "images_b": ("*", {})}, + "required": { + "images1": ("IMAGE",), + "images2": ("IMAGE",), + "ssim_threshold": ("STRING", {"default": "0.9"}), + "raise_if_diff": (["enable", "disable"],), + "image_id": ("STRING", {"default": "ComfyUI"}), + }, + "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, } - OUTPUT_NODE = True RETURN_TYPES = () + FUNCTION = "save_images" + + OUTPUT_NODE = True + CATEGORY = "images_test" - FUNCTION = "test" - def test(self, images_a: torch.Tensor, images_b: torch.Tensor): - print(f"{torch.allclose(images_a, images_b, rtol=0.1, atol=0.1)=}") # True + def save_images( + self, + images1, + images2, + ssim_threshold: float, + raise_if_diff: str, + image_id="ComfyUI", + prompt=None, + extra_pnginfo=None, + ): from skimage.metrics import structural_similarity as ssim - for image_a, image_b in zip(images_a, images_b): - if image_a.shape != image_b.shape: - raise ValueError( - f"Images must have the same dimensions, {image_a.shape=} {image_b.shape=}" - ) - mse = F.mse_loss(image_a, image_b) - print(f"MSE value: {mse}") # MSE value: 3.59503501385916e-05 - i = 255.0 * image_a.cpu().numpy() - img1 = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) - i = 255.0 * image_b.cpu().numpy() - img2 = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) - ssim_value = ssim(np.array(img1), np.array(img2), channel_axis=2) - print(f"SSIM value: {ssim_value}") # SSIM value: 0.9909630031265833 + assert len(images1) == len(images2) + filename_prefix = image_id[:] + filename_prefix += self.prefix_append + ( + full_output_folder, + filename, + counter, + subfolder, + filename_prefix, + ) = folder_paths.get_save_image_path( + filename_prefix, + self.output_dir, + images1[0].shape[1], + images1[0].shape[0], + ) + results = list() + for image1, image2 in zip(images1, images2): + + # image diff + image = image1.cpu() - image2.cpu() + + i = 255.0 * image.cpu().numpy() + img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) + metadata = None + file = f"{filename}_{counter:05}_.png" + img.save( + os.path.join(full_output_folder, file), + pnginfo=metadata, + compress_level=4, + ) + results.append( + {"filename": file, "subfolder": subfolder, "type": self.type} + ) + counter += 1 + + max_diff = image.abs().max().item() + + img1 = self.image_to_numpy(image1) + img2 = self.image_to_numpy(image2) + ssim = ssim(img1, img2, channel_axis=2) + + print( + "\033[91m" + f"[ShowImageDiff {image_id}] Max value of diff is {max_diff}, image simularity is {ssim:.6f}" + + "\033[0m" + ) + + if raise_if_diff == "enable": + assert ssim > float( + ssim_threshold + ), f"Image diff is too large, ssim is {ssim}, " + + return {"ui": {"images": results}} + + def image_to_numpy(self, image): + i = 255.0 * image.cpu().numpy() + img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) + return np.array(img) + + +class ImageEncodeDecodeTest: + @classmethod + def INPUT_TYPES(s): + return { + "required": {"image": ("IMAGE",)}, + "optional": { + "lossless": ( + "BOOLEAN", + {"default": False, "label_on": "yes", "label_off": "no"}, + ), + }, + } + + CATEGORY = "images_test" + FUNCTION = "test" + RETURN_TYPES = ("IMAGE",) - return (None,) + def test(self, image, lossless=False): + return (decode_data(encode_data(image, lossless=lossless)),) NODE_CLASS_MAPPINGS["Tools_ImagesTest"] = ImagesTest +NODE_CLASS_MAPPINGS["Tools_ImageEncodeDecodeTest"] = ImageEncodeDecodeTest diff --git a/src/bizyair/image_utils.py b/src/bizyair/image_utils.py index e498a087..e27ac8f3 100644 --- a/src/bizyair/image_utils.py +++ b/src/bizyair/image_utils.py @@ -32,11 +32,11 @@ def convert_image_to_rgb(image: Image.Image) -> Image.Image: def encode_image_to_base64( - image: Image.Image, format: str = "png", quality: int = 100 + image: Image.Image, format: str = "png", quality: int = 100, lossless=False ) -> str: image = convert_image_to_rgb(image) with io.BytesIO() as output: - image.save(output, format=format, quality=quality) + image.save(output, format=format, quality=quality, lossless=lossless) output.seek(0) img_bytes = output.getvalue() if BIZYAIR_DEBUG: @@ -105,7 +105,7 @@ def _legacy_decode_comfy_image( return output -def _new_encode_comfy_image(images: torch.Tensor, image_format="WEBP") -> str: +def _new_encode_comfy_image(images: torch.Tensor, image_format="WEBP", **kwargs) -> str: """https://docs.comfy.org/essentials/custom_node_snippets#save-an-image-batch Encode a batch of images to base64 strings. @@ -120,7 +120,7 @@ def _new_encode_comfy_image(images: torch.Tensor, image_format="WEBP") -> str: for batch_number, image in enumerate(images): i = 255.0 * image.cpu().numpy() img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) - base64ed_image = encode_image_to_base64(img, format=image_format) + base64ed_image = encode_image_to_base64(img, format=image_format, **kwargs) results[batch_number] = base64ed_image return json.dumps(results) @@ -149,11 +149,11 @@ def _new_decode_comfy_image(img_datas: str, image_format="WEBP") -> torch.tensor def encode_comfy_image( - image: torch.Tensor, image_format="WEBP", old_version=False + image: torch.Tensor, image_format="WEBP", old_version=False, lossless=False ) -> str: if old_version: return _legacy_encode_comfy_image(image, image_format) - return _new_encode_comfy_image(image, image_format) + return _new_encode_comfy_image(image, image_format, lossless=lossless) def decode_comfy_image( @@ -268,8 +268,9 @@ def is_image_tensor(tensor) -> bool: def _(output, **kwargs): if is_image_tensor(output) and not kwargs.get("disable_image_marker", False): old_version = kwargs.get("old_version", False) + lossless = kwargs.get("lossless", False) return IMAGE_MARKER + encode_comfy_image( - output, image_format="WEBP", old_version=old_version + output, image_format="WEBP", old_version=old_version, lossless=lossless ) return TENSOR_MARKER + tensor_to_base64(output) From 388a74ef31fc14bb2d39ab761426570d16f989e1 Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 00:53:03 +0800 Subject: [PATCH 03/11] Add LoadImageURL --- bizyair_extras/nodes_image_utils.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/bizyair_extras/nodes_image_utils.py b/bizyair_extras/nodes_image_utils.py index 5d1c514c..e891f993 100644 --- a/bizyair_extras/nodes_image_utils.py +++ b/bizyair_extras/nodes_image_utils.py @@ -1,5 +1,49 @@ +import os +import urllib.request + +import folder_paths + from bizyair import BizyAirBaseNode from bizyair.image_utils import encode_data +from nodes import LoadImage + + +class LoadImageURL(BizyAirBaseNode): + # https://mei-xia.imgbb.com + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "url": ( + "STRING", + { + "multiline": True, # True if you want the field to look like the one on the ClipTextEncode node + "default": "url or path", + "lazy": True, + }, + ), + }, + } + + CATEGORY = "image_utils" + RETURN_TYPES = ("IMAGE", "MASK") + FUNCTION = "apply" + NODE_DISPLAY_NAME = "Load Image (URL)" + + def apply(self, url: str): + url = url.strip() + input_dir = folder_paths.get_input_directory() + filename = os.path.basename(url) + file_path = os.path.join(input_dir, filename) + + # Check if the file already exists + if os.path.exists(file_path): + print(f"File {filename} already exists, skipping download.") + else: + # Download the image + urllib.request.urlretrieve(url, file_path) + print(f"Image successfully downloaded and saved as {filename}.") + return LoadImage().load_image(filename) class Image_Encode(BizyAirBaseNode): From 8cded21498ebae635ef03d04428c3116ee388440 Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 00:56:47 +0800 Subject: [PATCH 04/11] refine --- bizyair_extras/nodes_image_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bizyair_extras/nodes_image_utils.py b/bizyair_extras/nodes_image_utils.py index e891f993..a3084829 100644 --- a/bizyair_extras/nodes_image_utils.py +++ b/bizyair_extras/nodes_image_utils.py @@ -9,7 +9,6 @@ class LoadImageURL(BizyAirBaseNode): - # https://mei-xia.imgbb.com @classmethod def INPUT_TYPES(s): return { From ced21f6e72cf8605d97118b9eaeb0526d97f8066 Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 13:46:35 +0800 Subject: [PATCH 05/11] refine --- src/bizyair/configs/models.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bizyair/configs/models.json b/src/bizyair/configs/models.json index 772b2df5..bc0bbfd2 100644 --- a/src/bizyair/configs/models.json +++ b/src/bizyair/configs/models.json @@ -34,6 +34,9 @@ ], "sd15":[ "control_v11f1e_sd15_tile.pth" + ], + "instantid": [ + "diffusion_pytorch_model.safetensors" ] }, "ipadapter": { @@ -74,5 +77,6 @@ ], "upscale_models":[ "4x_NMKD-Siax_200k.pth" - ] + ], + "instantid":["ip-adapter.bin"] } From 96bb1f9e01e0458cfb02f8e374c9e0020563a584 Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 18:24:01 +0800 Subject: [PATCH 06/11] Add Instantid Demo --- bizyair_example_menu.json | 3 +- examples/bizyair_sdxl_InstantID_basic.json | 814 +++++++++++++++++++++ src/bizyair/configs/models.json | 107 +-- 3 files changed, 840 insertions(+), 84 deletions(-) create mode 100644 examples/bizyair_sdxl_InstantID_basic.json diff --git a/bizyair_example_menu.json b/bizyair_example_menu.json index 97f637a2..09c5cd86 100644 --- a/bizyair_example_menu.json +++ b/bizyair_example_menu.json @@ -18,7 +18,8 @@ "Image to Image by BizyAir KSampler": "bizyair_showcase_ksampler_img2img.json", "LoRA workflow by BizyAir KSampler": "bizyair_showcase_ksampler_lora.json", "ControlNet workflow by BizyAir KSampler": "bizyair_showcase_ksampler_controlnet.json", - "IP Adapter workflow by BizyAir KSampler": "bizyair_showcase_ksampler_ipadapter.json" + "IP Adapter workflow by BizyAir KSampler": "bizyair_showcase_ksampler_ipadapter.json", + "InstantID Basic workflow by BizyAir KSampler": "bizyair_sdxl_InstantID_basic.json" }, "Kolors": { "Kolors Text to Image": "bizyair_kolors_txt2img.json", diff --git a/examples/bizyair_sdxl_InstantID_basic.json b/examples/bizyair_sdxl_InstantID_basic.json new file mode 100644 index 00000000..37bc9681 --- /dev/null +++ b/examples/bizyair_sdxl_InstantID_basic.json @@ -0,0 +1,814 @@ +{ + "last_node_id": 125, + "last_link_id": 312, + "nodes": [ + { + "id": 95, + "type": "BizyAir_KSampler", + "pos": { + "0": 2073, + "1": -1134, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "BIZYAIR_MODEL", + "link": 259 + }, + { + "name": "positive", + "type": "BIZYAIR_CONDITIONING", + "link": 311 + }, + { + "name": "negative", + "type": "BIZYAIR_CONDITIONING", + "link": 312 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 262 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 272 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_KSampler" + }, + "widgets_values": [ + 1, + "fixed", + 30, + 4.5, + "ddpm", + "karras", + 1 + ] + }, + { + "id": 92, + "type": "BizyAir_ControlNetLoader", + "pos": { + "0": 1223, + "1": -1098, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 415.8472595214844, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CONTROL_NET", + "type": "BIZYAIR_CONTROL_NET", + "links": [ + 252 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_ControlNetLoader" + }, + "widgets_values": [ + "instantid/diffusion_pytorch_model.safetensors" + ] + }, + { + "id": 122, + "type": "BizyAir_LoadImageURL", + "pos": { + "0": 1232, + "1": -1246, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 398.8588562011719, + "1": 96 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 308, + 309 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_LoadImageURL" + }, + "widgets_values": [ + "https://bizy-air.oss-cn-beijing.aliyuncs.com/examples_asset/bizyair-instantid-example.webp" + ] + }, + { + "id": 91, + "type": "BizyAir_ApplyInstantID", + "pos": { + "0": 1716, + "1": -1252, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 326.1368408203125, + "1": 266 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "instantid", + "type": "BIZYAIR_INSTANTID", + "link": 250 + }, + { + "name": "insightface", + "type": "BIZYAIR_FACEANALYSIS", + "link": 251 + }, + { + "name": "control_net", + "type": "BIZYAIR_CONTROL_NET", + "link": 252 + }, + { + "name": "image", + "type": "IMAGE", + "link": 308 + }, + { + "name": "model", + "type": "BIZYAIR_MODEL", + "link": 253 + }, + { + "name": "positive", + "type": "BIZYAIR_CONDITIONING", + "link": 257 + }, + { + "name": "negative", + "type": "BIZYAIR_CONDITIONING", + "link": 258 + }, + { + "name": "image_kps", + "type": "IMAGE", + "link": null + }, + { + "name": "mask", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "BIZYAIR_MODEL", + "links": [ + 259 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "positive", + "type": "BIZYAIR_CONDITIONING", + "links": [ + 311 + ], + "slot_index": 1, + "shape": 3 + }, + { + "name": "negative", + "type": "BIZYAIR_CONDITIONING", + "links": [ + 312 + ], + "slot_index": 2, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_ApplyInstantID" + }, + "widgets_values": [ + 0.8, + 0, + 1 + ] + }, + { + "id": 83, + "type": "BizyAir_InstantIDModelLoader", + "pos": { + "0": 1722, + "1": -928, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 318.3161926269531, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "BIZYAIR_INSTANTID", + "type": "BIZYAIR_INSTANTID", + "links": [ + 250 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_InstantIDModelLoader" + }, + "widgets_values": [ + "ip-adapter.bin" + ] + }, + { + "id": 96, + "type": "BizyAir_VAEDecode", + "pos": { + "0": 2084, + "1": -1243, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 296.0434265136719, + "1": 46 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 272 + }, + { + "name": "vae", + "type": "BIZYAIR_VAE", + "link": 266 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 267 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_VAEDecode" + } + }, + { + "id": 123, + "type": "PreviewImage", + "pos": { + "0": 2459, + "1": -1252, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 559.7613525390625, + "1": 560.9844360351562 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 309 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 85, + "type": "BizyAir_InstantIDFaceAnalysis", + "pos": { + "0": 1725, + "1": -768, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 323.09515380859375, + "1": 64.16976928710938 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "BIZYAIR_FACEANALYSIS", + "type": "BIZYAIR_FACEANALYSIS", + "links": [ + 251 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_InstantIDFaceAnalysis" + }, + "widgets_values": [ + "CUDA" + ] + }, + { + "id": 94, + "type": "BizyAir_CLIPTextEncode", + "pos": { + "0": 1238, + "1": -785, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 417.4436340332031, + "1": 76 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "BIZYAIR_CLIP", + "link": 256 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "BIZYAIR_CONDITIONING", + "links": [ + 258 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_CLIPTextEncode" + }, + "widgets_values": [ + "photograph, deformed, glitch, noisy, realistic, stock photo" + ] + }, + { + "id": 93, + "type": "BizyAir_CLIPTextEncode", + "pos": { + "0": 1232, + "1": -951, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 415.1671447753906, + "1": 85.4674301147461 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "BIZYAIR_CLIP", + "link": 255 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "BIZYAIR_CONDITIONING", + "links": [ + 257 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_CLIPTextEncode" + }, + "widgets_values": [ + "comic character. graphic illustration, comic art, graphic novel art, vibrant, highly detailed" + ] + }, + { + "id": 97, + "type": "PreviewImage", + "pos": { + "0": 3040, + "1": -1255, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 534.6394653320312, + "1": 565.498291015625 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 267 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": { + "0": 2075, + "1": -803, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 262 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1016, + 1016, + 1 + ] + }, + { + "id": 90, + "type": "BizyAir_CheckpointLoaderSimple", + "pos": { + "0": 809, + "1": -1258, + "2": 0, + "3": 0, + "4": 0, + "5": 0, + "6": 0, + "7": 0, + "8": 0, + "9": 0 + }, + "size": { + "0": 343.9351806640625, + "1": 98 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "model", + "type": "BIZYAIR_MODEL", + "links": [ + 253 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "clip", + "type": "BIZYAIR_CLIP", + "links": [ + 255, + 256 + ], + "slot_index": 1, + "shape": 3 + }, + { + "name": "vae", + "type": "BIZYAIR_VAE", + "links": [ + 266 + ], + "slot_index": 2, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BizyAir_CheckpointLoaderSimple" + }, + "widgets_values": [ + "sdxl/samaritan3dCartoon_v40SDXL.safetensors" + ] + } + ], + "links": [ + [ + 250, + 83, + 0, + 91, + 0, + "BIZYAIR_INSTANTID" + ], + [ + 251, + 85, + 0, + 91, + 1, + "BIZYAIR_FACEANALYSIS" + ], + [ + 252, + 92, + 0, + 91, + 2, + "BIZYAIR_CONTROL_NET" + ], + [ + 253, + 90, + 0, + 91, + 4, + "BIZYAIR_MODEL" + ], + [ + 255, + 90, + 1, + 93, + 0, + "BIZYAIR_CLIP" + ], + [ + 256, + 90, + 1, + 94, + 0, + "BIZYAIR_CLIP" + ], + [ + 257, + 93, + 0, + 91, + 5, + "BIZYAIR_CONDITIONING" + ], + [ + 258, + 94, + 0, + 91, + 6, + "BIZYAIR_CONDITIONING" + ], + [ + 259, + 91, + 0, + 95, + 0, + "BIZYAIR_MODEL" + ], + [ + 262, + 5, + 0, + 95, + 3, + "LATENT" + ], + [ + 266, + 90, + 2, + 96, + 1, + "BIZYAIR_VAE" + ], + [ + 267, + 96, + 0, + 97, + 0, + "IMAGE" + ], + [ + 272, + 95, + 0, + 96, + 0, + "LATENT" + ], + [ + 308, + 122, + 0, + 91, + 3, + "IMAGE" + ], + [ + 309, + 122, + 0, + 123, + 0, + "IMAGE" + ], + [ + 311, + 91, + 1, + 95, + 1, + "BIZYAIR_CONDITIONING" + ], + [ + 312, + 91, + 2, + 95, + 2, + "BIZYAIR_CONDITIONING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.7449402268886804, + "offset": [ + -698.7654009040159, + 1431.2066920107438 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/src/bizyair/configs/models.json b/src/bizyair/configs/models.json index ba2d0529..5c2cdb83 100644 --- a/src/bizyair/configs/models.json +++ b/src/bizyair/configs/models.json @@ -1,80 +1,19 @@ { - "checkpoints": { - "sdxl": [ - "counterfeitxl_v25.safetensors", - "dreamshaperXL_lightningDPMSDE.safetensors", - "dreamshaperXL_v21TurboDPMSDE.safetensors", - "HelloWorldXL_v70.safetensors", - "juggernautXL_v9Rdphoto2Lightning.safetensors", - "Juggernaut-XL_v9_RunDiffusionPhoto_v2.safetensors", - "Juggernaut_X_RunDiffusion_Hyper.safetensors", - "mannEDreams_v004.safetensors", - "realisticStockPhoto_v20.safetensors", - "samaritan3dCartoon_v40SDXL.safetensors" - ], - "sd15": [ - "dreamshaper_8.safetensors" - ] - }, - "clip_vision": { - "models": [ - "CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors" - ], - "kolors": [ - "pytorch_model.bin" - ] - }, - "controlnet": { - "kolors": [ - "Kolors-ControlNet-Canny.safetensors", - "Kolors-ControlNet-Depth.safetensors" - ], - "sdxl": [ - "diffusion_pytorch_model_promax.safetensors" - ], - "sd15": [ - "control_v11f1e_sd15_tile.pth" - ] - }, - "ipadapter": { - "kolors": [ - "ip_adapter_plus_general.bin" - ] - }, - "loras": { - "sdxl": [ - "Cute_Animals.safetensors", - "watercolor_v1_sdxl_lora.safetensors" - ], - "flux": [ - "meijia_flux_lora_rank16_bf16.safetensors" - ] - }, - "unet": { - "kolors": [ - "Kolors-Inpainting.safetensors", - "Kolors.safetensors" - ], - "flux": [ - "flux1-schnell.sft", - "flux1-dev.sft" - ] - }, - "vae": { - "sdxl": [ - "sdxl_vae.safetensors" - ], - "flux": [ - "ae.sft" - ] - }, - "clip": [ - "clip_l.safetensors", - "t5xxl_fp16.safetensors", - "t5xxl_fp8_e4m3fn.safetensors" + "checkpoints": { + "sdxl": [ + "counterfeitxl_v25.safetensors", + "dreamshaperXL_lightningDPMSDE.safetensors", + "dreamshaperXL_v21TurboDPMSDE.safetensors", + "HelloWorldXL_v70.safetensors", + "juggernautXL_v9Rdphoto2Lightning.safetensors", + "Juggernaut-XL_v9_RunDiffusionPhoto_v2.safetensors", + "Juggernaut_X_RunDiffusion_Hyper.safetensors", + "mannEDreams_v004.safetensors", + "realisticStockPhoto_v20.safetensors", + "samaritan3dCartoon_v40SDXL.safetensors" ], - "upscale_models": [ - "4x_NMKD-Siax_200k.pth" + "sd15": [ + "dreamshaper_8.safetensors" ] }, "clip_vision": { @@ -93,7 +32,7 @@ "sdxl": [ "diffusion_pytorch_model_promax.safetensors" ], - "sd15":[ + "sd15": [ "control_v11f1e_sd15_tile.pth" ], "instantid": [ @@ -109,6 +48,9 @@ "sdxl": [ "Cute_Animals.safetensors", "watercolor_v1_sdxl_lora.safetensors" + ], + "flux": [ + "meijia_flux_lora_rank16_bf16.safetensors" ] }, "unet": { @@ -129,16 +71,15 @@ "ae.sft" ] }, - "clip":[ + "clip": [ "clip_l.safetensors", "t5xxl_fp16.safetensors", - "t5xxl_fp8_e4m3fn.safetensors", - "ViT-L-14-TEXT-detail-improved-hiT-GmP-TE-only-HF.safetensors" - + "t5xxl_fp8_e4m3fn.safetensors" ], - "upscale_models":[ + "upscale_models": [ "4x_NMKD-Siax_200k.pth" ], - "instantid":["ip-adapter.bin"] - + "instantid": [ + "ip-adapter.bin" + ] } From a2c38d5afad8bdfc7dc0d390018cf3c52790321d Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 18:25:16 +0800 Subject: [PATCH 07/11] refine --- src/bizyair/configs/models.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bizyair/configs/models.yaml b/src/bizyair/configs/models.yaml index 93a98a4b..3d5f1138 100644 --- a/src/bizyair/configs/models.yaml +++ b/src/bizyair/configs/models.yaml @@ -12,7 +12,7 @@ model_types: routing_configs: sdxl: service_address: https://bizyair-api.siliconflow.cn/x/v1 - route: /supernode/bizyair-sdxl-comfy-ksampler + route: /supernode/bizyair-sdxl-comfy-ksampler-v2 kolors: service_address: https://bizyair-api.siliconflow.cn/x/v1 route: /supernode/kolors-bizyair-sdxl-comfy-ksampler From b198de9c05eb0cf009dc28e2991df8616070fcca Mon Sep 17 00:00:00 2001 From: FengWen Date: Fri, 6 Sep 2024 18:26:27 +0800 Subject: [PATCH 08/11] refine --- examples/bizyair_sdxl_InstantID_basic.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bizyair_sdxl_InstantID_basic.json b/examples/bizyair_sdxl_InstantID_basic.json index 37bc9681..90f518a8 100644 --- a/examples/bizyair_sdxl_InstantID_basic.json +++ b/examples/bizyair_sdxl_InstantID_basic.json @@ -811,4 +811,4 @@ } }, "version": 0.4 -} \ No newline at end of file +} From 687ba3f0e88673335343fb73cee9f1b74f1205d2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 6 Sep 2024 19:33:33 +0800 Subject: [PATCH 09/11] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 51cbf0ed..e27f15b4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # BizyAir +- [2024/09/06] 🌩️ BizyAir supports InstantID for SDXL now. [SDXL InstantID workflow](https://github.com/siliconflow/BizyAir/blob/master/examples/bizyair_sdxl_InstantID_basic.json) - [2024/09/05] 🌩️ BizyAir supports users in running custom LoRA models, including SDXL and Flux LoRA. [How to upload and run custom model](https://siliconflow.github.io/BizyAir/model-host/introduce.html) - [2024/08/23] 🌩️ BizyAir now support ultimateSDupscale nodes [upscale workflow](./examples/bizyair_ultimate_sd_upscale.json) - [2024/08/14] 🌩️ BizyAir JoyCaption node has been released. [Try the example to recreate a image by JoyCaption and Flux](./examples/bizyair_flux_joycaption_img2img_workflow.json), thanks to [fancyfeast/joy-caption-pre-alpha](https://huggingface.co/spaces/fancyfeast/joy-caption-pre-alpha) From 77e15bc109b06052e71a01d1424fb63ff8d4d13b Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 6 Sep 2024 19:34:26 +0800 Subject: [PATCH 10/11] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e27f15b4..ba57e723 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # BizyAir -- [2024/09/06] 🌩️ BizyAir supports InstantID for SDXL now. [SDXL InstantID workflow](https://github.com/siliconflow/BizyAir/blob/master/examples/bizyair_sdxl_InstantID_basic.json) +- [2024/09/06] 🌩️ BizyAir supports InstantID for SDXL now. [SDXL InstantID workflow](./examples/bizyair_sdxl_InstantID_basic.json) - [2024/09/05] 🌩️ BizyAir supports users in running custom LoRA models, including SDXL and Flux LoRA. [How to upload and run custom model](https://siliconflow.github.io/BizyAir/model-host/introduce.html) - [2024/08/23] 🌩️ BizyAir now support ultimateSDupscale nodes [upscale workflow](./examples/bizyair_ultimate_sd_upscale.json) - [2024/08/14] 🌩️ BizyAir JoyCaption node has been released. [Try the example to recreate a image by JoyCaption and Flux](./examples/bizyair_flux_joycaption_img2img_workflow.json), thanks to [fancyfeast/joy-caption-pre-alpha](https://huggingface.co/spaces/fancyfeast/joy-caption-pre-alpha) From 3fa3d61dca51c2ef1455ad7970c855b550202509 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 6 Sep 2024 19:38:36 +0800 Subject: [PATCH 11/11] rename node --- bizyair_extras/nodes_comfyui_instantid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bizyair_extras/nodes_comfyui_instantid.py b/bizyair_extras/nodes_comfyui_instantid.py index 30b5f8fa..1d2ab52a 100644 --- a/bizyair_extras/nodes_comfyui_instantid.py +++ b/bizyair_extras/nodes_comfyui_instantid.py @@ -79,6 +79,8 @@ def INPUT_TYPES(s): class ApplyInstantIDAdvanced(ApplyInstantID): + NODE_DISPLAY_NAME = "Apply InstantID Adavanced" + @classmethod def INPUT_TYPES(s): return {