From 92883fa10f634413a96482d822817b3d137846d2 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj Date: Wed, 23 Aug 2023 23:15:35 +0300 Subject: [PATCH 01/97] add support to custom comfyui workflow --- index.html | 5 +- index.js | 1 + typescripts/comfyui/comfyui.tsx | 726 ++++++++++++++++++++++++++++++++ typescripts/comfyui/prompt.json | 149 +++++++ typescripts/entry.ts | 1 + 5 files changed, 881 insertions(+), 1 deletion(-) create mode 100644 typescripts/comfyui/comfyui.tsx create mode 100644 typescripts/comfyui/prompt.json diff --git a/index.html b/index.html index 240e8dd4..de8cb365 100644 --- a/index.html +++ b/index.html @@ -898,7 +898,7 @@
- +
Explore Lexica for prompts and inspiration
+
+ +
diff --git a/index.js b/index.js index 1b9e492e..a5297dac 100644 --- a/index.js +++ b/index.js @@ -88,6 +88,7 @@ const { extra_page, selection_ts, stores, + comfyui, } = require('./typescripts/dist/bundle') const io = require('./utility/io') diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx new file mode 100644 index 00000000..449ae032 --- /dev/null +++ b/typescripts/comfyui/comfyui.tsx @@ -0,0 +1,726 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import { requestGet, requestPost } from '../util/ts/api' +import { observer } from 'mobx-react' +import { + MoveToCanvasSvg, + SpMenu, + SpSlider, + SpTextfield, +} from '../util/elements' +import { ErrorBoundary } from '../util/errorBoundary' +import { Collapsible } from '../util/collapsible' +import Locale from '../locale/locale' +import { AStore } from '../main/astore' + +import hi_res_prompt from './prompt.json' +import { Grid } from '../util/grid' +import { io } from '../util/oldSystem' +import { app } from 'photoshop' +import { reaction } from 'mobx' +export let hi_res_prompt_temp = hi_res_prompt +console.log('hi_res_prompt: ', hi_res_prompt) + +interface Error { + type: string + message: string + details: string + extra_info: any +} + +interface NodeError { + errors: Error[] + dependent_outputs: string[] + class_type: string +} + +interface Result { + error?: Error + node_errors?: { [key: string]: NodeError } +} + +const result: Result = { + error: { + type: 'prompt_outputs_failed_validation', + message: 'Prompt outputs failed validation', + details: '', + extra_info: {}, + }, + node_errors: { + '16': { + errors: [ + { + type: 'value_not_in_list', + message: 'Value not in list', + details: + "ckpt_name: 'v2-1_768-ema-pruned.ckpt' not in ['anythingV5Anything_anythingV5PrtRE.safetensors', 'deliberate_v2.safetensors', 'dreamshaper_631BakedVae.safetensors', 'dreamshaper_631Inpainting.safetensors', 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', 'juggernaut_final-inpainting.safetensors', 'juggernaut_final.safetensors', 'loraChGirl.safetensors', 'sd-v1-5-inpainting.ckpt', 'sd_xl_base_1.0.safetensors', 'sd_xl_refiner_1.0.safetensors', 'v1-5-pruned-emaonly.ckpt']", + extra_info: { + input_name: 'ckpt_name', + input_config: [ + [ + 'anythingV5Anything_anythingV5PrtRE.safetensors', + 'deliberate_v2.safetensors', + 'dreamshaper_631BakedVae.safetensors', + 'dreamshaper_631Inpainting.safetensors', + 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', + 'juggernaut_final-inpainting.safetensors', + 'juggernaut_final.safetensors', + 'loraChGirl.safetensors', + 'sd-v1-5-inpainting.ckpt', + 'sd_xl_base_1.0.safetensors', + 'sd_xl_refiner_1.0.safetensors', + 'v1-5-pruned-emaonly.ckpt', + ], + ], + received_value: 'v2-1_768-ema-pruned.ckpt', + }, + }, + ], + dependent_outputs: ['9', '12'], + class_type: 'CheckpointLoaderSimple', + }, + }, +} + +function logError(result: Result) { + // Top-level error + let has_error = false + let errorMessage = '' + + // Top-level error + if (result.error) { + errorMessage += `Error: ${result.error.message}\n` + has_error = true + if (result.error.details) { + errorMessage += `Details: ${result.error.details}\n` + } + } + + // Node errors + if (result.node_errors) { + for (const [node_id, node_error] of Object.entries( + result.node_errors + )) { + errorMessage += `Node ${node_id}:\n` + has_error = true + for (const error of node_error.errors) { + errorMessage += `- Error: ${error.message}\n` + if (error.details) { + errorMessage += ` Details: ${error.details}\n` + } + } + } + } + + if (errorMessage) { + app.showAlert(errorMessage) + } + return has_error +} +export async function postPrompt(prompt: any) { + try { + const url = 'http://127.0.0.1:8188/prompt' + const payload = { + prompt: prompt, + } + const result = await requestPost(url, payload) + + return result + } catch (e) { + console.error(e) + } +} + +const timer = (ms: any) => new Promise((res) => setTimeout(res, ms)) + +export async function generateRequest(prompt: any) { + try { + const prompt_result = await postPrompt(prompt) + const prompt_id = prompt_result.prompt_id + let history_result: any + let numberOfAttempts = 0 + let has_error = logError(prompt_result) + while (true && !has_error) { + try { + console.log('get history attempt: ', numberOfAttempts) + history_result = await getHistory(prompt_id) + console.log('history_result:', history_result) + if (history_result?.[prompt_id]) break + + numberOfAttempts += 1 + await timer(5000) + } catch (e) { + console.log('getHistory failed, retrying...') + } + } + return { history_result: history_result, prompt_id: prompt_id } + } catch (e) { + console.error(e) + } +} +export async function generateImage(prompt: any) { + try { + let { history_result, prompt_id }: any = await generateRequest(prompt) + const outputs: any[] = Object.values(history_result[prompt_id].outputs) + const images: any[] = [] + for (const output of outputs) { + if (Array.isArray(output.images)) { + images.push(...output.images) + } + } + + const base64_imgs = [] + for (const image of images) { + debugger + const img = await loadImage( + image.filename, + image.subfolder, + image.type + ) + base64_imgs.push(img) + } + + store.data.comfyui_output_images = base64_imgs + + const thumbnails = [] + for (const image of base64_imgs) { + thumbnails.push(await io.createThumbnail(image, 300)) + } + store.data.comfyui_output_thumbnail_images = thumbnails + + return base64_imgs + } catch (e) { + console.error(e) + } +} + +export async function getHistory(prompt_id: string) { + try { + const url = `http://127.0.0.1:8188/history/${prompt_id}` + const history_result: any = await requestGet(url) + + return history_result + } catch (e) { + console.error(e) + } +} + +export async function loadImage( + filename = 'ComfyUI_00003_.png', + subfolder = '', + type = 'output' +) { + try { + const url = `http://127.0.0.1:8188/view?filename=${filename}&subfolder=${subfolder}&type=${type}` + const image = await fetch(url) + + const array_buffer = await image.arrayBuffer() + //@ts-ignore + const b64 = _arrayBufferToBase64(array_buffer) + + return b64 + } catch (e) { + console.error(e) + } +} + +export async function getConfig() { + try { + const prompt = { + '1': { + inputs: {}, + class_type: 'GetConfig', + }, + } + + let { history_result, prompt_id }: any = await generateRequest(prompt) + //@ts-ignore + const config: ComfyUIConfig = Object.values( + history_result?.[prompt_id].outputs + )[0] + store.data.comfyui_config = { ...config } + return config + } catch (e) { + console.error(e) + } +} +export async function getWorkflowApi(image_path: string) { + try { + const prompt = { + '1': { + inputs: { + // image: 'C:/Users/abdul/Downloads/img2img_workflow.png', + image_path: image_path, + }, + class_type: 'LoadImageWithMetaData', + }, + } + let { history_result, prompt_id }: any = await generateRequest(prompt) + //@ts-ignore + let { prompt: result_prompt, workflow } = Object.values( + history_result?.[prompt_id].outputs + )[0] + result_prompt = JSON.parse(result_prompt.join('')) + workflow = JSON.parse(workflow.join('')) + + return { prompt: result_prompt, workflow } + } catch (e) { + console.error(e) + } +} +enum InputTypeEnum { + NumberField = 'NumberField', + TextField = 'TextField', + TextArea = 'TextArea', + Menu = 'Menu', + ImageBase64 = 'ImageBase64', +} +const ValidNodes: any = { + KSampler: { + inputs: { + seed: 'TextField', + steps: 'NumberField', + cfg: 'NumberField', + sampler_name: 'Menu', + scheduler: 'Menu', + denoise: 'NumberField', + }, + list_id: { + sampler_name: 'samplers', + scheduler: 'schedulers', + }, + }, + EmptyLatentImage: { + inputs: { + width: 'NumberField', + height: 'NumberField', + batch_size: 'NumberField', + }, + }, + CLIPTextEncode: { + inputs: { + text: 'TextArea', + }, + }, + + LatentUpscale: { + inputs: { + upscale_method: 'Menu', + width: 'NumberField', + height: 'NumberField', + crop: 'Menu', + }, + list_id: { + upscale_method: 'latent_upscale_methods', + crop: 'latent_upscale_crop_methods', + }, + }, + CheckpointLoaderSimple: { + inputs: { + ckpt_name: 'Menu', + }, + list_id: { + ckpt_name: 'checkpoints', + }, + }, + LoadImage: { + inputs: { + image: 'ImageBase64', + }, + }, + LoadImageBase64: { + inputs: { + image_base64: 'ImageBase64', + }, + }, +} + +interface ComfyUINode { + inputs: any + class_type: string +} +function filterObjectProperties(node_inputs: any, valid_keys: string[]) { + return Object.fromEntries( + valid_keys.map((key: string) => [key, node_inputs[key]]) + ) +} +export function parseUIFromNode(node: ComfyUINode, node_id: string) { + //convert node to array of ui element definition + try { + const valid_ui = ValidNodes[node.class_type]?.inputs // all the valid inputs of a node + const list_ids = ValidNodes[node.class_type]?.list_id + if (valid_ui) { + const keys = Object.keys(valid_ui) + const filtered_values = filterObjectProperties(node.inputs, keys) + const entires = keys.map((key) => { + //example values: + // "sampler_name": { + // "label": "sampler_name", + // "value": "dpmpp_2m", + // "type": "Menu", + // "list_id": "samplers" + // }, + + return [ + key, + { + label: key, + value: filtered_values[key], + type: valid_ui[key], + list_id: list_ids?.[key], + node_id: node_id, + }, + ] + }) + + const valid_node_input = Object.fromEntries(entires) + store.data.comfyui_valid_nodes[node_id] = valid_node_input + + return valid_node_input + } + } catch (e) { + console.error(e) + } +} +interface ValidInput { + [key: string]: any + value: string | number + label: string + list?: any[] + type: InputTypeEnum + id?: string +} +interface PhotoshopNode { + inputs: ValidInput[] + id: string +} + +interface ComfyUIConfig { + [key: string]: any + checkpoints: string[] + samplers: string[] + schedulers: string[] +} +export const store = new AStore({ + comfyui_valid_nodes: {} as any, // comfyui nodes like structure that contain all info necessary to create plugin ui elements + + comfyui_output_images: [] as string[], //store the output images from generation + comfyui_output_thumbnail_images: [] as string[], // store thumbnail size images + comfyui_config: {} as ComfyUIConfig, // all config data like samplers, checkpoints ...etc + workflow_path: '', // the path of an image that contains prompt information + current_prompt: {} as any, // current prompt extracted from the workflow + thumbnail_image_size: 100, + load_image_nodes: {} as any, //our custom loadImageBase64 nodes, we need to substitute comfyui LoadImage nodes with before generating a prompt + // load_image_base64_strings: {} as any, //images the user added to the plugin comfy ui +}) + +export function storeToPrompt(store: any, basePrompt: any) { + //TODO change .map to .forEach + let modified_prompt = { ...basePrompt } // the original prompt but with the value of the ui + Object.entries(store.data.comfyui_valid_nodes).forEach( + ([node_id, node_inputs]: [string, any]) => { + Object.entries(node_inputs).forEach( + ([input_id, node_input]: [string, any]) => { + return (modified_prompt[node_id]['inputs'][input_id] = + node_input.value) + } + ) + } + ) + // store.data. + // modified_propmt[load_image_node_id] + // prompt = { ...store.data.comfyui_valid_nodes } + + return modified_prompt +} +function createMenu(input: ValidInput) { + return ( + <> + + {input.label} + + { + input.value = value.item + }} + > + + ) +} + +function createTextField(input: ValidInput) { + let element = ( + <> + {input.label} + { + input.value = evt.target.value + }} + > + + ) + return element +} +function createTextArea(input: ValidInput) { + let element = ( + <> + {input.label} + { + input.value = event.target.value + }} + placeholder={`${input.label}`} + value={input.value} + > + + ) + return element +} +function createImageBase64(input: ValidInput) { + let element = ( + <> +
+ +
+
+ +
+ + ) + return element +} +function nodeInputToHtmlElement(input: ValidInput) { + debugger + let element + if ( + [InputTypeEnum.NumberField, InputTypeEnum.TextField].includes( + input.type + ) + ) { + element = createTextField(input) + } else if ([InputTypeEnum.Menu].includes(input.type)) { + element = createMenu(input) + } else if ([InputTypeEnum.TextArea].includes(input.type)) { + element = createTextArea(input) + } else if ([InputTypeEnum.ImageBase64].includes(input.type)) { + element = createImageBase64(input) + } + return element +} +@observer +class ComfyNodeComponent extends React.Component<{}> { + async componentDidMount(): Promise { + await getConfig() + } + + render(): React.ReactNode { + return ( +
+
+ workflow path: + { + store.data.workflow_path = evt.target.value + }} + > +
+ + +
+
+ {Object.keys(store.data.comfyui_valid_nodes).map( + (node_id: string, index) => { + return ( +
+ {Object.keys( + store.data.comfyui_valid_nodes[node_id] + ).map((input_id: string, index: number) => { + return ( +
+ {nodeInputToHtmlElement( + store.data.comfyui_valid_nodes[ + node_id + ][input_id] + )} +
+ ) + })} +
+ ) + } + )} +
+ Result: + { + store.data.thumbnail_image_size = evt.target.value + }} + > + + Thumbnail Size: + + + {parseInt(store.data.thumbnail_image_size as any)} + + + { + io.IO.base64ToLayer( + store.data.comfyui_output_images[index] + ) + }, + title: 'Copy Image to Canvas', + }, + ]} + > +
+
+ ) + } +} + +const container = document.getElementById('ComfyUIContainer')! +const root = ReactDOM.createRoot(container) +root.render( + + +
+ + + +
+ {/* */} + + +) diff --git a/typescripts/comfyui/prompt.json b/typescripts/comfyui/prompt.json new file mode 100644 index 00000000..25b43bb0 --- /dev/null +++ b/typescripts/comfyui/prompt.json @@ -0,0 +1,149 @@ +{ + "3": { + "inputs": { + "seed": 89848141647836, + "steps": 12, + "cfg": 8, + "sampler_name": "dpmpp_sde", + "scheduler": "normal", + "denoise": 1, + "model": [ + "16", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler" + }, + "5": { + "inputs": { + "width": 768, + "height": 768, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage" + }, + "6": { + "inputs": { + "text": "masterpiece HDR victorian portrait painting of woman, blonde hair, mountain nature, blue sky\n", + "clip": [ + "16", + 1 + ] + }, + "class_type": "CLIPTextEncode" + }, + "7": { + "inputs": { + "text": "bad hands, text, watermark\n", + "clip": [ + "16", + 1 + ] + }, + "class_type": "CLIPTextEncode" + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "16", + 2 + ] + }, + "class_type": "VAEDecode" + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage" + }, + "10": { + "inputs": { + "upscale_method": "nearest-exact", + "width": 1152, + "height": 1152, + "crop": "disabled", + "samples": [ + "3", + 0 + ] + }, + "class_type": "LatentUpscale" + }, + "11": { + "inputs": { + "seed": 463499690269752, + "steps": 14, + "cfg": 8, + "sampler_name": "dpmpp_2m", + "scheduler": "simple", + "denoise": 0.5, + "model": [ + "16", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "10", + 0 + ] + }, + "class_type": "KSampler" + }, + "12": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "13", + 0 + ] + }, + "class_type": "SaveImage" + }, + "13": { + "inputs": { + "samples": [ + "11", + 0 + ], + "vae": [ + "16", + 2 + ] + }, + "class_type": "VAEDecode" + }, + "16": { + "inputs": { + "ckpt_name": "juggernaut_final.safetensors" + }, + "class_type": "CheckpointLoaderSimple" + } +} \ No newline at end of file diff --git a/typescripts/entry.ts b/typescripts/entry.ts index 10628161..4de4915d 100644 --- a/typescripts/entry.ts +++ b/typescripts/entry.ts @@ -34,5 +34,6 @@ export * as tool_bar from './tool_bar/tool_bar' export * as extra_page from './extra_page/extra_page' export * as selection_ts from './util/ts/selection' export * as stores from './stores' +export * as comfyui from './comfyui/comfyui' export { toJS } from 'mobx' export { default as node_fs } from 'fs' From cc1572414342faa7974e05b887e39082020c5624 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj Date: Wed, 6 Sep 2023 03:40:47 +0300 Subject: [PATCH 02/97] add workflow menu --- typescripts/comfyui/comfyui.tsx | 167 +++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 57 deletions(-) diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index 449ae032..14dd3570 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -18,6 +18,7 @@ import { Grid } from '../util/grid' import { io } from '../util/oldSystem' import { app } from 'photoshop' import { reaction } from 'mobx' +import { storage } from 'uxp' export let hi_res_prompt_temp = hi_res_prompt console.log('hi_res_prompt: ', hi_res_prompt) @@ -117,6 +118,19 @@ function logError(result: Result) { } return has_error } + +export async function workflowEntries() { + const workflow_folder = await storage.localFileSystem.getFolder() + + let entries = await workflow_folder.getEntries() + const workflow_entries = entries.filter( + (e: any) => e.isFile && e.name.toLowerCase().includes('.png') // must be a file and has the of the type .png + ) + + console.log('workflow_entries: ', workflow_entries) + + return workflow_entries +} export async function postPrompt(prompt: any) { try { const url = 'http://127.0.0.1:8188/prompt' @@ -171,7 +185,6 @@ export async function generateImage(prompt: any) { const base64_imgs = [] for (const image of images) { - debugger const img = await loadImage( image.filename, image.subfolder, @@ -375,7 +388,7 @@ export function parseUIFromNode(node: ComfyUINode, node_id: string) { const valid_node_input = Object.fromEntries(entires) store.data.comfyui_valid_nodes[node_id] = valid_node_input - + store.data.uuids[node_id] = window.crypto.randomUUID() return valid_node_input } } catch (e) { @@ -403,11 +416,17 @@ interface ComfyUIConfig { } export const store = new AStore({ comfyui_valid_nodes: {} as any, // comfyui nodes like structure that contain all info necessary to create plugin ui elements + uuids: {} as any, comfyui_output_images: [] as string[], //store the output images from generation comfyui_output_thumbnail_images: [] as string[], // store thumbnail size images comfyui_config: {} as ComfyUIConfig, // all config data like samplers, checkpoints ...etc workflow_path: '', // the path of an image that contains prompt information + workflow_dir_path: '', // the path of the directory that contains all workflow files + // workflows_paths: [] as string[], + // workflows_names: [] as string[], + workflows: {} as any, + selected_workflow: '', // the selected workflow from the workflow menu current_prompt: {} as any, // current prompt extracted from the workflow thumbnail_image_size: 100, load_image_nodes: {} as any, //our custom loadImageBase64 nodes, we need to substitute comfyui LoadImage nodes with before generating a prompt @@ -527,7 +546,6 @@ function createImageBase64(input: ValidInput) { return element } function nodeInputToHtmlElement(input: ValidInput) { - debugger let element if ( [InputTypeEnum.NumberField, InputTypeEnum.TextField].includes( @@ -544,6 +562,41 @@ function nodeInputToHtmlElement(input: ValidInput) { } return element } + +export async function loadWorkflow(workflow_path: string) { + store.data.current_prompt = (await getWorkflowApi(workflow_path))?.prompt + + const current_prompt: any = store.toJsFunc().data.current_prompt + const loadImageNodes = Object.keys(current_prompt) + .filter((key: any) => current_prompt[key].class_type === 'LoadImage') + .reduce( + (acc, key) => ({ + ...acc, + [key]: current_prompt[key], + }), + {} + ) + + Object.keys(loadImageNodes).forEach((node_id: any) => { + store.data.current_prompt[node_id] = { + inputs: { + image_base64: '', + }, + class_type: 'LoadImageBase64', + } + }) + + const node_obj = Object.entries(store.data.current_prompt) + //clear both node structure and base64 images store values + store.data.comfyui_valid_nodes = {} + store.data.uuids = {} + // store.data.load_image_base64_strings = {} + node_obj.forEach(([node_id, node]: [string, any]) => { + console.log(node_id, node) + const valid_input = parseUIFromNode(node, node_id) + }) +} + @observer class ComfyNodeComponent extends React.Component<{}> { async componentDidMount(): Promise { @@ -554,6 +607,53 @@ class ComfyNodeComponent extends React.Component<{}> { return (
+ + { + store.data.workflow_dir_path = evt.target.value + }} + > + +
+ + {'select workflow:'} + + { + store.data.selected_workflow = value.item + await loadWorkflow( + store.data.workflows[value.item] + ) + }} + > +
workflow path: { className="btnSquare" style={{ marginRight: '3px' }} onClick={async () => { - store.data.current_prompt = ( - await getWorkflowApi( - store.data.workflow_path - ) - )?.prompt - - const current_prompt: any = - store.toJsFunc().data.current_prompt - const loadImageNodes = Object.keys( - current_prompt - ) - .filter( - (key: any) => - current_prompt[key].class_type === - 'LoadImage' - ) - .reduce( - (acc, key) => ({ - ...acc, - [key]: current_prompt[key], - }), - {} - ) - - Object.keys(loadImageNodes).forEach( - (node_id: any) => { - store.data.current_prompt[node_id] = { - inputs: { - image_base64: '', - }, - class_type: 'LoadImageBase64', - } - } - ) - - const node_obj = Object.entries( - store.data.current_prompt - ) - //clear both node structure and base64 images store values - store.data.comfyui_valid_nodes = {} - // store.data.load_image_base64_strings = {} - node_obj.forEach( - ([node_id, node]: [string, any]) => { - console.log(node_id, node) - const valid_input = parseUIFromNode( - node, - node_id - ) - } - ) + await loadWorkflow(store.data.workflow_path) }} > Load Workflow @@ -639,10 +690,10 @@ class ComfyNodeComponent extends React.Component<{}> {
{Object.keys(store.data.comfyui_valid_nodes).map( - (node_id: string, index) => { + (node_id: string, index_i) => { return (
{ > {Object.keys( store.data.comfyui_valid_nodes[node_id] - ).map((input_id: string, index: number) => { + ).map((input_id: string, index_j: number) => { return ( -
+
{nodeInputToHtmlElement( store.data.comfyui_valid_nodes[ node_id From 0987576339cc76338ce61b3738459591798009e3 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj Date: Fri, 8 Sep 2023 03:26:46 +0300 Subject: [PATCH 03/97] add support to LoraLoader node --- typescripts/comfyui/comfyui.tsx | 139 ++++++++++++++++++++------------ 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index 14dd3570..e2409ce7 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -17,7 +17,7 @@ import hi_res_prompt from './prompt.json' import { Grid } from '../util/grid' import { io } from '../util/oldSystem' import { app } from 'photoshop' -import { reaction } from 'mobx' +import { reaction, toJS } from 'mobx' import { storage } from 'uxp' export let hi_res_prompt_temp = hi_res_prompt console.log('hi_res_prompt: ', hi_res_prompt) @@ -120,16 +120,20 @@ function logError(result: Result) { } export async function workflowEntries() { - const workflow_folder = await storage.localFileSystem.getFolder() + try { + const workflow_folder = await storage.localFileSystem.getFolder() - let entries = await workflow_folder.getEntries() - const workflow_entries = entries.filter( - (e: any) => e.isFile && e.name.toLowerCase().includes('.png') // must be a file and has the of the type .png - ) + let entries = await workflow_folder.getEntries() + const workflow_entries = entries.filter( + (e: any) => e.isFile && e.name.toLowerCase().includes('.png') // must be a file and has the of the type .png + ) - console.log('workflow_entries: ', workflow_entries) + console.log('workflow_entries: ', workflow_entries) - return workflow_entries + return workflow_entries + } catch (e) { + console.error(e) + } } export async function postPrompt(prompt: any) { try { @@ -346,6 +350,16 @@ const ValidNodes: any = { image_base64: 'ImageBase64', }, }, + LoraLoader: { + inputs: { + lora_name: 'Menu', + strength_model: 'NumberField', + strength_clip: 'NumberField', + }, + list_id: { + lora_name: 'loras', + }, + }, } interface ComfyUINode { @@ -453,6 +467,7 @@ export function storeToPrompt(store: any, basePrompt: any) { return modified_prompt } function createMenu(input: ValidInput) { + console.log('input: ', toJS(input)) return ( <> current_prompt[key].class_type === 'LoadImage') - .reduce( - (acc, key) => ({ - ...acc, - [key]: current_prompt[key], - }), - {} - ) + try { + store.data.current_prompt = ( + await getWorkflowApi(workflow_path) + )?.prompt + + const current_prompt: any = store.toJsFunc().data.current_prompt + const loadImageNodes = Object.keys(current_prompt) + .filter( + (key: any) => current_prompt[key].class_type === 'LoadImage' + ) + .reduce( + (acc, key) => ({ + ...acc, + [key]: current_prompt[key], + }), + {} + ) - Object.keys(loadImageNodes).forEach((node_id: any) => { - store.data.current_prompt[node_id] = { - inputs: { - image_base64: '', - }, - class_type: 'LoadImageBase64', - } - }) - - const node_obj = Object.entries(store.data.current_prompt) - //clear both node structure and base64 images store values - store.data.comfyui_valid_nodes = {} - store.data.uuids = {} - // store.data.load_image_base64_strings = {} - node_obj.forEach(([node_id, node]: [string, any]) => { - console.log(node_id, node) - const valid_input = parseUIFromNode(node, node_id) - }) + Object.keys(loadImageNodes).forEach((node_id: any) => { + store.data.current_prompt[node_id] = { + inputs: { + image_base64: '', + }, + class_type: 'LoadImageBase64', + } + }) + + const node_obj = Object.entries(store.data.current_prompt) + //clear both node structure and base64 images store values + store.data.comfyui_valid_nodes = {} + store.data.uuids = {} + // store.data.load_image_base64_strings = {} + node_obj.forEach(([node_id, node]: [string, any]) => { + console.log(node_id, node) + const valid_input = parseUIFromNode(node, node_id) + }) + } catch (e) { + console.error(e) + } } @observer class ComfyNodeComponent extends React.Component<{}> { async componentDidMount(): Promise { - await getConfig() + try { + await getConfig() + } catch (e) { + console.error(e) + } } render(): React.ReactNode { @@ -620,12 +647,16 @@ class ComfyNodeComponent extends React.Component<{}> { > +
- {'select workflow:'} - + */} { store.data.workflows[value.item] ) }} - > + >{' '} +
workflow path: Date: Wed, 18 Oct 2023 08:34:45 +0300 Subject: [PATCH 06/97] add an option to disabled to SpMenu and SpSliderWithLabel --- typescripts/after_detailer/after_detailer.tsx | 2 -- typescripts/main/main.tsx | 1 - typescripts/ultimate_sd_upscaler/scripts.tsx | 2 +- typescripts/util/elements.tsx | 8 ++++++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/typescripts/after_detailer/after_detailer.tsx b/typescripts/after_detailer/after_detailer.tsx index 88768516..a7150cc9 100644 --- a/typescripts/after_detailer/after_detailer.tsx +++ b/typescripts/after_detailer/after_detailer.tsx @@ -155,7 +155,6 @@ export class AfterDetailerComponent extends React.Component<{ { { // const [sliderValue,setSliderValue] = useState(0) state = { output_value: this.props.output_value || 0, slider_value: 0 } @@ -164,6 +165,7 @@ export class SpSliderWithLabel extends React.Component<{ show-value="false" // id="slControlNetWeight_0" class="slControlNetWeight_" + disabled={this.props.disabled ? true : void 0} min={this.in_min} max={this.in_max} value={this.state.slider_value} @@ -197,7 +199,8 @@ export class SpMenu extends React.Component<{ title?: string style?: CSSProperties items?: string[] - disabled?: boolean[] + disabled?: boolean + disabled_items?: boolean[] label_item?: string onChange?: any selected_index?: number @@ -238,6 +241,7 @@ export class SpMenu extends React.Component<{ title={this.props.title} size={this.props.size || 'm'} style={this.props.style} + disabled={this.props.disabled ? 'disabled' : undefined} // style={{ width: '199px', marginRight: '5px' }} > @@ -263,7 +267,7 @@ export class SpMenu extends React.Component<{ : undefined } disabled={ - this.props.disabled?.[index] + this.props.disabled_items?.[index] ? 'disabled' : undefined } From dda6185bdb2df31a5229bc9659dd89e3396efa51 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sat, 21 Oct 2023 18:28:05 +0300 Subject: [PATCH 07/97] a better approach to loading comfy workflow without the use of special custom nodes --- index.html | 1 + index.js | 5 + package-lock.json | 38 +- package.json | 5 +- typescripts/comfyui/animatediff_workflow.json | 92 ++ typescripts/comfyui/comfyui.tsx | 898 ++++++++++++++++-- .../{prompt.json => hi_res_workflow.json} | 87 +- typescripts/comfyui/img2img_workflow.json | 65 ++ typescripts/comfyui/lora_less_workflow.json | 187 ++++ typescripts/comfyui/util.ts | 356 +++++++ typescripts/entry.ts | 3 + 11 files changed, 1598 insertions(+), 139 deletions(-) create mode 100644 typescripts/comfyui/animatediff_workflow.json rename typescripts/comfyui/{prompt.json => hi_res_workflow.json} (58%) create mode 100644 typescripts/comfyui/img2img_workflow.json create mode 100644 typescripts/comfyui/lora_less_workflow.json create mode 100644 typescripts/comfyui/util.ts diff --git a/index.html b/index.html index 421ac8ac..fdb4e07e 100644 --- a/index.html +++ b/index.html @@ -1185,6 +1185,7 @@
+
diff --git a/index.js b/index.js index 9198809a..569677d2 100644 --- a/index.js +++ b/index.js @@ -94,6 +94,8 @@ const { lexica, api_ts, comfyui, + comfyui_util, + diffusion_chain, } = require('./typescripts/dist/bundle') const io = require('./utility/io') @@ -1814,3 +1816,6 @@ async function openFileFromUrlExe(url, format = 'gif') { await openFileFromUrl(url, format) }) } + +let comfy_server = new diffusion_chain.ComfyServer('http://127.0.0.1:8188') +let comfy_object_info = diffusion_chain.ComfyApi.objectInfo(comfy_server) diff --git a/package-lock.json b/package-lock.json index a04a78f0..47dab9e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "changedpi": "^1.0.4", + "diffusion-chain": "file:../diffusion-chain", "fastify": "^4.10.2", "jimp": "^0.16.2", "madge": "^6.0.0", @@ -49,6 +50,16 @@ "yazl": "^2.5.1" } }, + "../diffusion-chain": { + "version": "1.0.7", + "license": "MIT", + "dependencies": { + "@types/node": "^20.4.0", + "mkdirp": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.1.6" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -4731,6 +4742,10 @@ "node": ">=4.2.0" } }, + "node_modules/diffusion-chain": { + "resolved": "../diffusion-chain", + "link": true + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -9272,15 +9287,15 @@ } }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/undefsafe": { @@ -13368,6 +13383,15 @@ } } }, + "diffusion-chain": { + "version": "file:../diffusion-chain", + "requires": { + "@types/node": "^20.4.0", + "mkdirp": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.1.6" + } + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -16652,9 +16676,9 @@ } }, "typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==" + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", + "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==" }, "undefsafe": { "version": "2.0.5", diff --git a/package.json b/package.json index 356303a8..bd4b0058 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@types/react": "^18.2.6", "@types/react-dom": "^18.2.4", "changedpi": "^1.0.4", + "diffusion-chain": "file:../diffusion-chain", "fastify": "^4.10.2", "jimp": "^0.16.2", "madge": "^6.0.0", @@ -30,6 +31,7 @@ "@babel/plugin-transform-react-jsx": "^7.21.5", "@svgr/webpack": "^8.0.1", "babel-loader": "^9.1.2", + "chalk": "^5.3.0", "clean-webpack-plugin": "^4.0.0", "commander": "^11.0.0", "copy-webpack-plugin": "^11.0.0", @@ -45,7 +47,6 @@ "url-loader": "^4.1.1", "webpack": "^5.82.1", "webpack-cli": "^5.1.1", - "chalk": "^5.3.0", "yazl": "^2.5.1" }, "scripts": { @@ -64,4 +65,4 @@ "url": "https://github.com/AbdullahAlfaraj/Auto-Photoshop-StableDiffusion-Plugin/issues" }, "homepage": "https://github.com/AbdullahAlfaraj/Auto-Photoshop-StableDiffusion-Plugin#readme" -} \ No newline at end of file +} diff --git a/typescripts/comfyui/animatediff_workflow.json b/typescripts/comfyui/animatediff_workflow.json new file mode 100644 index 00000000..c1353e16 --- /dev/null +++ b/typescripts/comfyui/animatediff_workflow.json @@ -0,0 +1,92 @@ +{ + "1": { + "inputs": { + "ckpt_name": "cardosAnime_v20.safetensors", + "beta_schedule": "sqrt_linear (AnimateDiff)" + }, + "class_type": "CheckpointLoaderSimpleWithNoiseSelect" + }, + "2": { + "inputs": { + "vae_name": "MoistMix.vae.pt" + }, + "class_type": "VAELoader" + }, + "3": { + "inputs": { + "text": "ship in storm, waves, dark, night, Artstation ", + "clip": ["4", 0] + }, + "class_type": "CLIPTextEncode" + }, + "4": { + "inputs": { + "stop_at_clip_layer": -2, + "clip": ["1", 1] + }, + "class_type": "CLIPSetLastLayer" + }, + "6": { + "inputs": { + "text": "(ugly:1.2), (worst quality, low quality: 1.4)", + "clip": ["4", 0] + }, + "class_type": "CLIPTextEncode" + }, + "7": { + "inputs": { + "seed": 711493021904285, + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": ["8", 0], + "positive": ["3", 0], + "negative": ["6", 0], + "latent_image": ["8", 1] + }, + "class_type": "KSampler" + }, + "8": { + "inputs": { + "model_name": "mm_sd_v14.ckpt", + "unlimited_area_hack": true, + "model": ["1", 0], + "latents": ["9", 0] + }, + "class_type": "AnimateDiffLoaderV1" + }, + "9": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 16 + }, + "class_type": "EmptyLatentImage" + }, + "10": { + "inputs": { + "samples": ["7", 0], + "vae": ["2", 0] + }, + "class_type": "VAEDecode" + }, + "12": { + "inputs": { + "filename_prefix": "AA_readme", + "images": ["10", 0] + }, + "class_type": "SaveImage" + }, + "26": { + "inputs": { + "frame_rate": 8, + "loop_count": 0, + "save_image": "Enabled", + "filename_prefix": "AA_readme_gif", + "images": ["10", 0] + }, + "class_type": "ADE_AnimateDiffCombine" + } +} diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index aad1d330..b3a2686f 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -2,10 +2,13 @@ import React from 'react' import ReactDOM from 'react-dom/client' import { requestGet, requestPost } from '../util/ts/api' import { observer } from 'mobx-react' +import { runInAction } from 'mobx' import { MoveToCanvasSvg, + SliderType, SpMenu, SpSlider, + SpSliderWithLabel, SpTextfield, } from '../util/elements' import { ErrorBoundary } from '../util/errorBoundary' @@ -13,15 +16,15 @@ import { Collapsible } from '../util/collapsible' import Locale from '../locale/locale' import { AStore } from '../main/astore' -import hi_res_prompt from './prompt.json' import { Grid } from '../util/grid' import { io } from '../util/oldSystem' import { app } from 'photoshop' import { reaction, toJS } from 'mobx' import { storage } from 'uxp' -export let hi_res_prompt_temp = hi_res_prompt -console.log('hi_res_prompt: ', hi_res_prompt) +import util from './util' +import * as diffusion_chain from 'diffusion-chain' +import { urlToCanvas } from '../util/ts/general' interface Error { type: string message: string @@ -40,49 +43,6 @@ interface Result { node_errors?: { [key: string]: NodeError } } -const result: Result = { - error: { - type: 'prompt_outputs_failed_validation', - message: 'Prompt outputs failed validation', - details: '', - extra_info: {}, - }, - node_errors: { - '16': { - errors: [ - { - type: 'value_not_in_list', - message: 'Value not in list', - details: - "ckpt_name: 'v2-1_768-ema-pruned.ckpt' not in ['anythingV5Anything_anythingV5PrtRE.safetensors', 'deliberate_v2.safetensors', 'dreamshaper_631BakedVae.safetensors', 'dreamshaper_631Inpainting.safetensors', 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', 'juggernaut_final-inpainting.safetensors', 'juggernaut_final.safetensors', 'loraChGirl.safetensors', 'sd-v1-5-inpainting.ckpt', 'sd_xl_base_1.0.safetensors', 'sd_xl_refiner_1.0.safetensors', 'v1-5-pruned-emaonly.ckpt']", - extra_info: { - input_name: 'ckpt_name', - input_config: [ - [ - 'anythingV5Anything_anythingV5PrtRE.safetensors', - 'deliberate_v2.safetensors', - 'dreamshaper_631BakedVae.safetensors', - 'dreamshaper_631Inpainting.safetensors', - 'edgeOfRealism_eorV20Fp16BakedVAE.safetensors', - 'juggernaut_final-inpainting.safetensors', - 'juggernaut_final.safetensors', - 'loraChGirl.safetensors', - 'sd-v1-5-inpainting.ckpt', - 'sd_xl_base_1.0.safetensors', - 'sd_xl_refiner_1.0.safetensors', - 'v1-5-pruned-emaonly.ckpt', - ], - ], - received_value: 'v2-1_768-ema-pruned.ckpt', - }, - }, - ], - dependent_outputs: ['9', '12'], - class_type: 'CheckpointLoaderSimple', - }, - }, -} - function logError(result: Result) { // Top-level error let has_error = false @@ -179,6 +139,7 @@ export async function generateRequest(prompt: any) { export async function generateImage(prompt: any) { try { let { history_result, prompt_id }: any = await generateRequest(prompt) + const outputs: any[] = Object.values(history_result[prompt_id].outputs) const images: any[] = [] for (const output of outputs) { @@ -188,6 +149,8 @@ export async function generateImage(prompt: any) { } const base64_imgs = [] + const formats: string[] = [] + for (const image of images) { const img = await loadImage( image.filename, @@ -195,14 +158,20 @@ export async function generateImage(prompt: any) { image.type ) base64_imgs.push(img) + formats.push(util.getFileFormat(image.filename)) } store.data.comfyui_output_images = base64_imgs const thumbnails = [] - for (const image of base64_imgs) { - thumbnails.push(await io.createThumbnail(image, 300)) + for (let i = 0; i < base64_imgs.length; ++i) { + if (['png', 'webp', 'jpg'].includes(formats[i])) { + thumbnails.push(await io.createThumbnail(base64_imgs[i], 300)) + } else if (['gif'].includes(formats[i])) { + thumbnails.push('data:image/gif;base64,' + base64_imgs[i]) + } } + store.data.comfyui_output_thumbnail_images = thumbnails return base64_imgs @@ -242,6 +211,7 @@ export async function loadImage( } export async function getConfig() { + //TODO: replace this method with get_object_info from comfyapi try { const prompt = { '1': { @@ -376,11 +346,31 @@ export const store = new AStore({ // workflows_paths: [] as string[], // workflows_names: [] as string[], workflows: {} as any, - selected_workflow: '', // the selected workflow from the workflow menu + selected_workflow_name: '', // the selected workflow from the workflow menu current_prompt: {} as any, // current prompt extracted from the workflow thumbnail_image_size: 100, load_image_nodes: {} as any, //our custom loadImageBase64 nodes, we need to substitute comfyui LoadImage nodes with before generating a prompt // load_image_base64_strings: {} as any, //images the user added to the plugin comfy ui + object_info: undefined as any, + current_prompt2: {} as any, + current_prompt2_output: {} as any, + output_thumbnail_image_size: {} as Record, + comfy_server: new diffusion_chain.ComfyServer( + 'http://127.0.0.1:8188' + ) as diffusion_chain.ComfyServer, + loaded_images_base64_url: [] as string[], + current_loaded_image: {} as Record, + loaded_images_list: [] as string[], // store an array of all images in the comfy's input directory + nodes_order: [] as string[], // nodes with smaller index will be rendered first, + can_edit_nodes: false as boolean, + nodes_label: {} as Record, + workflows2: { + hi_res_workflow: util.hi_res_workflow, + lora_less_workflow: util.lora_less_workflow, + img2img_workflow: util.img2img_workflow, + animatediff_workflow: util.animatediff_workflow, + } as Record, + progress_value: 0, }) export function storeToPrompt(store: any, basePrompt: any) { @@ -615,9 +605,9 @@ class ComfyNodeComponent extends React.Component<{}> { label_item="Select a workflow" selected_index={Object.values( store.data.workflows - ).indexOf(store.data.selected_workflow)} + ).indexOf(store.data.selected_workflow_name)} onChange={async (id: any, value: any) => { - store.data.selected_workflow = value.item + store.data.selected_workflow_name = value.item await loadWorkflow( store.data.workflows[value.item] ) @@ -745,17 +735,803 @@ class ComfyNodeComponent extends React.Component<{}> { } } +function setSliderValue(store: any, node_id: string, name: string, value: any) { + runInAction(() => { + store.data.current_prompt2[node_id].inputs[name] = value + }) +} +async function onChangeLoadImage(node_id: string, filename: string) { + try { + store.data.current_loaded_image[node_id] = + await util.base64UrlFromComfy(store.data.comfy_server, { + filename: encodeURIComponent(filename), + type: 'input', + subfolder: '', + }) + } catch (e) { + console.warn(e) + } +} +function renderNode(node_id: string, node: any) { + const comfy_node_info = toJS(store.data.object_info[node.class_type]) + const is_output = comfy_node_info.output_node + console.log('comfy_node_info: ', comfy_node_info) + const node_type = util.getNodeType(node.class_type) + let node_html + if (node_type === util.ComfyNodeType.LoadImage) { + const loaded_images = store.data.loaded_images_list + const inputs = store.data.current_prompt2[node_id].inputs + const node_name = node.class_type + node_html = ( +
+ New load image component + + {node_name} + +
+ + ) => { + console.log('onChange value.item: ', item) + inputs.image = item + //load image store for each LoadImage Node + //use node_id to store these + + onChangeLoadImage(node_id, item) + }} + > +
+
+ + + +
+ { + console.error( + 'error loading image: ', + store.data.current_loaded_image[node_id] + ) + // try { + // const filename = inputs.image + // store.data.current_loaded_image[node_id] = + // await util.base64UrlFromComfy( + // store.data.comfy_server, + // { + // filename: encodeURIComponent(filename), + // type: 'input', + // subfolder: '', + // } + // ) + // console.log( + // 'store.data.current_loaded_image[node_id]: ', + // toJS(store.data.current_loaded_image[node_id]) + // ) + // } catch (e) { + // console.warn(e) + // } + onChangeLoadImage(node_id, inputs.image) + }} + /> + {/* */} +
+ ) + } else if (node_type === util.ComfyNodeType.Normal) { + node_html = Object.entries(node.inputs).map(([name, value], index) => { + // store.data.current_prompt2[node_id].inputs[name] = value + try { + const input = comfy_node_info.input.required[name] + let { type, config } = util.parseComfyInput(input) + const html_element = renderInput( + node_id, + name, + type, + config, + `${node_id}_${name}_${type}_${index}` + ) + return html_element + } catch (e) { + console.error(e) + } + }) + } + if (is_output) { + const output_node_element = ( +
+ { + store.data.output_thumbnail_image_size[node_id] = + evt.target.value + }} + > + + Thumbnail Size: + + + {parseInt( + store.data.output_thumbnail_image_size[ + node_id + ] as any + )} + + + { + // io.IO.base64ToLayer( + // store.data.current_prompt2_output[node_id][ + // index + // ] + // ) + urlToCanvas( + store.data.current_prompt2_output[node_id][ + index + ], + 'comfy_output.png' + ) + }, + title: 'Copy Image to Canvas', + }, + ]} + > +
+ ) + + return output_node_element + } + return node_html +} +function renderInput( + node_id: string, + name: string, + type: any, + config: any, + key?: string +) { + let html_element = ( +
+ {name},{type}, {JSON.stringify(config)} +
+ ) + const inputs = store.data.current_prompt2[node_id].inputs + if (type === util.ComfyInputType.BigNumber) { + html_element = ( + <> + {name}: + { + // store.data.search_query = event.target.value + inputs[name] = event.target.value + console.log(`${name}: ${event.target.value}`) + }} + > + + ) + } else if (type === util.ComfyInputType.TextFieldNumber) { + html_element = ( + <> + {name}: + { + const v = e.target.value + let new_value = + v !== '' + ? Math.max(config.min, Math.min(config.max, v)) + : v + inputs[name] = new_value + + console.log(`${name}: ${e.target.value}`) + }} + > + + ) + } else if (type === util.ComfyInputType.Slider) { + html_element = ( + { + // inputs[name] = new_value + // setSliderValue(store, node_id, name, new_value) + store.data.current_prompt2[node_id].inputs[name] = new_value + console.log('slider_change: ', new_value) + }} + /> + ) + } else if (type === util.ComfyInputType.Menu) { + html_element = ( + <> + + {name} + + + ) => { + console.log('onChange value.item: ', item) + inputs[name] = item + }} + > + + ) + } else if (type === util.ComfyInputType.TextArea) { + html_element = ( + { + try { + // this.changePositivePrompt( + // event.target.value, + // store.data.current_index + // ) + // autoResize( + // event.target, + // store.data.positivePrompts[ + // store.data.current_index + // ] + // ) + inputs[name] = event.target.value + } catch (e) { + console.warn(e) + } + }} + placeholder={`${name}`} + value={inputs[name]} + > + ) + } else if (type === util.ComfyInputType.TextField) { + html_element = ( + <> + {name}: + + { + inputs[name] = e.target.value + console.log(`${name}: ${e.target.value}`) + }} + > + + ) + } + + return
{html_element}
+} + +export function swap(index1: number, index2: number) { + const { length } = store.data.nodes_order + if (index1 >= 0 && index1 < length && index2 >= 0 && index2 < length) { + ;[store.data.nodes_order[index1], store.data.nodes_order[index2]] = [ + store.data.nodes_order[index2], + store.data.nodes_order[index1], + ] + } +} + +export function saveWorkflowData( + workflow_name: string, + { prompt, nodes_order, nodes_label }: WorkflowData +) { + storage.localStorage.setItem( + workflow_name, + JSON.stringify({ prompt, nodes_order, nodes_label }) + ) +} +export function loadWorkflowData(workflow_name: string): WorkflowData { + const workflow_data: WorkflowData = JSON.parse( + storage.localStorage.getItem(workflow_name) + ) + return workflow_data +} +interface WorkflowData { + prompt: any + nodes_order: string[] + nodes_label: Record +} +function loadWorkflow2(workflow: any) { + const copyJson = (originalObject: any) => + JSON.parse(JSON.stringify(originalObject)) + //1) get prompt + store.data.current_prompt2 = copyJson(workflow) + + //2) get the original order + store.data.nodes_order = Object.keys(toJS(store.data.current_prompt2)) + + //3) get labels for each nodes + store.data.nodes_label = Object.fromEntries( + Object.entries(toJS(store.data.current_prompt2)).map( + ([node_id, node]: [string, any]) => { + return [ + node_id, + toJS(store.data.object_info[node.class_type]).display_name, + ] + } + ) + ) + + // parse the output nodes + // Note: we can't modify the node directly in the prompt like we do for input nodes. + //.. since this data doesn't exist on the prompt. so we create separate container for the output images + store.data.current_prompt2_output = Object.entries( + store.data.current_prompt2 + ).reduce( + ( + output_entries: Record, + [node_id, node]: [string, any] + ) => { + if (store.data.object_info[node.class_type].output_node) { + output_entries[node_id] = [] + } + return output_entries + }, + {} + ) + + //slider variables for output nodes + //TODO: delete store.data.output_thumbnail_image_size before loading a new workflow + for (let key in toJS(store.data.current_prompt2_output)) { + store.data.output_thumbnail_image_size[key] = 200 + } + + const workflow_name = store.data.selected_workflow_name + if (workflow_name) { + // check if the workflow has a name + + if (workflow_name in storage.localStorage) { + //load the workflow data from local storage + //1) load the last parameters used in generation + //2) load the order of the nodes + //3) load the labels of the nodes + + const workflow_data: WorkflowData = loadWorkflowData(workflow_name) + if ( + util.isSameStructure( + workflow_data.prompt, + toJS(store.data.current_prompt2) + ) + ) { + //load 1) + store.data.current_prompt2 = workflow_data.prompt + //load 2) + store.data.nodes_order = workflow_data.nodes_order + //load 3) + store.data.nodes_label = workflow_data.nodes_label + } else { + // do not load. instead override the localStorage with the new values + workflow_data.prompt = toJS(store.data.current_prompt2) + workflow_data.nodes_order = toJS(store.data.nodes_order) + workflow_data.nodes_label = toJS(store.data.nodes_label) + + saveWorkflowData(workflow_name, workflow_data) + } + } else { + // if workflow data is missing from local storage then save it for next time. + //1) save parameters values + //2) save nodes order + //3) save nodes label + + const prompt = toJS(store.data.current_prompt2) + const nodes_order = toJS(store.data.nodes_order) + const nodes_label = toJS(store.data.nodes_label) + saveWorkflowData(workflow_name, { + prompt, + nodes_order, + nodes_label, + }) + } + } +} +@observer +class ComfyWorkflowComponent extends React.Component<{}, { value?: number }> { + async componentDidMount(): Promise { + try { + store.data.object_info = await diffusion_chain.ComfyApi.objectInfo( + store.data.comfy_server + ) + + loadWorkflow2(util.lora_less_workflow) + + //convert all of comfyui loaded images into base64url that the plugin can use + const loaded_images = + store.data.object_info.LoadImage.input.required['image'][0] + const loaded_images_base64_url = await Promise.all( + loaded_images.map(async (filename: string) => { + try { + return await util.base64UrlFromComfy( + store.data.comfy_server, + { + filename: encodeURIComponent(filename), + type: 'input', + subfolder: '', + } + ) + } catch (e) { + console.warn(e) + } + }) + ) + store.data.loaded_images_list = + store.data.object_info.LoadImage.input.required['image'][0] + + store.data.loaded_images_base64_url = loaded_images_base64_url + } catch (e) { + console.error(e) + } + } + + render(): React.ReactNode { + const comfy_server = store.data.comfy_server + return ( +
+
+ {/* {util.getNodes(util.hi_res_workflow).map((node, index) => { + // return
{node.class_type}
+ return ( +
{this.renderNode(node)}
+ ) + })} */} + + +
+
+ +
+
+ { + store.data.selected_workflow_name = value.item + loadWorkflow2(store.data.workflows2[value.item]) + }} + >{' '} + +
+ + {store.data.object_info ? ( + <> +
+ {util + .getNodes(store.data.current_prompt2) + .sort( + ([node_id1, node1], [node_id2, node2]) => { + return ( + store.data.nodes_order.indexOf( + node_id1 + ) - + store.data.nodes_order.indexOf( + node_id2 + ) + ) + } + ) + + .map(([node_id, node], index) => { + return ( +
+
+
+ +
+
+ + {/* */} +
+
+ + "{node_id}":{' '} + + { + store.data.nodes_label[ + node_id + ] + } + {' '} + {' '} + + {node.class_type} + +
+ { + store.data.nodes_label[ + node_id + ] = event.target.value + }} + > +
+ {renderNode(node_id, node)} + {/* + */} +
+ ) + })} +
+ + ) : ( + void 0 + )} +
+ ) + } +} const container = document.getElementById('ComfyUIContainer')! const root = ReactDOM.createRoot(container) root.render( - - -
- - - -
- {/* */} - - + // + +
+ + {/* */} + + + +
+ {/* */} + + // ) diff --git a/typescripts/comfyui/prompt.json b/typescripts/comfyui/hi_res_workflow.json similarity index 58% rename from typescripts/comfyui/prompt.json rename to typescripts/comfyui/hi_res_workflow.json index 25b43bb0..e2075d02 100644 --- a/typescripts/comfyui/prompt.json +++ b/typescripts/comfyui/hi_res_workflow.json @@ -7,22 +7,10 @@ "sampler_name": "dpmpp_sde", "scheduler": "normal", "denoise": 1, - "model": [ - "16", - 0 - ], - "positive": [ - "6", - 0 - ], - "negative": [ - "7", - 0 - ], - "latent_image": [ - "5", - 0 - ] + "model": ["16", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["5", 0] }, "class_type": "KSampler" }, @@ -37,43 +25,28 @@ "6": { "inputs": { "text": "masterpiece HDR victorian portrait painting of woman, blonde hair, mountain nature, blue sky\n", - "clip": [ - "16", - 1 - ] + "clip": ["16", 1] }, "class_type": "CLIPTextEncode" }, "7": { "inputs": { "text": "bad hands, text, watermark\n", - "clip": [ - "16", - 1 - ] + "clip": ["16", 1] }, "class_type": "CLIPTextEncode" }, "8": { "inputs": { - "samples": [ - "3", - 0 - ], - "vae": [ - "16", - 2 - ] + "samples": ["3", 0], + "vae": ["16", 2] }, "class_type": "VAEDecode" }, "9": { "inputs": { "filename_prefix": "ComfyUI", - "images": [ - "8", - 0 - ] + "images": ["8", 0] }, "class_type": "SaveImage" }, @@ -83,10 +56,7 @@ "width": 1152, "height": 1152, "crop": "disabled", - "samples": [ - "3", - 0 - ] + "samples": ["3", 0] }, "class_type": "LatentUpscale" }, @@ -98,45 +68,24 @@ "sampler_name": "dpmpp_2m", "scheduler": "simple", "denoise": 0.5, - "model": [ - "16", - 0 - ], - "positive": [ - "6", - 0 - ], - "negative": [ - "7", - 0 - ], - "latent_image": [ - "10", - 0 - ] + "model": ["16", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["10", 0] }, "class_type": "KSampler" }, "12": { "inputs": { "filename_prefix": "ComfyUI", - "images": [ - "13", - 0 - ] + "images": ["13", 0] }, "class_type": "SaveImage" }, "13": { "inputs": { - "samples": [ - "11", - 0 - ], - "vae": [ - "16", - 2 - ] + "samples": ["11", 0], + "vae": ["16", 2] }, "class_type": "VAEDecode" }, @@ -146,4 +95,4 @@ }, "class_type": "CheckpointLoaderSimple" } -} \ No newline at end of file +} diff --git a/typescripts/comfyui/img2img_workflow.json b/typescripts/comfyui/img2img_workflow.json new file mode 100644 index 00000000..fd644217 --- /dev/null +++ b/typescripts/comfyui/img2img_workflow.json @@ -0,0 +1,65 @@ +{ + "3": { + "inputs": { + "seed": 280823642470253, + "steps": 20, + "cfg": 8, + "sampler_name": "dpmpp_2m", + "scheduler": "normal", + "denoise": 0.8700000000000001, + "model": ["14", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["12", 0] + }, + "class_type": "KSampler" + }, + "6": { + "inputs": { + "text": "photograph of victorian woman with wings, sky clouds, meadow grass\n", + "clip": ["14", 1] + }, + "class_type": "CLIPTextEncode" + }, + "7": { + "inputs": { + "text": "watermark, text\n", + "clip": ["14", 1] + }, + "class_type": "CLIPTextEncode" + }, + "8": { + "inputs": { + "samples": ["3", 0], + "vae": ["14", 2] + }, + "class_type": "VAEDecode" + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["8", 0] + }, + "class_type": "SaveImage" + }, + "10": { + "inputs": { + "image": "example.png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "12": { + "inputs": { + "pixels": ["10", 0], + "vae": ["14", 2] + }, + "class_type": "VAEEncode" + }, + "14": { + "inputs": { + "ckpt_name": "v1-5-pruned-emaonly.ckpt" + }, + "class_type": "CheckpointLoaderSimple" + } +} diff --git a/typescripts/comfyui/lora_less_workflow.json b/typescripts/comfyui/lora_less_workflow.json new file mode 100644 index 00000000..cbf12ae4 --- /dev/null +++ b/typescripts/comfyui/lora_less_workflow.json @@ -0,0 +1,187 @@ +{ + "8": { + "inputs": { + "vae_name": "klF8Anime2VAE_klF8Anime2VAE.ckpt" + }, + "class_type": "VAELoader" + }, + "16": { + "inputs": { + "ckpt_name": "juggernaut_final.safetensors" + }, + "class_type": "CheckpointLoaderSimple" + }, + "18": { + "inputs": { + "text": "(front view:1.2)", + "clip": ["16", 1] + }, + "class_type": "CLIPTextEncode" + }, + "21": { + "inputs": { + "width": 512, + "height": 512, + "batch_size": 2 + }, + "class_type": "EmptyLatentImage" + }, + "30": { + "inputs": { + "text": "(a dog sitting:1.3) on a tile floor with a blue eyes and a white nose and tail, looking at the camera, artist, extremely detailed oil painting, a photorealistic painting, photorealism", + "clip": ["16", 1] + }, + "class_type": "CLIPTextEncode" + }, + "31": { + "inputs": { + "tile_size": 512, + "samples": ["97", 0], + "vae": ["8", 0] + }, + "class_type": "VAEDecodeTiled" + }, + "48": { + "inputs": { + "weight": ["163", 0], + "model_name": "ip-adapter-plus_sd15.bin", + "dtype": "fp32", + "model": ["162", 0], + "image": ["168", 0], + "clip_vision": ["57", 0] + }, + "class_type": "IPAdapter" + }, + "50": { + "inputs": { + "strength": ["163", 0], + "noise_augmentation": 0, + "conditioning": ["30", 0], + "clip_vision_output": ["48", 1] + }, + "class_type": "unCLIPConditioning" + }, + "57": { + "inputs": { + "clip_name": "model.safetensors" + }, + "class_type": "CLIPVisionLoader" + }, + "97": { + "inputs": { + "seed": 229741993160779, + "steps": 32, + "cfg": 6.5, + "sampler_name": "dpmpp_2s_ancestral", + "scheduler": "karras", + "denoise": 1, + "model": ["48", 0], + "positive": ["50", 0], + "negative": ["18", 0], + "latent_image": ["21", 0] + }, + "class_type": "KSampler" + }, + "122": { + "inputs": { + "images": ["31", 0] + }, + "class_type": "PreviewImage" + }, + "155": { + "inputs": { + "seed": 216203953003378, + "steps": 40, + "cfg": 5, + "sampler_name": "ddim", + "scheduler": "normal", + "denoise": 0.45, + "model": ["48", 0], + "positive": ["50", 0], + "negative": ["18", 0], + "latent_image": ["156", 0] + }, + "class_type": "KSampler" + }, + "156": { + "inputs": { + "tile_size": 640, + "pixels": ["159", 0], + "vae": ["8", 0] + }, + "class_type": "VAEEncodeTiled" + }, + "157": { + "inputs": { + "upscale_model": ["158", 0], + "image": ["31", 0] + }, + "class_type": "ImageUpscaleWithModel" + }, + "158": { + "inputs": { + "model_name": "RealESRGAN_x4plus_anime_6B.pth" + }, + "class_type": "UpscaleModelLoader" + }, + "159": { + "inputs": { + "upscale_method": "nearest-exact", + "scale_by": 0.45, + "image": ["157", 0] + }, + "class_type": "ImageScaleBy" + }, + "160": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": ["161", 0] + }, + "class_type": "SaveImage" + }, + "161": { + "inputs": { + "tile_size": 512, + "samples": ["155", 0], + "vae": ["8", 0] + }, + "class_type": "VAEDecodeTiled" + }, + "162": { + "inputs": { + "b1": 1.1500000000000001, + "b2": 1.35, + "s1": 0.9500000000000001, + "s2": 0.18, + "model": ["16", 0] + }, + "class_type": "FreeU" + }, + "163": { + "inputs": { + "Value": 0.5 + }, + "class_type": "Float" + }, + "168": { + "inputs": { + "image": "Layer 4 (3).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "169": { + "inputs": { + "image": "Layer 3 (1).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + }, + "188": { + "inputs": { + "image": "01285-3246154361-a photo of charddim15 person looking happy, beautiful, ((cloth)), ((full body)), ((chest)), ((far away)), ((waist up)).png", + "choose file to upload": "image" + }, + "class_type": "LoadImage" + } +} diff --git a/typescripts/comfyui/util.ts b/typescripts/comfyui/util.ts new file mode 100644 index 00000000..d1970864 --- /dev/null +++ b/typescripts/comfyui/util.ts @@ -0,0 +1,356 @@ +import hi_res_workflow from './hi_res_workflow.json' +import img2img_workflow from './img2img_workflow.json' +import animatediff_workflow from './animatediff_workflow.json' +import lora_less_workflow from './lora_less_workflow.json' +import { diffusion_chain } from '../entry' +import { ComfyPrompt } from 'diffusion-chain/dist/backends/comfyui-api.mjs' +// import { ComfyPrompt } from 'diffusion-chain/dist/backends/comfyui-api.mjs' +// import { ComfyPrompt } from 'diffusion-chain/' +export function getWorkflow() {} + +interface Workflow {} +export function getNodes(workflow: Workflow) { + // Object.values(workflow).forEach((node) => { + // console.log(node.class_type) + // }) + return Object.entries(workflow) +} + +export enum ComfyInputType { + TextField = 'TextField', + TextArea = 'TextArea', + Menu = 'Menu', + Number = 'Number', + Slider = 'Slider', + BigNumber = 'BigNumber', + TextFieldNumber = 'TextFieldNumber', + Skip = 'Skip', +} +export enum ComfyNodeType { + LoadImage = 'LoadImage', + Normal = 'Normal', + Skip = 'Skip', +} + +interface ComfyOutputImage { + filename: string + subfolder: string + type: string +} + +export function getNodeType(node_name: any) { + let node_type: ComfyNodeType = ComfyNodeType.Normal + switch (node_name) { + case 'LoadImage': + node_type = ComfyNodeType.LoadImage + break + default: + break + } + return node_type +} +export function parseComfyInput(input_info: any): { + type: ComfyInputType + config: any +} { + const value = input_info[0] + + let input_type: ComfyInputType = ComfyInputType.Skip + let input_config + + if (typeof value === 'string') { + if (value === 'FLOAT') { + input_type = ComfyInputType.Slider + input_config = input_info[1] + } else if (value === 'INT') { + if (input_info[1].max > Number.MAX_SAFE_INTEGER) { + input_type = ComfyInputType.BigNumber + input_config = input_info[1] + } else { + input_type = ComfyInputType.TextFieldNumber + input_config = input_info[1] + } + } else if (value === 'STRING') { + if (input_info[1]?.multiline) { + input_type = ComfyInputType.TextArea + input_config = input_info[1] + } else { + input_type = ComfyInputType.TextField + input_config = input_info[1] + } + } + } else if (Array.isArray(value)) { + input_type = ComfyInputType.Menu + input_config = value + } + + return { type: input_type, config: input_config } +} + +export function makeHtmlInput() {} + +export function nodeToUIConfig( + node: { inputs: { [key: string]: any }; class_type: string }, + object_info: any +) { + let comfy_node_info = object_info[node.class_type] + let node_ui_config = Object.entries(node.inputs).map( + ([name, value]: [string, any]) => { + const first_value = comfy_node_info[name][0] + let { type, config } = parseComfyInput(first_value) + + return + } + ) + // comfy_node_info.input.required[] +} + +async function getHistory(comfy_server: diffusion_chain.ComfyServer) { + while (true) { + const res = await diffusion_chain.ComfyApi.queue(comfy_server) + if (res.queue_pending.length || res.queue_running.length) { + await new Promise((resolve) => setTimeout(resolve, 500)) + } else { + break + } + await new Promise((resolve) => setTimeout(resolve, 500)) + } + const history = await diffusion_chain.ComfyApi.history(comfy_server) + return history +} +export async function postPromptAndGetBase64JsonResult( + comfy_server: diffusion_chain.ComfyServer, + prompt: Record +) { + try { + const res = await diffusion_chain.ComfyApi.prompt(comfy_server, { + prompt, + } as ComfyPrompt) + if (res.error) { + const readable_error = comfy_server.getReadableError(res) + throw new Error(readable_error) + } + const prompt_id = res.prompt_id + const history = await getHistory(comfy_server) + const promptInfo = history[prompt_id] + const store_output = await mapComfyOutputToStoreOutput( + comfy_server, + promptInfo.outputs + ) + // // [4][0] for output id. + // const fileName = promptInfo.outputs[promptInfo.prompt[4][0]].images[0].filename + // const resultB64 = await ComfyApi.view(this, fileName); + // resultImages.push(resultB64) + // if (option.imageFinishCallback) { + // try { option.imageFinishCallback(resultB64, index) } catch (e) { } + // } + // } + return store_output + } catch (e) { + console.error(e) + } +} +export const getFileFormat = (fileName: string): string => + fileName.includes('.') ? fileName.split('.').pop()! : '' + +export async function base64UrlFromComfy( + comfy_server: diffusion_chain.ComfyServer, + { filename, type, subfolder }: ComfyOutputImage +) { + const base64 = await diffusion_chain.ComfyApi.view( + comfy_server, + filename, + type, + subfolder + ) + return base64Url(base64, getFileFormat(filename)) +} +export function base64UrlFromFileName(base64: string, filename: string) { + return base64Url(base64, getFileFormat(filename)) +} +export function base64Url(base64: string, format: string = 'png') { + return `data:image/${format};base64,${base64}` +} +export function generatePrompt(prompt: Record) { + prompt +} +export function updateOutput(output: any, output_store_obj: any) { + // store.data.current_prompt2_output[26] = [image, image] + output_store_obj = output +} + +export async function mapComfyOutputToStoreOutput( + comfy_server: diffusion_chain.ComfyServer, + comfy_output: Record +) { + // const comfy_output: Record = { + // '12': { + // images: [ + // { + // filename: 'AA_readme_00506_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00507_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00508_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00509_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00510_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00511_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00512_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00513_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00514_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00515_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00516_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00517_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00518_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00519_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00520_.png', + // subfolder: '', + // type: 'output', + // }, + // { + // filename: 'AA_readme_00521_.png', + // subfolder: '', + // type: 'output', + // }, + // ], + // }, + // '26': { + // images: [ + // { + // filename: 'AA_readme_gif_00079_.gif', + // subfolder: '', + // type: 'output', + // }, + // ], + // }, + // } + + const store_output: Record = {} + for (let key in comfy_output) { + if (comfy_output[key].hasOwnProperty('images')) { + let base64_url_list = await Promise.all( + comfy_output[key].images.map( + async (image: ComfyOutputImage) => + await base64UrlFromComfy(comfy_server, image) + ) + ) + store_output[key] = base64_url_list + } + } + return store_output +} + +interface LooseObject { + [key: string]: any +} + +function isSameStructure(obj1: LooseObject, obj2: LooseObject): boolean { + // Get keys + const keys1 = Object.keys(obj1) + const keys2 = Object.keys(obj2) + + // Check if both objects have the same number of keys + if (keys1.length !== keys2.length) { + return false + } + + // Check if all keys in obj1 exist in obj2 and have the same structure + for (let i = 0; i < keys1.length; i++) { + const key = keys1[i] + + // Check if the key exists in obj2 + if (!obj2.hasOwnProperty(key)) { + return false + } + + // If the value of this key is an object, check their structure recursively + if ( + typeof obj1[key] === 'object' && + obj1[key] !== null && + typeof obj2[key] === 'object' && + obj2[key] !== null + ) { + if (!isSameStructure(obj1[key], obj2[key])) { + return false + } + } + } + + // If all checks passed, the structures are the same + return true +} + +export default { + getNodes, + parseComfyInput, + getNodeType, + base64Url, + getFileFormat, + base64UrlFromComfy, + generatePrompt, + updateOutput, + getHistory, + mapComfyOutputToStoreOutput, + postPromptAndGetBase64JsonResult, + isSameStructure, + hi_res_workflow, + img2img_workflow, + animatediff_workflow, + lora_less_workflow, + ComfyInputType, + ComfyNodeType, +} diff --git a/typescripts/entry.ts b/typescripts/entry.ts index e9b152b6..b25e7608 100644 --- a/typescripts/entry.ts +++ b/typescripts/entry.ts @@ -40,3 +40,6 @@ export * as api_ts from './util/ts/api' export * as comfyui from './comfyui/comfyui' export { toJS } from 'mobx' export { default as node_fs } from 'fs' +export { default as comfyui_util } from './comfyui/util' + +export * as diffusion_chain from 'diffusion-chain' From 1c81da23589467ad46fcc73fbb7d9d1d6accacad Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sun, 22 Oct 2023 03:21:53 +0300 Subject: [PATCH 08/97] target es2020 to have support for bigint literal --- typescripts/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescripts/tsconfig.json b/typescripts/tsconfig.json index 49d3e7ba..e2a9531c 100644 --- a/typescripts/tsconfig.json +++ b/typescripts/tsconfig.json @@ -11,7 +11,7 @@ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ - "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "jsx": "react", /* Specify what JSX code is generated. */ "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ From 651dcbcd3a8bb3f3cf007f26787ccd64ac41627e Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sun, 22 Oct 2023 03:24:07 +0300 Subject: [PATCH 09/97] add seed as an input type and allow for it be randomized --- typescripts/comfyui/comfyui.tsx | 74 ++++++++++++++++++++++++++++++++- typescripts/comfyui/util.ts | 13 ++++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index b3a2686f..a45e29b8 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -371,6 +371,7 @@ export const store = new AStore({ animatediff_workflow: util.animatediff_workflow, } as Record, progress_value: 0, + is_random_seed: {} as Record, }) export function storeToPrompt(store: any, basePrompt: any) { @@ -510,6 +511,13 @@ export async function loadWorkflow(workflow_path: string) { await getWorkflowApi(workflow_path) )?.prompt + store.data.is_random_seed = Object.fromEntries( + Object.keys(toJS(store.data.current_prompt)).map( + (node_id: string) => { + return [node_id, false] + } + ) + ) const current_prompt: any = store.toJsFunc().data.current_prompt const loadImageNodes = Object.keys(current_prompt) .filter( @@ -864,7 +872,7 @@ function renderNode(node_id: string, node: any) { // store.data.current_prompt2[node_id].inputs[name] = value try { const input = comfy_node_info.input.required[name] - let { type, config } = util.parseComfyInput(input) + let { type, config } = util.parseComfyInput(name, input) const html_element = renderInput( node_id, name, @@ -952,7 +960,37 @@ function renderInput(
) const inputs = store.data.current_prompt2[node_id].inputs - if (type === util.ComfyInputType.BigNumber) { + + if (type === util.ComfyInputType.Seed) { + html_element = ( + <> + {name}: + { + // store.data.search_query = event.target.value + inputs[name] = event.target.value + console.log(`${name}: ${event.target.value}`) + }} + > + { + store.data.is_random_seed[node_id] = evt.target.checked + }} + style={{ display: 'inline-flex' }} + > + random + + + ) + } else if (type === util.ComfyInputType.BigNumber) { html_element = ( <> {name}: @@ -1277,6 +1315,38 @@ class ComfyWorkflowComponent extends React.Component<{}, { value?: number }> { try { // Start the progress update + function runScript() { + function getRandomBigIntApprox( + min: bigint, + max: bigint + ): bigint { + min = BigInt(min) + max = BigInt(max) + const range = Number(max - min) + const rand = Math.floor( + Math.random() * range + ) + return BigInt(rand) + min + } + + Object.entries( + toJS(store.data.is_random_seed) + ).forEach(([node_id, is_random]) => { + if (is_random) { + const min: bigint = 0n + const max: bigint = + 18446744073709552000n + const random_seed: bigint = + getRandomBigIntApprox(min, max) + store.data.current_prompt2[ + node_id + ].inputs['seed'] = + random_seed.toString() + // Usage + } + }) + } + runScript() let store_output = await util.postPromptAndGetBase64JsonResult( comfy_server, diff --git a/typescripts/comfyui/util.ts b/typescripts/comfyui/util.ts index d1970864..fa73131b 100644 --- a/typescripts/comfyui/util.ts +++ b/typescripts/comfyui/util.ts @@ -25,6 +25,7 @@ export enum ComfyInputType { BigNumber = 'BigNumber', TextFieldNumber = 'TextFieldNumber', Skip = 'Skip', + Seed = 'Seed', } export enum ComfyNodeType { LoadImage = 'LoadImage', @@ -49,7 +50,10 @@ export function getNodeType(node_name: any) { } return node_type } -export function parseComfyInput(input_info: any): { +export function parseComfyInput( + name: string, + input_info: any +): { type: ComfyInputType config: any } { @@ -58,7 +62,10 @@ export function parseComfyInput(input_info: any): { let input_type: ComfyInputType = ComfyInputType.Skip let input_config - if (typeof value === 'string') { + if (name === 'seed') { + input_type = ComfyInputType.Seed // similar to big number + input_config = input_info[1] + } else if (typeof value === 'string') { if (value === 'FLOAT') { input_type = ComfyInputType.Slider input_config = input_info[1] @@ -97,7 +104,7 @@ export function nodeToUIConfig( let node_ui_config = Object.entries(node.inputs).map( ([name, value]: [string, any]) => { const first_value = comfy_node_info[name][0] - let { type, config } = parseComfyInput(first_value) + let { type, config } = parseComfyInput(name, first_value) return } From 899da1e4c32f64e251116a276f6df8e5688f702b Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sun, 22 Oct 2023 03:24:58 +0300 Subject: [PATCH 10/97] add filename prefix to output nodes --- typescripts/comfyui/comfyui.tsx | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index a45e29b8..9840e41b 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -889,6 +889,39 @@ function renderNode(node_id: string, node: any) { if (is_output) { const output_node_element = (
+ <> + {'filename_prefix' in + store.data.current_prompt2[node_id].inputs ? ( + <> + + filename prefix: + + { + try { + store.data.current_prompt2[ + node_id + ].inputs['filename_prefix'] = + event.target.value + } catch (e) { + console.warn(e) + } + }} + placeholder={`filename_prefix`} + value={ + store.data.current_prompt2[node_id].inputs[ + 'filename_prefix' + ] + } + > + + ) : ( + void 0 + )} + Date: Sun, 22 Oct 2023 04:09:14 +0300 Subject: [PATCH 11/97] color code the last move node in edit mode --- typescripts/comfyui/comfyui.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/typescripts/comfyui/comfyui.tsx b/typescripts/comfyui/comfyui.tsx index 9840e41b..ab6cba68 100644 --- a/typescripts/comfyui/comfyui.tsx +++ b/typescripts/comfyui/comfyui.tsx @@ -372,6 +372,7 @@ export const store = new AStore({ } as Record, progress_value: 0, is_random_seed: {} as Record, + last_moved: undefined as string | undefined, // the last node that has been moved in the edit mode }) export function storeToPrompt(store: any, basePrompt: any) { @@ -927,7 +928,7 @@ function renderNode(node_id: string, node: any) { style={{ display: 'block' }} show-value="false" id="slUpscaleSize" - min="100" + min="25" max="300" value={store.data.output_thumbnail_image_size[node_id]} title="" @@ -1400,6 +1401,7 @@ class ComfyWorkflowComponent extends React.Component<{}, { value?: number }> { +
+ ) } - return this.props.children + // The key prop causes a remount of children when it changes + return
{this.props.children}
} } From 71d3dcd39958ed109add61d5a8fc765555385547 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sat, 4 Nov 2023 15:54:53 +0300 Subject: [PATCH 28/97] Fix bug: Ensure thumbnail click event triggers only once --- typescripts/util/grid.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/typescripts/util/grid.tsx b/typescripts/util/grid.tsx index dcc01fcf..51902a9b 100644 --- a/typescripts/util/grid.tsx +++ b/typescripts/util/grid.tsx @@ -58,7 +58,6 @@ export class Grid extends React.Component<{ event ) } - this.props?.callback(index, event) //todo: is this a typo why do we call callback twice? } } catch (e) { console.warn( From ec0f257125f2f02d1e18b9b47f59ca10a3353db4 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:01:05 +0300 Subject: [PATCH 29/97] rename file main.tsx to vae.tsx --- .../{main/main.tsx => settings/vae.tsx} | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) rename typescripts/{main/main.tsx => settings/vae.tsx} (86%) diff --git a/typescripts/main/main.tsx b/typescripts/settings/vae.tsx similarity index 86% rename from typescripts/main/main.tsx rename to typescripts/settings/vae.tsx index ed6f6ce7..3d4559c1 100644 --- a/typescripts/main/main.tsx +++ b/typescripts/settings/vae.tsx @@ -1,7 +1,7 @@ import React from 'react' import ReactDOM from 'react-dom/client' import { observer } from 'mobx-react' -import { AStore } from './astore' +import { AStore } from '../main/astore' import { SpMenu } from '../util/elements' import { api, python_replacement } from '../util/oldSystem' @@ -10,27 +10,11 @@ import '../locale/locale-for-old-html' import { ErrorBoundary } from '../util/errorBoundary' declare let g_sd_url: string -// class SDStore extends AStore { -// constructor(data: any) { -// super(data) -// } -// } -// const configValues = Object.entries(ui_config).reduce( -// (acc, [key, value]) => ({ ...acc, [key]: value.value }), -// {} -// ) -// const default_values: any = { -// _: '', -// ...configValues, -// } - -const default_values: any = { - vae_model_list: [], - current_vae: '', -} - -export const store = new AStore(default_values) +export const store = new AStore({ + vae_model_list: [] as string[], + current_vae: '' as string, +}) @observer export class VAEComponent extends React.Component<{ @@ -110,3 +94,8 @@ vaeRoot.render( // ) + +export default { + store, + populateVAE, +} From 747a006813f250536b23ffe2024e54670a57c9a4 Mon Sep 17 00:00:00 2001 From: Abdullah Alfaraj <7842232+AbdullahAlfaraj@users.noreply.github.com> Date: Sat, 4 Nov 2023 16:32:40 +0300 Subject: [PATCH 30/97] Ensure models always contain array of model names --- typescripts/sd_tab/sd_tab.tsx | 19 ++++--------------- typescripts/sd_tab/util.ts | 7 +++++-- typescripts/util/ts/sdapi.ts | 6 ++++-- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/typescripts/sd_tab/sd_tab.tsx b/typescripts/sd_tab/sd_tab.tsx index bd6b0af4..89962129 100644 --- a/typescripts/sd_tab/sd_tab.tsx +++ b/typescripts/sd_tab/sd_tab.tsx @@ -148,7 +148,6 @@ class SDTab extends React.Component<{}> { async componentDidMount() { try { await refreshUI() - await refreshModels() await initPlugin() helper_store.data.loras = await requestLoraModels() initInitMaskElement() @@ -202,28 +201,18 @@ class SDTab extends React.Component<{}> {