diff --git a/README.md b/README.md index c9fb729..223b85c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ # ComfyInject +** This is just a minor modification of https://github.com/Spadic21/ComfyInject to introduce: + +- Placeholders for CHAR_AVATAR and USER_AVATAR - I just didn't get what I was looking for, so now optional placeholders can be used (note you need to include those properly in your workflow). +- A minor change so the ComfyInject model can be chosen via select, instead of a textfield. + +And that's it ** + +************************************************************************************************************* + A SillyTavern extension that automatically generates images from `[[IMG: ... ]]` markers in bot messages using your local ComfyUI instance. When your LLM outputs a marker, ComfyInject intercepts it, sends the prompt to ComfyUI, and replaces the marker with the generated image, all without leaving the chat. Multiple images per message are supported. Images are saved permanently into the chat history and survive page reloads. Outbound prompts sent to the LLM replace injected images with a compact token so the model maintains visual continuity across the conversation. diff --git a/scripts/generate-workflow-index.cjs b/scripts/generate-workflow-index.cjs new file mode 100644 index 0000000..c044070 --- /dev/null +++ b/scripts/generate-workflow-index.cjs @@ -0,0 +1,10 @@ +const fs = require('fs'); +const path = require('path'); + +const workflowsDir = path.join(__dirname, '..', 'workflows'); +const files = fs.readdirSync(workflowsDir) + .filter(f => f.endsWith('.json') && f !== 'index.json') + .sort(); + +fs.writeFileSync(path.join(workflowsDir, 'index.json'), JSON.stringify(files, null, 2) + '\n'); +console.log(`Generated workflows/index.json with ${files.length} workflow(s):\n ${files.join('\n ')}`); diff --git a/settings.html b/settings.html index 9fbc60d..c85c8bd 100644 --- a/settings.html +++ b/settings.html @@ -13,26 +13,19 @@ -
+
-
-
- - -
- -
+
-
+
- Filename of the workflow JSON in the workflows folder. - +
diff --git a/src/comfy.js b/src/comfy.js index 9bf58a5..cb12836 100644 --- a/src/comfy.js +++ b/src/comfy.js @@ -1,5 +1,12 @@ import { MODULE_NAME } from "../settings.js"; import { resolveSeed } from "./state.js"; +import { + formatCharacterAvatar, + getCharacterAvatar, + user_avatar, + getUserAvatar, +} from '../../../../../script.js'; +import { getBase64Async } from '../../../../utils.js'; const EXTENSION_FOLDER = `scripts/extensions/third-party/ComfyInject`; @@ -121,6 +128,69 @@ function buildImageUrl(filename, subfolder, host) { return `${host}/view?${params.toString()}`; } +/** + * Gets the active character's avatar URL (or group fallback). + */ +function getCharacterAvatarUrl() { + const context = SillyTavern.getContext(); + if (context.groupId) { + const groupMembers = context.groups.find(x => x.id === context.groupId)?.members; + const lastMessageAvatar = context.chat?.filter(x => !x.is_system && !x.is_user)?.slice(-1)[0]?.original_avatar; + const randomMemberAvatar = Array.isArray(groupMembers) ? groupMembers[Math.floor(Math.random() * groupMembers.length)] : null; + const avatarToUse = lastMessageAvatar || randomMemberAvatar; + return formatCharacterAvatar(avatarToUse); + } else { + return getCharacterAvatar(context.characterId); + } +} + +/** + * Gets the active user's avatar URL. + */ +function getUserAvatarUrl() { + return getUserAvatar(user_avatar); +} + +/** + * Fetch avatar image and return as base64 string. + * @param {string} url - URL of the avatar to fetch + * @returns {Promise} Base64-encoded image data + */ +async function fetchAvatarAsBase64(url) { + if (!url) return ""; + const response = await fetch(url); + if (!response.ok) return ""; + const blob = await response.blob(); + const dataUrl = await getBase64Async(blob); + return dataUrl.split(",")[1] || ""; +} + +/** + * Fetch character avatar for {{CHAR_AVATAR}} placeholder. + * @returns {Promise} + */ +async function fetchCharacterAvatar() { + try { + return await fetchAvatarAsBase64(getCharacterAvatarUrl()); + } catch (e) { + console.warn("[ComfyInject] Error fetching character avatar:", e); + return ""; + } +} + +/** + * Fetch user avatar for {{USER_AVATAR}} placeholder. + * @returns {Promise} + */ +async function fetchUserAvatar() { + try { + return await fetchAvatarAsBase64(getUserAvatarUrl()); + } catch (e) { + console.warn("[ComfyInject] Error fetching user avatar:", e); + return ""; + } +} + /** * Main entry point. Takes parsed marker data and returns a usable image URL. * @param {object} params @@ -159,8 +229,13 @@ export async function generateImage({ prompt, ar, shot, seed, messageIndex, bypa ? resolveSeed(settings.seed_lock_mode === "CUSTOM" ? settings.seed_lock_value : settings.seed_lock_mode, messageIndex) : seed; - // Load and fill the workflow - const workflow = await loadWorkflow(); + // Load workflow and avatars in parallel + const [workflow, charAvatar, userAvatar] = await Promise.all([ + loadWorkflow(), + fetchCharacterAvatar().catch(() => ""), + fetchUserAvatar().catch(() => ""), + ]); + const filled = fillWorkflow(workflow, { CHECKPOINT: settings.checkpoint, POSITIVE_PROMPT: positivePrompt, @@ -173,6 +248,8 @@ export async function generateImage({ prompt, ar, shot, seed, messageIndex, bypa SAMPLER: settings.sampler, SCHEDULER: settings.scheduler, DENOISE: settings.denoise, + CHAR_AVATAR: charAvatar, + USER_AVATAR: userAvatar, }); // Submit to ComfyUI and wait for the result diff --git a/src/ui.js b/src/ui.js index 81f6df3..3cb8543 100644 --- a/src/ui.js +++ b/src/ui.js @@ -31,52 +31,55 @@ async function fetchCheckpoints() { if (!response.ok) return []; const data = await response.json(); return data?.CheckpointLoaderSimple?.input?.required?.ckpt_name?.[0] ?? []; - } catch (err) { + } catch { return []; } } /** - * Validates that a workflow file exists in the workflows folder. - * Shows a toastr error if the file doesn't exist, success if it does. - * @param {string} filename - The workflow filename to validate + * Populates the checkpoint . */ -async function refreshCheckpointList(showToast = true) { - const checkpoints = await fetchCheckpoints(); - const dropdown = $("#comfyinject_checkpoint_dropdown"); - dropdown.empty(); +async function populateWorkflows() { + const select = $("#comfyinject_workflow"); + const current = getSettings().workflow; - if (checkpoints.length > 0) { - const current = getSettings().checkpoint; - for (const name of checkpoints) { - dropdown.append( - `
${name}
` - ); + try { + const response = await fetch(`/${EXTENSION_FOLDER}/workflows/index.json`); + const workflows = await response.json(); + select.empty(); + for (const name of workflows) { + select.append(``); } - if (showToast) { - toastr.success(`Found ${checkpoints.length} checkpoint(s)`, "ComfyInject"); + if (!workflows.includes(current) && current) { + select.append(``); } - } else if (showToast) { - toastr.warning("Could not reach ComfyUI. Is it running?", "ComfyInject"); + } catch { + select.empty(); + select.append(``); } } @@ -220,61 +223,16 @@ function wireEvents() { saveSettings(); }); - // Checkpoint — text input - $("#comfyinject_checkpoint").on("input", function () { + // Checkpoint — select + $("#comfyinject_checkpoint").on("change", function () { getSettings().checkpoint = $(this).val(); saveSettings(); }); - // Checkpoint — arrow button toggles dropdown - $("#comfyinject_checkpoint_arrow").on("click", function () { - const dropdown = $("#comfyinject_checkpoint_dropdown"); - if (dropdown.children().length === 0) { - // No checkpoints fetched yet — trigger a fetch - refreshCheckpointList(true).then(() => { - if ($("#comfyinject_checkpoint_dropdown").children().length > 0) { - dropdown.show(); - } - }); - } else { - dropdown.toggle(); - } - }); - - // Checkpoint — clicking an option fills the text input and closes the dropdown - $("#comfyinject_checkpoint_dropdown").on("click", ".comfyinject-checkpoint-option", function () { - const value = $(this).data("value"); - $("#comfyinject_checkpoint").val(value); - getSettings().checkpoint = value; - saveSettings(); - $("#comfyinject_checkpoint_dropdown").hide(); - }); - - // Checkpoint — hover highlight - $("#comfyinject_checkpoint_dropdown").on("mouseenter", ".comfyinject-checkpoint-option", function () { - $(this).css("background", "var(--SmartThemeQuoteColor)"); - }).on("mouseleave", ".comfyinject-checkpoint-option", function () { - $(this).css("background", ""); - }); - - // Close dropdown when clicking outside - $(document).on("click", function (e) { - if (!$(e.target).closest("#comfyinject_checkpoint_arrow, #comfyinject_checkpoint_dropdown").length) { - $("#comfyinject_checkpoint_dropdown").hide(); - } - }); - - // Workflow — debounced validation after typing stops - let workflowValidateTimer = null; - $("#comfyinject_workflow").on("input", function () { + // Workflow — select + $("#comfyinject_workflow").on("change", function () { getSettings().workflow = $(this).val(); saveSettings(); - - // Debounce — validate 1.5s after the user stops typing - clearTimeout(workflowValidateTimer); - workflowValidateTimer = setTimeout(() => { - validateWorkflow($(this).val()); - }, 1500); }); // Negative prompt @@ -470,6 +428,7 @@ export async function initUI() { populateUI(); wireEvents(); - // Silently try to populate the checkpoint list on load — no toast if ComfyUI isn't running - refreshCheckpointList(false); + // Populate checkpoint and workflow dropdowns + populateCheckpoints(); + populateWorkflows(); } \ No newline at end of file diff --git a/workflows/comfyinject_avatar.json b/workflows/comfyinject_avatar.json new file mode 100644 index 0000000..0c6cf6d --- /dev/null +++ b/workflows/comfyinject_avatar.json @@ -0,0 +1,121 @@ +{ + "1": { + "inputs": { + "ckpt_name": "{{CHECKPOINT}}" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "2": { + "inputs": { + "text": "{{POSITIVE_PROMPT}}", + "clip": [ + "1", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "3": { + "inputs": { + "text": "{{NEGATIVE_PROMPT}}", + "clip": [ + "1", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "4": { + "inputs": { + "image": "{{CHAR_AVATAR}}" + }, + "class_type": "ETN_LoadImageBase64", + "_meta": { + "title": "Load Character Avatar" + } + }, + "8": { + "inputs": { + "pixels": [ + "4", + 0 + ], + "vae": [ + "1", + 2 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode (Avatar)" + } + }, + "5": { + "inputs": { + "seed": "{{SEED}}", + "steps": "{{STEPS}}", + "cfg": "{{CFG}}", + "sampler_name": "{{SAMPLER}}", + "scheduler": "{{SCHEDULER}}", + "denoise": "{{DENOISE}}", + "model": [ + "1", + 0 + ], + "positive": [ + "2", + 0 + ], + "negative": [ + "3", + 0 + ], + "latent_image": [ + "8", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "6": { + "inputs": { + "samples": [ + "5", + 0 + ], + "vae": [ + "1", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "7": { + "inputs": { + "filename_prefix": "ComfyInject", + "images": [ + "6", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +} diff --git a/workflows/comfyinject_avatar_facedetail.json b/workflows/comfyinject_avatar_facedetail.json new file mode 100644 index 0000000..903c0bc --- /dev/null +++ b/workflows/comfyinject_avatar_facedetail.json @@ -0,0 +1,281 @@ +{ + "1": { + "inputs": { + "cfg": "{{CFG}}", + "model": ["75", 0], + "positive": ["47", 0], + "negative": ["38", 0] + }, + "class_type": "CFGGuider", + "_meta": {"title": "CFG Guider"} + }, + "12": { + "inputs": { + "pixels": ["13", 0], + "vae": ["43", 2] + }, + "class_type": "VAEEncode", + "_meta": {"title": "VAE Encode"} + }, + "13": { + "inputs": { + "upscale_method": "bicubic", + "width": "{{WIDTH}}", + "height": "{{HEIGHT}}", + "crop": "center", + "image": ["70", 0] + }, + "class_type": "ImageScale", + "_meta": {"title": "Upscale Image"} + }, + "14": { + "inputs": { + "samples": ["15", 0], + "vae": ["43", 2] + }, + "class_type": "VAEDecode", + "_meta": {"title": "VAE Decode"} + }, + "15": { + "inputs": { + "noise": ["16", 0], + "guider": ["1", 0], + "sampler": ["45", 0], + "sigmas": ["44", 0], + "latent_image": ["12", 0] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": {"title": "SamplerCustomAdvanced"} + }, + "16": { + "inputs": { + "noise_seed": "{{SEED}}" + }, + "class_type": "RandomNoise", + "_meta": {"title": "RandomNoise"} + }, + "17": { + "inputs": {"model_name": "bbox/hand_yolov8s.pt"}, + "class_type": "UltralyticsDetectorProvider", + "_meta": {"title": "UltralyticsDetectorProvider (Hands 2)"} + }, + "24": { + "inputs": { + "guide_size": 256, + "guide_size_for": true, + "max_size": 768, + "seed": "{{SEED}}", + "steps": "{{STEPS}}", + "cfg": "{{CFG}}", + "sampler_name": "{{SAMPLER}}", + "scheduler": "{{SCHEDULER}}", + "denoise": 0.35, + "feather": 5, + "noise_mask": true, + "noise_mask_feather": 20, + "inpaint_model": false, + "bbox_threshold": 0.5, + "bbox_dilation": 10, + "bbox_crop_factor": 3.0, + "sam_detection_hint": "center-1", + "sam_dilation": 0, + "sam_threshold": 0.93, + "sam_bbox_expansion": 0, + "sam_mask_hint_threshold": 0.7, + "sam_mask_hint_use_negative": "False", + "drop_size": 10, + "wildcard": "", + "cycle": 1, + "force_inpaint": true, + "image": ["14", 0], + "model": ["75", 0], + "clip": ["43", 1], + "vae": ["43", 2], + "positive": ["47", 0], + "negative": ["38", 0], + "bbox_detector": ["61", 0], + "sam_model_opt": ["26", 0] + }, + "class_type": "FaceDetailer", + "_meta": {"title": "FaceDetailer (Faces)"} + }, + "25": { + "inputs": {"model_name": "bbox/hand_yolov8s.pt"}, + "class_type": "UltralyticsDetectorProvider", + "_meta": {"title": "UltralyticsDetectorProvider (Hands 1)"} + }, + "26": { + "inputs": {"model_name": "sam_vit_b_01ec64.pth", "device_mode": "AUTO"}, + "class_type": "SAMLoader", + "_meta": {"title": "SAMLoader"} + }, + "28": { + "inputs": { + "guide_size": 256, + "guide_size_for": true, + "max_size": 768, + "seed": "{{SEED}}", + "steps": "{{STEPS}}", + "cfg": "{{CFG}}", + "sampler_name": "{{SAMPLER}}", + "scheduler": "{{SCHEDULER}}", + "denoise": 0.35, + "feather": 5, + "noise_mask": true, + "noise_mask_feather": 20, + "inpaint_model": false, + "bbox_threshold": 0.5, + "bbox_dilation": 10, + "bbox_crop_factor": 3.0, + "sam_detection_hint": "center-1", + "sam_dilation": 0, + "sam_threshold": 0.93, + "sam_bbox_expansion": 0, + "sam_mask_hint_threshold": 0.7, + "sam_mask_hint_use_negative": "False", + "drop_size": 10, + "wildcard": "", + "cycle": 1, + "force_inpaint": true, + "image": ["24", 0], + "model": ["75", 0], + "clip": ["43", 1], + "vae": ["43", 2], + "positive": ["47", 0], + "negative": ["38", 0], + "bbox_detector": ["25", 0], + "sam_model_opt": ["26", 0] + }, + "class_type": "FaceDetailer", + "_meta": {"title": "FaceDetailer (Hands 1)"} + }, + "29": { + "inputs": { + "guide_size": 256, + "guide_size_for": true, + "max_size": 768, + "seed": "{{SEED}}", + "steps": "{{STEPS}}", + "cfg": "{{CFG}}", + "sampler_name": "{{SAMPLER}}", + "scheduler": "{{SCHEDULER}}", + "denoise": 0.35, + "feather": 5, + "noise_mask": true, + "noise_mask_feather": 20, + "inpaint_model": false, + "bbox_threshold": 0.5, + "bbox_dilation": 10, + "bbox_crop_factor": 3.0, + "sam_detection_hint": "center-1", + "sam_dilation": 0, + "sam_threshold": 0.93, + "sam_bbox_expansion": 0, + "sam_mask_hint_threshold": 0.7, + "sam_mask_hint_use_negative": "False", + "drop_size": 10, + "wildcard": "", + "cycle": 1, + "force_inpaint": true, + "image": ["28", 0], + "model": ["75", 0], + "clip": ["43", 1], + "vae": ["43", 2], + "positive": ["47", 0], + "negative": ["38", 0], + "bbox_detector": ["17", 0], + "sam_model_opt": ["26", 0] + }, + "class_type": "FaceDetailer", + "_meta": {"title": "FaceDetailer (Hands 2)"} + }, + "38": { + "inputs": { + "text": "{{NEGATIVE_PROMPT}}", + "clip": ["43", 1] + }, + "class_type": "CLIPTextEncode", + "_meta": {"title": "CLIP Text Encode (Negative)"} + }, + "43": { + "inputs": {"ckpt_name": "{{CHECKPOINT}}"}, + "class_type": "CheckpointLoaderSimple", + "_meta": {"title": "Load Checkpoint - BASE"} + }, + "44": { + "inputs": { + "scheduler": "{{SCHEDULER}}", + "steps": "{{STEPS}}", + "denoise": "{{DENOISE}}", + "model": ["75", 0] + }, + "class_type": "BasicScheduler", + "_meta": {"title": "BasicScheduler"} + }, + "45": { + "inputs": {"sampler_name": "{{SAMPLER}}"}, + "class_type": "KSamplerSelect", + "_meta": {"title": "KSamplerSelect"} + }, + "47": { + "inputs": { + "text": "{{POSITIVE_PROMPT}}", + "clip": ["43", 1] + }, + "class_type": "CLIPTextEncode", + "_meta": {"title": "CLIP Text Encode (Positive)"} + }, + "60": { + "inputs": { + "filename_prefix": "SillyTavern\\SillyTavern", + "images": ["29", 0] + }, + "class_type": "SaveImage", + "_meta": {"title": "Save Image"} + }, + "61": { + "inputs": {"model_name": "bbox/face_yolov8m.pt"}, + "class_type": "UltralyticsDetectorProvider", + "_meta": {"title": "UltralyticsDetectorProvider (Face)"} + }, + "70": { + "inputs": {"image": "{{CHAR_AVATAR}}"}, + "class_type": "ETN_LoadImageBase64", + "_meta": {"title": "Load Image (Base64)"} + }, + "71": { + "inputs": {"ipadapter_file": "ip-adapter-faceid_sd15.bin"}, + "class_type": "IPAdapterModelLoader", + "_meta": {"title": "Load IPAdapter FaceID Model"} + }, + "73": { + "inputs": {"provider": "CUDA"}, + "class_type": "IPAdapterInsightFaceLoader", + "_meta": {"title": "IPAdapter InsightFace Loader"} + }, + "74": { + "inputs": { + "clip_name": "CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors" + }, + "class_type": "CLIPVisionLoader", + "_meta": {"title": "Load CLIP Vision"} + }, + "75": { + "inputs": { + "weight": 0.85, + "weight_faceidv2": 1.0, + "weight_type": "style transfer", + "combine_embeds": "concat", + "start_at": 0.0, + "end_at": 1.0, + "embeds_scaling": "V only", + "model": ["43", 0], + "ipadapter": ["71", 0], + "image": ["70", 0], + "insightface": ["73", 0], + "clip_vision": ["74", 0] + }, + "class_type": "IPAdapterFaceID", + "_meta": {"title": "IPAdapter FaceID"} + } +} \ No newline at end of file diff --git a/workflows/index.json b/workflows/index.json new file mode 100644 index 0000000..e5ae376 --- /dev/null +++ b/workflows/index.json @@ -0,0 +1,5 @@ +[ + "comfyinject_avatar.json", + "comfyinject_avatar_facedetail.json", + "comfyinject_default.json" +]