Add files using upload-large-folder tool

rgthree-comfy/web/comfyui/services/bookmarks_services.js

@@ -0,0 +1,10 @@
+import { app } from "../../../scripts/app.js";
+import { NodeTypesString } from "../constants.js";
+class BookmarksService {
+    getCurrentBookmarks() {
+        return app.graph._nodes
+            .filter((n) => n.type === NodeTypesString.BOOKMARK)
+            .sort((a, b) => a.title.localeCompare(b.title));
+    }
+}
+export const SERVICE = new BookmarksService();

rgthree-comfy/web/comfyui/services/config_service.js

@@ -0,0 +1,28 @@
+import { rgthreeConfig } from "../../../rgthree/config.js";
+import { getObjectValue, setObjectValue } from "../../../rgthree/common/shared_utils.js";
+import { rgthreeApi } from "../../../rgthree/common/rgthree_api.js";
+class ConfigService extends EventTarget {
+    getConfigValue(key, def) {
+        return getObjectValue(rgthreeConfig, key, def);
+    }
+    getFeatureValue(key, def) {
+        key = "features." + key.replace(/^features\./, "");
+        return getObjectValue(rgthreeConfig, key, def);
+    }
+    async setConfigValues(changed) {
+        const body = new FormData();
+        body.append("json", JSON.stringify(changed));
+        const response = await rgthreeApi.fetchJson("/config", { method: "POST", body });
+        if (response.status === "ok") {
+            for (const [key, value] of Object.entries(changed)) {
+                setObjectValue(rgthreeConfig, key, value);
+                this.dispatchEvent(new CustomEvent("config-change", { detail: { key, value } }));
+            }
+        }
+        else {
+            return false;
+        }
+        return true;
+    }
+}
+export const SERVICE = new ConfigService();

rgthree-comfy/web/comfyui/services/context_service.js

@@ -0,0 +1,51 @@
+import { getConnectedOutputNodesAndFilterPassThroughs } from "../utils.js";
+export let SERVICE;
+const OWNED_PREFIX = "+";
+const REGEX_PREFIX = /^[\+⚠️]\s*/;
+const REGEX_EMPTY_INPUT = /^\+\s*$/;
+export function stripContextInputPrefixes(name) {
+    return name.replace(REGEX_PREFIX, "");
+}
+export function getContextOutputName(inputName) {
+    if (inputName === "base_ctx")
+        return "CONTEXT";
+    return stripContextInputPrefixes(inputName).toUpperCase();
+}
+export var InputMutationOperation;
+(function (InputMutationOperation) {
+    InputMutationOperation[InputMutationOperation["UNKNOWN"] = 0] = "UNKNOWN";
+    InputMutationOperation[InputMutationOperation["ADDED"] = 1] = "ADDED";
+    InputMutationOperation[InputMutationOperation["REMOVED"] = 2] = "REMOVED";
+    InputMutationOperation[InputMutationOperation["RENAMED"] = 3] = "RENAMED";
+})(InputMutationOperation || (InputMutationOperation = {}));
+export class ContextService {
+    constructor() {
+        if (SERVICE) {
+            throw new Error("ContextService was already instantiated.");
+        }
+    }
+    onInputChanges(node, mutation) {
+        const childCtxs = getConnectedOutputNodesAndFilterPassThroughs(node, node, 0);
+        for (const childCtx of childCtxs) {
+            childCtx.handleUpstreamMutation(mutation);
+        }
+    }
+    getDynamicContextInputsData(node) {
+        return node
+            .getContextInputsList()
+            .map((input, index) => ({
+            name: stripContextInputPrefixes(input.name),
+            type: String(input.type),
+            index,
+        }))
+            .filter((i) => i.type !== "*");
+    }
+    getDynamicContextOutputsData(node) {
+        return node.outputs.map((output, index) => ({
+            name: stripContextInputPrefixes(output.name),
+            type: String(output.type),
+            index,
+        }));
+    }
+}
+SERVICE = new ContextService();

rgthree-comfy/web/comfyui/services/fast_groups_service.js

@@ -0,0 +1,138 @@
+import { app } from "../../../scripts/app.js";
+class FastGroupsService {
+    constructor() {
+        this.msThreshold = 400;
+        this.msLastUnsorted = 0;
+        this.msLastAlpha = 0;
+        this.msLastPosition = 0;
+        this.groupsUnsorted = [];
+        this.groupsSortedAlpha = [];
+        this.groupsSortedPosition = [];
+        this.fastGroupNodes = [];
+        this.runScheduledForMs = null;
+        this.runScheduleTimeout = null;
+        this.runScheduleAnimation = null;
+        this.cachedNodeBoundings = null;
+    }
+    addFastGroupNode(node) {
+        this.fastGroupNodes.push(node);
+        this.scheduleRun(8);
+    }
+    removeFastGroupNode(node) {
+        var _a;
+        const index = this.fastGroupNodes.indexOf(node);
+        if (index > -1) {
+            this.fastGroupNodes.splice(index, 1);
+        }
+        if (!((_a = this.fastGroupNodes) === null || _a === void 0 ? void 0 : _a.length)) {
+            this.clearScheduledRun();
+            this.groupsUnsorted = [];
+            this.groupsSortedAlpha = [];
+            this.groupsSortedPosition = [];
+        }
+    }
+    run() {
+        if (!this.runScheduledForMs) {
+            return;
+        }
+        for (const node of this.fastGroupNodes) {
+            node.refreshWidgets();
+        }
+        this.clearScheduledRun();
+        this.scheduleRun();
+    }
+    scheduleRun(ms = 500) {
+        if (this.runScheduledForMs && ms < this.runScheduledForMs) {
+            this.clearScheduledRun();
+        }
+        if (!this.runScheduledForMs && this.fastGroupNodes.length) {
+            this.runScheduledForMs = ms;
+            this.runScheduleTimeout = setTimeout(() => {
+                this.runScheduleAnimation = requestAnimationFrame(() => this.run());
+            }, ms);
+        }
+    }
+    clearScheduledRun() {
+        this.runScheduleTimeout && clearTimeout(this.runScheduleTimeout);
+        this.runScheduleAnimation && cancelAnimationFrame(this.runScheduleAnimation);
+        this.runScheduleTimeout = null;
+        this.runScheduleAnimation = null;
+        this.runScheduledForMs = null;
+    }
+    getBoundingsForAllNodes() {
+        if (!this.cachedNodeBoundings) {
+            this.cachedNodeBoundings = {};
+            for (const node of app.graph._nodes) {
+                this.cachedNodeBoundings[node.id] = node.getBounding();
+            }
+            setTimeout(() => {
+                this.cachedNodeBoundings = null;
+            }, 50);
+        }
+        return this.cachedNodeBoundings;
+    }
+    recomputeInsideNodesForGroup(group) {
+        const cachedBoundings = this.getBoundingsForAllNodes();
+        const nodes = group.graph._nodes;
+        group._nodes.length = 0;
+        for (const node of nodes) {
+            const node_bounding = cachedBoundings[node.id];
+            if (!node_bounding || !LiteGraph.overlapBounding(group._bounding, node_bounding)) {
+                continue;
+            }
+            group._nodes.push(node);
+        }
+    }
+    getGroupsUnsorted(now) {
+        const canvas = app.canvas;
+        const graph = app.graph;
+        if (!canvas.selected_group_moving &&
+            (!this.groupsUnsorted.length || now - this.msLastUnsorted > this.msThreshold)) {
+            this.groupsUnsorted = [...graph._groups];
+            for (const group of this.groupsUnsorted) {
+                this.recomputeInsideNodesForGroup(group);
+                group._rgthreeHasAnyActiveNode = group._nodes.some((n) => n.mode === LiteGraph.ALWAYS);
+            }
+            this.msLastUnsorted = now;
+        }
+        return this.groupsUnsorted;
+    }
+    getGroupsAlpha(now) {
+        const graph = app.graph;
+        if (!this.groupsSortedAlpha.length || now - this.msLastAlpha > this.msThreshold) {
+            this.groupsSortedAlpha = [...this.getGroupsUnsorted(now)].sort((a, b) => {
+                return a.title.localeCompare(b.title);
+            });
+            this.msLastAlpha = now;
+        }
+        return this.groupsSortedAlpha;
+    }
+    getGroupsPosition(now) {
+        const graph = app.graph;
+        if (!this.groupsSortedPosition.length || now - this.msLastPosition > this.msThreshold) {
+            this.groupsSortedPosition = [...this.getGroupsUnsorted(now)].sort((a, b) => {
+                const aY = Math.floor(a._pos[1] / 30);
+                const bY = Math.floor(b._pos[1] / 30);
+                if (aY == bY) {
+                    const aX = Math.floor(a._pos[0] / 30);
+                    const bX = Math.floor(b._pos[0] / 30);
+                    return aX - bX;
+                }
+                return aY - bY;
+            });
+            this.msLastPosition = now;
+        }
+        return this.groupsSortedPosition;
+    }
+    getGroups(sort) {
+        const now = +new Date();
+        if (sort === "alphanumeric") {
+            return this.getGroupsAlpha(now);
+        }
+        if (sort === "position") {
+            return this.getGroupsPosition(now);
+        }
+        return this.getGroupsUnsorted(now);
+    }
+}
+export const SERVICE = new FastGroupsService();

rgthree-comfy/web/common/css/buttons.css

@@ -0,0 +1,90 @@
+:not(#fakeid) .rgthree-button-reset {
+  position: relative;
+  appearance: none;
+  cursor: pointer;
+  border: 0;
+  background: transparent;
+  color: inherit;
+  padding: 0;
+  margin: 0;
+}
+
+:not(#fakeid) .rgthree-button {
+  --padding-top: 7px;
+  --padding-bottom: 9px;
+  --padding-x: 16px;
+  position: relative;
+  cursor: pointer;
+  border: 0;
+  border-radius: 0.25rem;
+  background: rgba(0, 0, 0, 0.5);
+  color: white;
+  font-family: system-ui, sans-serif;
+  font-size: 1rem;
+  line-height: 1;
+  white-space: nowrap;
+  text-decoration: none;
+  margin: 0.25rem;
+  box-shadow: 0px 0px 2px rgb(0, 0, 0);
+  background: #212121;
+  transition: all 0.1s ease-in-out;
+  padding: var(--padding-top) var(--padding-x) var(--padding-bottom);
+  display: inline-flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: center;
+}
+:not(#fakeid) .rgthree-button::before, :not(#fakeid) .rgthree-button::after {
+  content: "";
+  display: block;
+  position: absolute;
+  border-radius: 0.25rem;
+  left: 0;
+  top: 0;
+  width: 100%;
+  height: 100%;
+  box-shadow: inset 1px 1px 0px rgba(255, 255, 255, 0.12), inset -1px -1px 0px rgba(0, 0, 0, 0.75);
+  background: linear-gradient(to bottom, rgba(255, 255, 255, 0.06), rgba(0, 0, 0, 0.15));
+  mix-blend-mode: screen;
+}
+:not(#fakeid) .rgthree-button::after {
+  mix-blend-mode: multiply;
+}
+:not(#fakeid) .rgthree-button:hover {
+  background: #303030;
+}
+:not(#fakeid) .rgthree-button:active {
+  box-shadow: 0px 0px 0px rgba(0, 0, 0, 0);
+  background: #121212;
+  padding: calc(var(--padding-top) + 1px) calc(var(--padding-x) - 1px) calc(var(--padding-bottom) - 1px) calc(var(--padding-x) + 1px);
+}
+:not(#fakeid) .rgthree-button:active::before, :not(#fakeid) .rgthree-button:active::after {
+  box-shadow: 1px 1px 0px rgba(255, 255, 255, 0.15), inset 1px 1px 0px rgba(0, 0, 0, 0.5), inset 1px 3px 5px rgba(0, 0, 0, 0.33);
+}
+:not(#fakeid) .rgthree-button.-blue {
+  background: #346599 !important;
+}
+:not(#fakeid) .rgthree-button.-blue:hover {
+  background: #3b77b8 !important;
+}
+:not(#fakeid) .rgthree-button.-blue:active {
+  background: #1d5086 !important;
+}
+:not(#fakeid) .rgthree-button.-green {
+  background: linear-gradient(to bottom, rgba(255, 255, 255, 0.06), rgba(0, 0, 0, 0.15)), #14580b;
+}
+:not(#fakeid) .rgthree-button.-green:hover {
+  background: linear-gradient(to bottom, rgba(255, 255, 255, 0.06), rgba(0, 0, 0, 0.15)), #1a6d0f;
+}
+:not(#fakeid) .rgthree-button.-green:active {
+  background: linear-gradient(to bottom, rgba(0, 0, 0, 0.15), rgba(255, 255, 255, 0.06)), #0f3f09;
+}
+:not(#fakeid) .rgthree-button[disabled] {
+  box-shadow: none;
+  background: #666 !important;
+  color: #aaa;
+  pointer-events: none;
+}
+:not(#fakeid) .rgthree-button[disabled]::before, :not(#fakeid) .rgthree-button[disabled]::after {
+  display: none;
+}

rgthree-comfy/web/common/css/dialog.css

@@ -0,0 +1,124 @@
+@charset "UTF-8";
+.rgthree-dialog {
+  outline: 0;
+  border: 0;
+  border-radius: 6px;
+  background: #414141;
+  color: #fff;
+  box-shadow: inset 1px 1px 0px rgba(255, 255, 255, 0.05), inset -1px -1px 0px rgba(0, 0, 0, 0.5), 2px 2px 20px rgb(0, 0, 0);
+  max-width: 800px;
+  box-sizing: border-box;
+  font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif;
+  font-size: 1rem;
+  padding: 0;
+  max-height: calc(100% - 32px);
+}
+.rgthree-dialog *, .rgthree-dialog *::before, .rgthree-dialog *::after {
+  box-sizing: inherit;
+}
+
+.rgthree-dialog-container > * {
+  padding: 8px 16px;
+}
+.rgthree-dialog-container > *:first-child {
+  padding-top: 16px;
+}
+.rgthree-dialog-container > *:last-child {
+  padding-bottom: 16px;
+}
+
+.rgthree-dialog.-iconed::after {
+  content: "";
+  font-size: 276px;
+  position: absolute;
+  right: 0px;
+  bottom: 0px;
+  opacity: 0.15;
+  display: block;
+  width: 237px;
+  overflow: hidden;
+  height: 186px;
+  line-height: 1;
+  pointer-events: none;
+  z-index: -1;
+}
+
+.rgthree-dialog.-iconed.-help::after {
+  content: "🛟";
+}
+
+.rgthree-dialog.-iconed.-settings::after {
+  content: "⚙️";
+}
+
+@media (max-width: 832px) {
+  .rgthree-dialog {
+    max-width: calc(100% - 32px);
+  }
+}
+.rgthree-dialog-container-title {
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: start;
+}
+
+.rgthree-dialog-container-title > svg:first-child {
+  width: 36px;
+  height: 36px;
+  margin-right: 16px;
+}
+
+.rgthree-dialog-container-title h2 {
+  font-size: 1.375rem;
+  margin: 0;
+  font-weight: bold;
+}
+
+.rgthree-dialog-container-title h2 small {
+  font-size: 0.8125rem;
+  font-weight: normal;
+  opacity: 0.75;
+}
+
+.rgthree-dialog-container-content {
+  overflow: auto;
+  max-height: calc(100vh - 200px); /* Arbitrary height to copensate for margin, title, and footer.*/
+}
+
+.rgthree-dialog-container-content p {
+  font-size: 0.8125rem;
+  margin-top: 0;
+}
+
+.rgthree-dialog-container-content ul li p {
+  margin-bottom: 4px;
+}
+
+.rgthree-dialog-container-content ul li p + p {
+  margin-top: 0.5em;
+}
+
+.rgthree-dialog-container-content ul li ul {
+  margin-top: 0.5em;
+  margin-bottom: 1em;
+}
+
+.rgthree-dialog-container-content p code {
+  display: inline-block;
+  padding: 2px 4px;
+  margin: 0px 2px;
+  border: 1px solid rgba(255, 255, 255, 0.25);
+  border-radius: 3px;
+  background: rgba(255, 255, 255, 0.1);
+}
+
+.rgthree-dialog-container-footer {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+body.rgthree-dialog-open > *:not(.rgthree-dialog):not(.rgthree-top-messages-container) {
+  filter: blur(5px);
+}

rgthree-comfy/web/common/css/dialog_model_info.css

@@ -0,0 +1,333 @@
+.rgthree-info-dialog {
+  width: 90vw;
+  max-width: 960px;
+}
+.rgthree-info-dialog .rgthree-info-area {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+  display: flex;
+}
+.rgthree-info-dialog .rgthree-info-area > li {
+  display: inline-flex;
+  margin: 0;
+  vertical-align: top;
+}
+.rgthree-info-dialog .rgthree-info-area > li + li {
+  margin-left: 6px;
+}
+.rgthree-info-dialog .rgthree-info-area > li:not(.-link) + li.-link {
+  margin-left: auto;
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-tag > * {
+  min-height: 24px;
+  border-radius: 4px;
+  line-height: 1;
+  color: rgba(255, 255, 255, 0.85);
+  background: rgb(69, 92, 85);
+  font-size: 14px;
+  font-weight: bold;
+  text-decoration: none;
+  display: flex;
+  height: 1.6em;
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+  padding-bottom: 0.1em;
+  align-content: center;
+  justify-content: center;
+  align-items: center;
+  box-shadow: inset 0px 0px 0 1px rgba(0, 0, 0, 0.5);
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-tag > * > svg {
+  width: 16px;
+  height: 16px;
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-tag > * > svg:last-child {
+  margin-left: 0.5em;
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-tag > *[href] {
+  box-shadow: inset 0px 1px 0px rgba(255, 255, 255, 0.25), inset 0px -1px 0px rgba(0, 0, 0, 0.66);
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-tag > *:empty {
+  display: none;
+}
+.rgthree-info-dialog .rgthree-info-area > li.-type > * {
+  background: rgb(73, 54, 94);
+  color: rgb(228, 209, 248);
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-menu {
+  margin-left: auto;
+}
+:not(#fakeid) .rgthree-info-dialog .rgthree-info-area > li.rgthree-info-menu .rgthree-button {
+  margin: 0;
+  min-height: 24px;
+  padding: 0 12px;
+}
+.rgthree-info-dialog .rgthree-info-area > li.rgthree-info-menu svg {
+  width: 16px;
+  height: 16px;
+}
+.rgthree-info-dialog .rgthree-info-table {
+  border-collapse: collapse;
+  margin: 16px 0px;
+  width: 100%;
+  font-size: 12px;
+}
+.rgthree-info-dialog .rgthree-info-table tr.editable button {
+  display: flex;
+  width: 28px;
+  height: 28px;
+  align-items: center;
+  justify-content: center;
+}
+.rgthree-info-dialog .rgthree-info-table tr.editable button svg + svg {
+  display: none;
+}
+.rgthree-info-dialog .rgthree-info-table tr.editable.-rgthree-editing button svg {
+  display: none;
+}
+.rgthree-info-dialog .rgthree-info-table tr.editable.-rgthree-editing button svg + svg {
+  display: inline-block;
+}
+.rgthree-info-dialog .rgthree-info-table td {
+  position: relative;
+  border: 1px solid rgba(255, 255, 255, 0.25);
+  padding: 0;
+  vertical-align: top;
+}
+.rgthree-info-dialog .rgthree-info-table td:first-child {
+  background: rgba(255, 255, 255, 0.075);
+  width: 10px;
+}
+.rgthree-info-dialog .rgthree-info-table td:first-child > *:first-child {
+  white-space: nowrap;
+  padding-right: 32px;
+}
+.rgthree-info-dialog .rgthree-info-table td:first-child small {
+  display: block;
+  margin-top: 2px;
+  opacity: 0.75;
+}
+.rgthree-info-dialog .rgthree-info-table td:first-child small > [data-action] {
+  text-decoration: underline;
+  cursor: pointer;
+}
+.rgthree-info-dialog .rgthree-info-table td:first-child small > [data-action]:hover {
+  text-decoration: none;
+}
+.rgthree-info-dialog .rgthree-info-table td a, .rgthree-info-dialog .rgthree-info-table td a:hover, .rgthree-info-dialog .rgthree-info-table td a:visited {
+  color: inherit;
+}
+.rgthree-info-dialog .rgthree-info-table td svg {
+  width: 1.3333em;
+  height: 1.3333em;
+  vertical-align: -0.285em;
+}
+.rgthree-info-dialog .rgthree-info-table td svg.logo-civitai {
+  margin-right: 0.3333em;
+}
+.rgthree-info-dialog .rgthree-info-table td > *:first-child {
+  display: block;
+  padding: 6px 10px;
+}
+.rgthree-info-dialog .rgthree-info-table td > input, .rgthree-info-dialog .rgthree-info-table td > textarea {
+  padding: 5px 10px;
+  border: 0;
+  box-shadow: inset 1px 1px 5px 0px rgba(0, 0, 0, 0.5);
+  font: inherit;
+  appearance: none;
+  background: #fff;
+  color: #121212;
+  resize: vertical;
+}
+.rgthree-info-dialog .rgthree-info-table td > input:only-child, .rgthree-info-dialog .rgthree-info-table td > textarea:only-child {
+  width: 100%;
+}
+:not(#fakeid) .rgthree-info-dialog .rgthree-info-table td .rgthree-button[data-action=fetch-civitai] {
+  font-size: inherit;
+  padding: 6px 16px;
+  margin: 2px;
+}
+.rgthree-info-dialog .rgthree-info-table tr[data-field-name=userNote] td > span:first-child {
+  white-space: pre;
+}
+.rgthree-info-dialog .rgthree-info-table tr.rgthree-info-table-break-row td {
+  border: 0;
+  background: transparent;
+  padding: 12px 4px 4px;
+  font-size: 1.2em;
+}
+.rgthree-info-dialog .rgthree-info-table tr.rgthree-info-table-break-row td > small {
+  font-style: italic;
+  opacity: 0.66;
+}
+.rgthree-info-dialog .rgthree-info-table tr.rgthree-info-table-break-row td:empty {
+  padding: 4px;
+}
+.rgthree-info-dialog .rgthree-info-table td .-help {
+  border: 1px solid currentColor;
+  position: absolute;
+  right: 5px;
+  top: 6px;
+  line-height: 1;
+  font-size: 11px;
+  width: 12px;
+  height: 12px;
+  border-radius: 8px;
+  display: flex;
+  align-content: center;
+  justify-content: center;
+  cursor: help;
+}
+.rgthree-info-dialog .rgthree-info-table td .-help::before {
+  content: "?";
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list {
+  list-style: none;
+  padding: 2px 8px;
+  margin: 0;
+  display: flex;
+  flex-direction: row;
+  flex-wrap: wrap;
+  max-height: 15vh;
+  overflow: auto;
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li {
+  display: inline-flex;
+  margin: 2px;
+  vertical-align: top;
+  border-radius: 4px;
+  line-height: 1;
+  color: rgba(255, 255, 255, 0.85);
+  background: rgb(73, 91, 106);
+  font-size: 1.2em;
+  font-weight: 600;
+  text-decoration: none;
+  display: flex;
+  height: 1.6em;
+  align-content: center;
+  justify-content: center;
+  align-items: center;
+  box-shadow: inset 0px 0px 0 1px rgba(0, 0, 0, 0.5);
+  cursor: pointer;
+  white-space: nowrap;
+  max-width: 183px;
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li:hover {
+  background: rgb(68, 109, 142);
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li > svg {
+  width: auto;
+  height: 1.2em;
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li > span {
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+  padding-bottom: 0.1em;
+  text-overflow: ellipsis;
+  overflow: hidden;
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li > small {
+  align-self: stretch;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  padding: 0 0.5em;
+  background: rgba(0, 0, 0, 0.2);
+}
+.rgthree-info-dialog .rgthree-info-table td > ul.rgthree-info-trained-words-list > li.-rgthree-is-selected {
+  background: rgb(42, 126, 193);
+}
+.rgthree-info-dialog .rgthree-info-images {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+  scroll-snap-type: x mandatory;
+  display: flex;
+  flex-direction: row;
+  overflow: auto;
+}
+.rgthree-info-dialog .rgthree-info-images > li {
+  scroll-snap-align: start;
+  max-width: 90%;
+  flex: 0 0 auto;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  flex-direction: column;
+  overflow: hidden;
+  padding: 0;
+  margin: 6px;
+  font-size: 0;
+  position: relative;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure {
+  margin: 0;
+  position: static;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption {
+  position: absolute;
+  left: 0;
+  width: 100%;
+  bottom: 0;
+  padding: 12px;
+  font-size: 12px;
+  background: rgba(0, 0, 0, 0.85);
+  opacity: 0;
+  transform: translateY(50px);
+  transition: all 0.25s ease-in-out;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption > span {
+  display: inline-block;
+  padding: 2px 4px;
+  margin: 2px;
+  border-radius: 2px;
+  border: 1px solid rgba(255, 255, 255, 0.2);
+  word-break: break-word;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption > span label {
+  display: inline;
+  padding: 0;
+  margin: 0;
+  opacity: 0.5;
+  pointer-events: none;
+  user-select: none;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption > span a {
+  color: inherit;
+  text-decoration: underline;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption > span a:hover {
+  text-decoration: none;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption > span a svg {
+  height: 10px;
+  margin-left: 4px;
+  fill: currentColor;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption:empty {
+  text-align: center;
+}
+.rgthree-info-dialog .rgthree-info-images > li figure figcaption:empty::before {
+  content: "No data.";
+}
+.rgthree-info-dialog .rgthree-info-images > li:hover figure figcaption {
+  opacity: 1;
+  transform: translateY(0px);
+}
+.rgthree-info-dialog .rgthree-info-images > li .rgthree-info-table {
+  width: calc(100% - 16px);
+}
+.rgthree-info-dialog .rgthree-info-civitai-link {
+  margin: 8px;
+  color: #eee;
+}
+.rgthree-info-dialog .rgthree-info-civitai-link a, .rgthree-info-dialog .rgthree-info-civitai-link a:hover, .rgthree-info-dialog .rgthree-info-civitai-link a:visited {
+  color: inherit;
+  text-decoration: none;
+}
+.rgthree-info-dialog .rgthree-info-civitai-link > svg {
+  width: 16px;
+  height: 16px;
+  margin-right: 8px;
+}

rgthree-comfy/web/common/css/menu.css

@@ -0,0 +1,91 @@
+.rgthree-menu {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+  position: fixed;
+  z-index: 999999;
+  pointer-events: none;
+  opacity: 0;
+  transition: opacity 0.08s ease-in-out;
+  color: #dde;
+  background-color: #111;
+  font-size: 12px;
+  box-shadow: 0 0 10px black !important;
+}
+.rgthree-menu > li {
+  position: relative;
+  padding: 4px 6px;
+  z-index: 9999;
+  white-space: nowrap;
+}
+.rgthree-menu > li[role=button] {
+  background-color: var(--comfy-menu-bg) !important;
+  color: var(--input-text);
+  cursor: pointer;
+}
+.rgthree-menu > li[role=button]:hover {
+  filter: brightness(155%);
+}
+.rgthree-menu[state^=measuring] {
+  display: block;
+  opacity: 0;
+}
+.rgthree-menu[state=open] {
+  display: block;
+  opacity: 1;
+  pointer-events: all;
+}
+
+.rgthree-top-menu {
+  box-sizing: border-box;
+  white-space: nowrap;
+  background: var(--content-bg);
+  color: var(--content-fg);
+  display: flex;
+  flex-direction: column;
+}
+.rgthree-top-menu * {
+  box-sizing: inherit;
+}
+.rgthree-top-menu menu {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+}
+.rgthree-top-menu menu > li:not(#fakeid) {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+}
+.rgthree-top-menu menu > li:not(#fakeid) > button {
+  cursor: pointer;
+  padding: 8px 12px 8px 8px;
+  width: 100%;
+  text-align: start;
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: start;
+}
+.rgthree-top-menu menu > li:not(#fakeid) > button:hover {
+  background-color: var(--comfy-input-bg);
+}
+.rgthree-top-menu menu > li:not(#fakeid) > button svg {
+  height: 16px;
+  width: auto;
+  margin-inline-end: 0.6em;
+}
+.rgthree-top-menu menu > li:not(#fakeid) > button svg.github-star {
+  fill: rgb(227, 179, 65);
+}
+.rgthree-top-menu menu > li:not(#fakeid).rgthree-message {
+  min-height: 32px;
+}
+.rgthree-top-menu menu > li:not(#fakeid).rgthree-message > span {
+  padding: 8px 12px;
+  display: block;
+  width: 100%;
+  text-align: center;
+  font-style: italic;
+  font-size: 12px;
+}

rgthree-comfy/web/common/css/pages_base.css

@@ -0,0 +1,66 @@
+html {
+  font-size: 100%;
+  overflow-y: scroll;
+  -webkit-text-size-adjust: 100%;
+  -ms-text-size-adjust: 100%;
+  box-sizing: border-box;
+}
+
+*, *:before, *:after {
+  box-sizing: inherit;
+}
+
+:root {
+  --header-height: 56px;
+  --progress-height: 12px;
+}
+
+button {
+  all: unset;
+}
+
+.-bevel {
+  position: relative;
+}
+
+.-bevel::before {
+  content: "";
+  position: absolute;
+  left: 0;
+  top: 0;
+  width: 100%;
+  height: 100%;
+  border: 1px solid red;
+  border-color: rgba(255, 255, 255, 0.15) rgba(255, 255, 255, 0.15) rgba(0, 0, 0, 0.5) rgba(0, 0, 0, 0.5);
+  z-index: 5;
+  pointer-events: none;
+}
+
+body {
+  background: #202020;
+  font-family: Arial, sans-serif;
+  font-size: 1rem;
+  font-weight: 400;
+  margin: 0;
+  padding-top: calc(var(--header-height) + var(--progress-height));
+  color: #ffffff;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: start;
+}
+
+.app-header {
+  height: var(--header-height);
+  padding: 0;
+  position: fixed;
+  z-index: 99;
+  top: 0;
+  left: 0;
+  width: 100%;
+  background: #353535;
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: start;
+}

rgthree-comfy/web/common/media/svgs.js

@@ -0,0 +1,160 @@
+import { createElement as $el } from "../utils_dom.js";
+export const logoRgthree = `<svg viewBox="0 0 256 256" fill="currentColor" class="rgthree-logo">
+    <path d="M88.503,158.997 L152.731,196.103 L152.738,196.092 L152.762,196.103 L152.769,196.106 L152.771,196.103 L183.922,142.084     L174.153,136.437 L148.611,180.676 L101.512,153.484 L132.193,30.415 L156.124,71.869 L165.896,66.225 L128.002,0.59 "></path>
+    <path d="M55.586,148.581l13.44,47.521l0.014,0.051l0.168-0.051l10.689-3.022l-6.589-23.313l45.609,26.335l0.087,0.051l0.027-0.051     l5.617-9.718l-42.648-24.622l35.771-143.45L33.232,164.729l9.77,5.645L55.586,148.581z M87.394,93.484l-16.708,67.018l-5.018-17.747     l-8.028,2.27L87.394,93.484z"></path>
+    <path d="M189.85,107.717 L137.892,137.718 L143.532,147.49 L185.723,123.133 L231.109,201.746 L24.895,201.746 L37.363,180.146     L27.592,174.505 L5.347,213.03 L250.653,213.03 "></path>
+    <path d="M5.347,247.299v8.111h245.307v-8.111l-41.94-0.003c-1.336,0-2.404-1.065-2.441-2.396v-12.14     c0.037-1.315,1.089-2.368,2.41-2.385h41.972v-8.11H5.347v8.11h41.951c1.338,0.017,2.427,1.104,2.427,2.449v12.01     c0,1.365-1.105,2.462-2.457,2.462L5.347,247.299z M139.438,247.296c-1.334,0-2.406-1.065-2.439-2.396v-12.14     c0.033-1.315,1.085-2.368,2.41-2.385h46.415c1.335,0.017,2.425,1.104,2.425,2.449v12.01c0,1.365-1.103,2.462-2.459,2.462H139.438z       M70.193,247.296c-1.339,0-2.408-1.065-2.441-2.396v-12.14c0.033-1.315,1.086-2.368,2.407-2.385h46.418     c1.336,0.017,2.425,1.104,2.425,2.449v12.01c0,1.365-1.103,2.462-2.458,2.462H70.193z"></path>
+  </svg>`;
+export const github = `<svg viewBox="0 0 16 16" fill="currentColor" class="github-logo">
+    <path d="M8 0c4.42 0 8 3.58 8 8a8.013 8.013 0 0 1-5.45 7.59c-.4.08-.55-.17-.55-.38 0-.27.01-1.13.01-2.2 0-.75-.25-1.23-.54-1.48 1.78-.2 3.65-.88 3.65-3.95 0-.88-.31-1.59-.82-2.15.08-.2.36-1.02-.08-2.12 0 0-.67-.22-2.2.82-.64-.18-1.32-.27-2-.27-.68 0-1.36.09-2 .27-1.53-1.03-2.2-.82-2.2-.82-.44 1.1-.16 1.92-.08 2.12-.51.56-.82 1.28-.82 2.15 0 3.06 1.86 3.75 3.64 3.95-.23.2-.44.55-.51 1.07-.46.21-1.61.55-2.33-.66-.15-.24-.6-.83-1.23-.82-.67.01-.27.38.01.53.34.19.73.9.82 1.13.16.45.68 1.31 2.69.94 0 .67.01 1.3.01 1.49 0 .21-.15.45-.55.38A7.995 7.995 0 0 1 0 8c0-4.42 3.58-8 8-8Z"></path>
+  </svg>`;
+export const iconStarFilled = `<svg viewBox="0 0 16 16" fill="currentColor" class="github-star">
+    <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Z"></path>
+  </svg>`;
+export const iconReplace = `<svg viewBox="0 0 52 52" fill="currentColor">
+    <path d="M20,37.5c0-0.8-0.7-1.5-1.5-1.5h-15C2.7,36,2,36.7,2,37.5v11C2,49.3,2.7,50,3.5,50h15c0.8,0,1.5-0.7,1.5-1.5 V37.5z"/>
+    <path d="M8.1,22H3.2c-1,0-1.5,0.9-0.9,1.4l8,8.3c0.4,0.3,1,0.3,1.4,0l8-8.3c0.6-0.6,0.1-1.4-0.9-1.4h-4.7 c0-5,4.9-10,9.9-10V6C15,6,8.1,13,8.1,22z"/>
+    <path d="M41.8,20.3c-0.4-0.3-1-0.3-1.4,0l-8,8.3c-0.6,0.6-0.1,1.4,0.9,1.4h4.8c0,6-4.1,10-10.1,10v6 c9,0,16.1-7,16.1-16H49c1,0,1.5-0.9,0.9-1.4L41.8,20.3z"/>
+    <path d="M50,3.5C50,2.7,49.3,2,48.5,2h-15C32.7,2,32,2.7,32,3.5v11c0,0.8,0.7,1.5,1.5,1.5h15c0.8,0,1.5-0.7,1.5-1.5 V3.5z"/>
+  </svg>`;
+export const iconNode = `<svg viewBox="0 -0.5 25 25" fill="none">
+    <path fill-rule="evenodd" clip-rule="evenodd" d="M15.5 19H9.5C7.29086 19 5.5 17.2091 5.5 15V9C5.5 6.79086 7.29086 5 9.5 5H15.5C17.7091 5 19.5 6.79086 19.5 9V15C19.5 17.2091 17.7091 19 15.5 19Z" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+    <path d="M19.5 9.75C19.9142 9.75 20.25 9.41421 20.25 9C20.25 8.58579 19.9142 8.25 19.5 8.25V9.75ZM5.5 8.25C5.08579 8.25 4.75 8.58579 4.75 9C4.75 9.41421 5.08579 9.75 5.5 9.75V8.25ZM11.5 14.25C11.0858 14.25 10.75 14.5858 10.75 15C10.75 15.4142 11.0858 15.75 11.5 15.75V14.25ZM13.5 15.75C13.9142 15.75 14.25 15.4142 14.25 15C14.25 14.5858 13.9142 14.25 13.5 14.25V15.75ZM19.5 8.25H5.5V9.75H19.5V8.25ZM11.5 15.75H13.5V14.25H11.5V15.75Z" fill="currentColor" />
+  </svg>`;
+export const iconGear = `<svg viewBox="0 0 24 24" fill="currentColor">
+    <path fill-rule="evenodd" clip-rule="evenodd" d="M12.7848 0.449982C13.8239 0.449982 14.7167 1.16546 14.9122 2.15495L14.9991 2.59495C15.3408 4.32442 17.1859 5.35722 18.9016 4.7794L19.3383 4.63233C20.3199 4.30175 21.4054 4.69358 21.9249 5.56605L22.7097 6.88386C23.2293 7.75636 23.0365 8.86366 22.2504 9.52253L21.9008 9.81555C20.5267 10.9672 20.5267 13.0328 21.9008 14.1844L22.2504 14.4774C23.0365 15.1363 23.2293 16.2436 22.7097 17.1161L21.925 18.4339C21.4054 19.3064 20.3199 19.6982 19.3382 19.3676L18.9017 19.2205C17.1859 18.6426 15.3408 19.6754 14.9991 21.405L14.9122 21.845C14.7167 22.8345 13.8239 23.55 12.7848 23.55H11.2152C10.1761 23.55 9.28331 22.8345 9.08781 21.8451L9.00082 21.4048C8.65909 19.6754 6.81395 18.6426 5.09822 19.2205L4.66179 19.3675C3.68016 19.6982 2.59465 19.3063 2.07505 18.4338L1.2903 17.1161C0.770719 16.2436 0.963446 15.1363 1.74956 14.4774L2.09922 14.1844C3.47324 13.0327 3.47324 10.9672 2.09922 9.8156L1.74956 9.52254C0.963446 8.86366 0.77072 7.75638 1.2903 6.8839L2.07508 5.56608C2.59466 4.69359 3.68014 4.30176 4.66176 4.63236L5.09831 4.77939C6.81401 5.35722 8.65909 4.32449 9.00082 2.59506L9.0878 2.15487C9.28331 1.16542 10.176 0.449982 11.2152 0.449982H12.7848ZM12 15.3C13.8225 15.3 15.3 13.8225 15.3 12C15.3 10.1774 13.8225 8.69998 12 8.69998C10.1774 8.69998 8.69997 10.1774 8.69997 12C8.69997 13.8225 10.1774 15.3 12 15.3Z" />
+  </svg>`;
+export const checkmark = `<svg viewBox="0 0 32 32" fill="currentColor" class="icon-checkmark">
+    <g transform="translate(-518.000000, -1039.000000)">
+      <path d="M548.783,1040.2 C547.188,1038.57 544.603,1038.57 543.008,1040.2 L528.569,1054.92 L524.96,1051.24 C523.365,1049.62 520.779,1049.62 519.185,1051.24 C517.59,1052.87 517.59,1055.51 519.185,1057.13 L525.682,1063.76 C527.277,1065.39 529.862,1065.39 531.457,1063.76 L548.783,1046.09 C550.378,1044.46 550.378,1041.82 548.783,1040.2"></path>
+    </g>
+  </svg>`;
+export const logoCivitai = `<svg viewBox="0 0 178 178" class="logo-civitai">
+    <defs>
+      <linearGradient id="bgblue" gradientUnits="userSpaceOnUse" x1="89.3" y1="-665.5" x2="89.3" y2="-841.1" gradientTransform="matrix(1 0 0 -1 0 -664)">
+        <stop offset="0" style="stop-color:#1284F7"/>
+        <stop offset="1" style="stop-color:#0A20C9"/>
+      </linearGradient>
+    </defs>
+    <path fill="#000" d="M13.3,45.4v87.7l76,43.9l76-43.9V45.4l-76-43.9L13.3,45.4z"/>
+    <path style="fill:url(#bgblue);" d="M89.3,29.2l52,30v60l-52,30l-52-30v-60 L89.3,29.2 M89.3,1.5l-76,43.9v87.8l76,43.9l76-43.9V45.4L89.3,1.5z" />
+    <path fill="#FFF" d="M104.1,97.2l-14.9,8.5l-14.9-8.5v-17l14.9-8.5l14.9,8.5h18.2V69.7l-33-19l-33,19v38.1l33,19l33-19V97.2H104.1z" />
+  </svg>`;
+export const iconOutLink = `<svg viewBox="0 0 32 32">
+    <path d="M 18 5 L 18 7 L 23.5625 7 L 11.28125 19.28125 L 12.71875 20.71875 L 25 8.4375 L 25 14 L 27 14 L 27 5 Z M 5 9 L 5 27 L 23 27 L 23 14 L 21 16 L 21 25 L 7 25 L 7 11 L 16 11 L 18 9 Z"></path>
+  </svg>`;
+export const link = `<svg viewBox="0 0 640 512">
+    <path d="M598.6 41.41C570.1 13.8 534.8 0 498.6 0s-72.36 13.8-99.96 41.41l-43.36 43.36c15.11 8.012 29.47 17.58 41.91 30.02c3.146 3.146 5.898 6.518 8.742 9.838l37.96-37.96C458.5 72.05 477.1 64 498.6 64c20.67 0 40.1 8.047 54.71 22.66c14.61 14.61 22.66 34.04 22.66 54.71s-8.049 40.1-22.66 54.71l-133.3 133.3C405.5 343.1 386 352 365.4 352s-40.1-8.048-54.71-22.66C296 314.7 287.1 295.3 287.1 274.6s8.047-40.1 22.66-54.71L314.2 216.4C312.1 212.5 309.9 208.5 306.7 205.3C298.1 196.7 286.8 192 274.6 192c-11.93 0-23.1 4.664-31.61 12.97c-30.71 53.96-23.63 123.6 22.39 169.6C293 402.2 329.2 416 365.4 416c36.18 0 72.36-13.8 99.96-41.41L598.6 241.3c28.45-28.45 42.24-66.01 41.37-103.3C639.1 102.1 625.4 68.16 598.6 41.41zM234 387.4L196.1 425.3C181.5 439.1 162 448 141.4 448c-20.67 0-40.1-8.047-54.71-22.66c-14.61-14.61-22.66-34.04-22.66-54.71s8.049-40.1 22.66-54.71l133.3-133.3C234.5 168 253.1 160 274.6 160s40.1 8.048 54.71 22.66c14.62 14.61 22.66 34.04 22.66 54.71s-8.047 40.1-22.66 54.71L325.8 295.6c2.094 3.939 4.219 7.895 7.465 11.15C341.9 315.3 353.3 320 365.4 320c11.93 0 23.1-4.664 31.61-12.97c30.71-53.96 23.63-123.6-22.39-169.6C346.1 109.8 310.8 96 274.6 96C238.4 96 202.3 109.8 174.7 137.4L41.41 270.7c-27.6 27.6-41.41 63.78-41.41 99.96c-.0001 36.18 13.8 72.36 41.41 99.97C69.01 498.2 105.2 512 141.4 512c36.18 0 72.36-13.8 99.96-41.41l43.36-43.36c-15.11-8.012-29.47-17.58-41.91-30.02C239.6 394.1 236.9 390.7 234 387.4z"/>
+  </svg>`;
+export const pencil = `<svg viewBox="0 0 24 24">
+    <path d="M 16.9375 1.0625 L 3.875 14.125 L 1.0742188 22.925781 L 9.875 20.125 L 22.9375 7.0625 C 22.9375 7.0625 22.8375 4.9615 20.9375 3.0625 C 19.0375 1.1625 16.9375 1.0625 16.9375 1.0625 z M 17.3125 2.6875 C 18.3845 2.8915 19.237984 3.3456094 19.896484 4.0214844 C 20.554984 4.6973594 21.0185 5.595 21.3125 6.6875 L 19.5 8.5 L 15.5 4.5 L 16.9375 3.0625 L 17.3125 2.6875 z M 4.9785156 15.126953 C 4.990338 15.129931 6.1809555 15.430955 7.375 16.625 C 8.675 17.825 8.875 18.925781 8.875 18.925781 L 8.9179688 18.976562 L 5.3691406 20.119141 L 3.8730469 18.623047 L 4.9785156 15.126953 z"/>
+  </svg>`;
+export const dotdotdot = `<svg viewBox="0 0 24 24" fill="currentColor">
+  <circle cy="12" r="3" cx="3"></circle>
+  <circle cy="12" r="3" cx="12"></circle>
+  <circle cx="21" cy="12" r="3"></circle>
+</svg>`;
+export const models = `<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+    <path d="M4 4h6v6h-6z"></path>
+<path d="M14 4h6v6h-6z"></path>
+<path d="M4 14h6v6h-6z"></path>
+<path d="M17 17m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
+</svg>`;
+export const pencilColored = `<svg viewBox="0 0 64 64">
+    <path fill="#ffce31" d="M7.934 41.132L39.828 9.246l14.918 14.922l-31.895 31.886z"></path>
+    <path d="M61.3 4.6l-1.9-1.9C55.8-.9 50-.9 46.3 2.7l-6.5 6.5l15 15l6.5-6.5c3.6-3.6 3.6-9.5 0-13.1" fill="#ed4c5c"></path>
+    <path fill="#93a2aa" d="M35.782 13.31l4.1-4.102l14.92 14.92l-4.1 4.101z"></path>
+    <path fill="#c7d3d8" d="M37.338 14.865l4.1-4.101l11.739 11.738l-4.102 4.1z"></path>
+    <path fill="#fed0ac" d="M7.9 41.1l-6.5 17l4.5 4.5l17-6.5z"/>
+    <path d="M.3 61.1c-.9 2.4.3 3.5 2.7 2.6l8.2-3.1l-7.7-7.7l-3.2 8.2" fill="#333"></path>
+    <path fill="#ffdf85" d="M7.89 41.175l27.86-27.86l4.95 4.95l-27.86 27.86z"/>
+    <path fill="#ff8736" d="M17.904 51.142l27.86-27.86l4.95 4.95l-27.86 27.86z"></path>
+  </svg>`;
+export const diskColored = `<svg viewBox="-0.01 -0.008 100.016 100.016">
+  <path fill="#26f" fill_="#23475F" d="M88.555-.008H83v.016a2 2 0 0 1-2 2H19a2 2 0 0 1-2-2v-.016H4a4 4 0 0 0-4 4v92.016a4 4 0 0 0 4 4h92a4 4 0 0 0 4-4V11.517c.049-.089-11.436-11.454-11.445-11.525z"/>
+  <path fill="#04d" fill_="#1C3C50" d="M81.04 53.008H18.96a2 2 0 0 0-2 2v45h66.08v-45c0-1.106-.895-2-2-2zm-61.957-10h61.834a2 2 0 0 0 2-2V.547A1.993 1.993 0 0 1 81 2.007H19c-.916 0-1.681-.62-1.917-1.46v40.46a2 2 0 0 0 2 2.001z"/>
+  <path fill="#EBF0F1" d="M22 55.977h56a2 2 0 0 1 2 2v37.031a2 2 0 0 1-2 2H22c-1.104 0-2-.396-2-1.5V57.977a2 2 0 0 1 2-2z"/>
+  <path fill="#BCC4C8" d="M25 77.008h50v1H25v-1zm0 10h50v1H25v-1z"/>
+  <path fill="#1C3C50" d="M7 84.008h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2v-3a2 2 0 0 1 2-2zm83 0h3a2 2 0 0 1 2 2v3a2 2 0 0 1-2 2h-3a2 2 0 0 1-2-2v-3a2 2 0 0 1 2-2z"/>
+  <path fill="#BCC4C8" d="M37 1.981v36.026a2 2 0 0 0 2 2h39a2 2 0 0 0 2-2V1.981c0 .007-42.982.007-43 0zm37 29.027a2 2 0 0 1-2 2h-6a2 2 0 0 1-2-2V10.981a2 2 0 0 1 2-2h6a2 2 0 0 1 2 2v20.027z"/>
+  <path fill="#FF9D00" d="M78 55.977H22a2 2 0 0 0-2 2v10.031h60V57.977a2 2 0 0 0-2-2z"/>
+</svg>`;
+export const folderColored = `<svg viewBox="0 0 501.379 501.379">
+  <path style="fill:#EF9F2C;" d="M406.423,93.889H205.889c-17.067,0-30.933-13.867-30.933-30.933s-13.867-30.933-30.933-30.933H30.956
+      c-17.067,0-30.933,13.867-30.933,30.933v375.467c0,17.067,13.867,30.933,30.933,30.933h375.467
+      c17.067,0,30.933-13.867,30.933-30.933v-313.6C436.289,107.756,422.423,93.889,406.423,93.889z"/>
+  <path style="fill:#FEC656;" d="M470.423,157.889H97.089c-13.867,0-26.667,9.6-29.867,22.4l-66.133,249.6
+    c-5.333,19.2,9.6,38.4,29.867,38.4h373.333c13.867,0,26.667-9.6,29.867-22.4l66.133-248.533
+    C505.623,177.089,490.689,157.889,470.423,157.889z"/>
+</svg>`;
+export const modelsColored = `<svg viewBox="0 0 24 24">
+  <path fill="#aa3366" d="M0 0h10v10h-10z"></path>
+  <path d="M14 0h10v10h-10z" fill="#3366aa"></path>
+  <path d="M0 14h10v10h-10z" fill="#66aa33"></path>
+  <path fill="#dd9922" d="M19 19m-5 0 a5 5 0 1 0 10 0 a5 5 0 1 0 -10 0"></path>
+</svg>`;
+export const legoBlocksColored = `<svg viewBox="0 0 512 512">
+  <g>
+    <rect x="57.67" style="fill:#00BAB9;" width="101.275" height="78.769"/>
+    <rect x="205.363" style="fill:#00BAB9;" width="101.275" height="78.769"/>
+    <rect x="353.055" style="fill:#00BAB9;" width="101.275" height="78.769"/>
+  </g>
+  <polygon style="fill:#B8DE6F;" points="478.242,289.758 478.242,512 33.758,512 33.758,289.758 256,267.253 "/>
+  <polygon style="fill:#41D4D3;" points="478.242,67.516 478.242,289.758 33.758,289.758 33.758,67.516 57.67,67.516 158.945,67.516
+    205.363,67.516 306.637,67.516 353.055,67.516 454.33,67.516 "/>
+  <g>
+    <circle style="fill:#00BAB9;" cx="402.286" cy="143.473" r="8.44"/>
+    <circle style="fill:#00BAB9;" cx="368.527" cy="177.231" r="8.44"/>
+  </g>
+  <circle style="fill:#7BD288;" cx="109.714" cy="436.044" r="8.44"/>
+</svg>`;
+export const legoBlockColored = `<svg viewBox="0 0 256 256">
+  <style>
+  .s0 { fill: #ff0000 }
+  .s1 { fill: #c30000 }
+  .s2 { fill: #800000 }
+  .s3 { fill: #cc0000 }
+  .s4 { fill: #e00000 }
+  </style>
+  <g id="Folder 2">
+  <path id="Shape 1 copy 2" class="s0" d="m128 61l116 45-116 139-116-139z"/>
+  <path id="Shape 1" class="s1" d="m12 106l116 45v95l-116-45z"/>
+  <path id="Shape 1 copy" class="s2" d="m244 106l-116 45v95l116-45z"/>
+  <g id="Folder 1">
+    <path id="Shape 2" class="s3" d="m102 111.2c0-6.1 11.4-9.9 25.5-9.9 14.1 0 25.5 3.8 25.5 9.9 0 3.3 0 13.3 0 16.6 0 6.1-11.4 10.9-25.5 10.9-14.1 0-25.5-4.8-25.5-10.9 0-3.3 0-13.3 0-16.6z"/>
+    <path id="Shape 2 copy 4" class="s1" d="m102 111.2c0-6.1 11.4-9.9 25.5-9.9 14.1 0 25.5 3.8 25.5 9.9 0 3.3 0 13.3 0 16.6 0 6.1-11.4 10.9-25.5 10.9-14.1 0-25.5-4.8-25.5-10.9 0-3.3 0-13.3 0-16.6z"/>
+    <path id="Shape 2 copy 2" class="s2" d="m127.5 101.3c14.1 0 25.5 3.8 25.5 9.9 0 3.3 0 13.3 0 16.6 0 6.1-11.4 10.9-25.5 10.9 0-13.1 0-25.7 0-37.4z"/>
+    <path id="Shape 2 copy" class="s0" d="m127.5 118.8c-12.2 0-22-3.4-22-7.6 0-4.2 9.8-7.7 22-7.7 12.2 0 22 3.5 22 7.7 0 4.2-9.8 7.6-22 7.6zm0 0c-12.2 0-22-3.4-22-7.6 0-4.2 9.8-7.7 22-7.7 12.2 0 22 3.5 22 7.7 0 4.2-9.8 7.6-22 7.6z"/>
+  </g>
+  <g id="Folder 1 copy">
+    <path id="Shape 2" class="s4" d="m103 67.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 4" class="s1" d="m103 67.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 2" class="s2" d="m127.5 58c13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5 0-12.6 0-24.8 0-36z"/>
+    <path id="Shape 2 copy" class="s0" d="m127.5 74.9c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4zm0 0c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4z"/>
+  </g>
+  <g id="Folder 1 copy 2">
+    <path id="Shape 2" class="s4" d="m161 89.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 4" class="s1" d="m161 89.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 2" class="s2" d="m185.5 80c13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5 0-12.6 0-24.8 0-36z"/>
+    <path id="Shape 2 copy" class="s0" d="m185.5 96.9c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4zm0 0c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4z"/>
+  </g>
+  <g id="Folder 1 copy 3">
+    <path id="Shape 2" class="s4" d="m45 89.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 4" class="s1" d="m45 89.5c0-5.8 11-9.5 24.5-9.5 13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5-13.5 0-24.5-4.7-24.5-10.5 0-3.2 0-12.8 0-16z"/>
+    <path id="Shape 2 copy 2" class="s2" d="m69.5 80c13.5 0 24.5 3.7 24.5 9.5 0 3.2 0 12.8 0 16 0 5.8-11 10.5-24.5 10.5 0-12.6 0-24.8 0-36z"/>
+    <path id="Shape 2 copy" class="s0" d="m69.5 96.9c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4zm0 0c-11.7 0-21.2-3.3-21.2-7.4 0-4.1 9.5-7.4 21.2-7.4 11.7 0 21.2 3.3 21.2 7.4 0 4.1-9.5 7.4-21.2 7.4z"/>
+  </g>
+  </g>
+</svg>`;
+export const gearColored = `<svg viewBox="0 0 128 128" preserveAspectRatio="xMidYMid meet">
+  <path d="M124 71.85v-15.7c0-.59-.45-1.09-1.03-1.15l-17.83-1.89c-.47-.05-.85-.38-.98-.83c-.86-2.95-2.03-5.76-3.48-8.39c-.23-.41-.19-.92.11-1.28l11.28-13.94c.37-.46.34-1.13-.08-1.54l-11.1-11.1a1.15 1.15 0 0 0-1.54-.08L85.39 27.22c-.37.3-.87.33-1.28.11a41.796 41.796 0 0 0-8.39-3.48c-.45-.13-.78-.51-.83-.98L73 5.03C72.94 4.45 72.44 4 71.85 4h-15.7c-.59 0-1.09.45-1.15 1.03l-1.89 17.83c-.05.47-.38.85-.83.98c-2.95.86-5.76 2.03-8.39 3.48c-.41.23-.92.19-1.28-.11L28.67 15.94a1.15 1.15 0 0 0-1.54.08l-11.1 11.1a1.15 1.15 0 0 0-.08 1.54L27.23 42.6c.3.37.33.87.11 1.28a41.796 41.796 0 0 0-3.48 8.39c-.13.45-.51.78-.98.83L5.03 55c-.58.06-1.03.56-1.03 1.15v15.7c0 .59.45 1.09 1.03 1.15l17.83 1.89c.47.05.85.38.98.83c.86 2.95 2.03 5.76 3.48 8.39c.23.41.19.92-.11 1.28L15.94 99.33c-.37.46-.34 1.13.08 1.54l11.1 11.1c.42.42 1.08.45 1.54.08l13.94-11.28c.37-.3.87-.33 1.28-.11c2.64 1.45 5.45 2.62 8.39 3.48c.45.13.78.51.83.98l1.9 17.85c.06.59.56 1.03 1.15 1.03h15.7c.59 0 1.09-.45 1.15-1.03l1.89-17.83c.05-.47.38-.85.83-.98c2.95-.86 5.76-2.03 8.39-3.48c.41-.23.92-.19 1.28.11l13.94 11.28c.46.37 1.13.34 1.54-.08l11.1-11.1c.42-.42.45-1.08.08-1.54l-11.28-13.94c-.3-.37-.33-.87-.11-1.28c1.45-2.64 2.62-5.45 3.48-8.39c.13-.45.51-.78.98-.83L122.97 73c.58-.06 1.03-.56 1.03-1.15zm-60 3.43c-6.23 0-11.28-5.05-11.28-11.28S57.77 52.72 64 52.72S75.28 57.77 75.28 64S70.23 75.28 64 75.28z" fill="#82aec0"></path>
+  <path d="M80.56 49.48c3.67 4.18 5.78 9.77 5.43 15.85c-.65 11.16-9.83 20.19-21 20.68c-4.75.21-9.18-1.09-12.86-3.45c-.28-.18-.58.2-.34.44a22.412 22.412 0 0 0 17.85 6.67c10.78-.85 19.56-9.5 20.55-20.27c.77-8.36-3.06-15.87-9.23-20.33c-.29-.2-.62.15-.4.41z" fill="#2f7889"></path>
+  <path d="M43.87 65.32c-.67-13.15 7.83-22.79 20.01-22.79c.65 0 1.68 0 2.48.92c1.01 1.18 1.1 2.6 0 3.77c-.81.86-1.95.92-2.53 1c-12.3 1.59-15.18 9.35-15.83 16.77c-.03.33.06 2.35-1.71 2.56c-2.15.25-2.41-1.91-2.42-2.23z" fill="#b9e4ea"></path>
+  <path d="M25.24 65.87c-.01-22.03 15.9-40.19 38.13-41.05c.68-.03 2.45 0 3.55.99c1.01.91 1.38 2.51.79 3.82c-.95 2.11-2.85 2.07-3.36 2.09c-18.51.66-34.18 15.73-34.19 33.95c0 .29-.05.58-.15.84l-.1.25c-.76 1.98-3.52 2.09-4.43.18c-.15-.34-.24-.7-.24-1.07z" fill="#94d1e0"></path>
+</svg>`;
+export function $svg(markup, attrs) {
+    if (!markup.match(/^\s*<svg/)) {
+        throw new Error("Cannot call $svg with non-svg markup.");
+    }
+    return $el(markup, attrs || {});
+}

rgthree-comfy/web/common/shared_utils.js

@@ -0,0 +1,142 @@
+export function getResolver(timeout = 5000) {
+    const resolver = {};
+    resolver.id = generateId(8);
+    resolver.completed = false;
+    resolver.resolved = false;
+    resolver.rejected = false;
+    resolver.promise = new Promise((resolve, reject) => {
+        resolver.reject = (e) => {
+            resolver.completed = true;
+            resolver.rejected = true;
+            reject(e);
+        };
+        resolver.resolve = (data) => {
+            resolver.completed = true;
+            resolver.resolved = true;
+            resolve(data);
+        };
+    });
+    resolver.timeout = setTimeout(() => {
+        if (!resolver.completed) {
+            resolver.reject();
+        }
+    }, timeout);
+    return resolver;
+}
+const DEBOUNCE_FN_TO_PROMISE = new WeakMap();
+export function debounce(fn, ms = 64) {
+    if (!DEBOUNCE_FN_TO_PROMISE.get(fn)) {
+        DEBOUNCE_FN_TO_PROMISE.set(fn, wait(ms).then(() => {
+            DEBOUNCE_FN_TO_PROMISE.delete(fn);
+            fn();
+        }));
+    }
+    return DEBOUNCE_FN_TO_PROMISE.get(fn);
+}
+export function wait(ms = 16) {
+    if (ms === 16) {
+        return new Promise((resolve) => {
+            requestAnimationFrame(() => {
+                resolve();
+            });
+        });
+    }
+    return new Promise((resolve) => {
+        setTimeout(() => {
+            resolve();
+        }, ms);
+    });
+}
+function dec2hex(dec) {
+    return dec.toString(16).padStart(2, "0");
+}
+export function generateId(length) {
+    const arr = new Uint8Array(length / 2);
+    crypto.getRandomValues(arr);
+    return Array.from(arr, dec2hex).join("");
+}
+export function getObjectValue(obj, objKey, def) {
+    if (!obj || !objKey)
+        return def;
+    const keys = objKey.split(".");
+    const key = keys.shift();
+    const found = obj[key];
+    if (keys.length) {
+        return getObjectValue(found, keys.join("."), def);
+    }
+    return found;
+}
+export function setObjectValue(obj, objKey, value, createMissingObjects = true) {
+    if (!obj || !objKey)
+        return obj;
+    const keys = objKey.split(".");
+    const key = keys.shift();
+    if (obj[key] === undefined) {
+        if (!createMissingObjects) {
+            return;
+        }
+        obj[key] = {};
+    }
+    if (!keys.length) {
+        obj[key] = value;
+    }
+    else {
+        if (typeof obj[key] != "object") {
+            obj[key] = {};
+        }
+        setObjectValue(obj[key], keys.join("."), value, createMissingObjects);
+    }
+    return obj;
+}
+export function moveArrayItem(arr, itemOrFrom, to) {
+    const from = typeof itemOrFrom === "number" ? itemOrFrom : arr.indexOf(itemOrFrom);
+    arr.splice(to, 0, arr.splice(from, 1)[0]);
+}
+export function removeArrayItem(arr, itemOrIndex) {
+    const index = typeof itemOrIndex === "number" ? itemOrIndex : arr.indexOf(itemOrIndex);
+    arr.splice(index, 1);
+}
+export function injectCss(href) {
+    if (document.querySelector(`link[href^="${href}"]`)) {
+        return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+        const link = document.createElement("link");
+        link.setAttribute("rel", "stylesheet");
+        link.setAttribute("type", "text/css");
+        const timeout = setTimeout(resolve, 1000);
+        link.addEventListener("load", (e) => {
+            clearInterval(timeout);
+            resolve();
+        });
+        link.href = href;
+        document.head.appendChild(link);
+    });
+}
+export function defineProperty(instance, property, desc) {
+    var _a, _b, _c, _d, _e, _f;
+    const existingDesc = Object.getOwnPropertyDescriptor(instance, property);
+    if ((existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.configurable) === false) {
+        throw new Error(`Error: rgthree-comfy cannot define un-configurable property "${property}"`);
+    }
+    if ((existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.get) && desc.get) {
+        const descGet = desc.get;
+        desc.get = () => {
+            existingDesc.get.apply(instance, []);
+            return descGet.apply(instance, []);
+        };
+    }
+    if ((existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.set) && desc.set) {
+        const descSet = desc.set;
+        desc.set = (v) => {
+            existingDesc.set.apply(instance, [v]);
+            return descSet.apply(instance, [v]);
+        };
+    }
+    desc.enumerable = (_b = (_a = desc.enumerable) !== null && _a !== void 0 ? _a : existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.enumerable) !== null && _b !== void 0 ? _b : true;
+    desc.configurable = (_d = (_c = desc.configurable) !== null && _c !== void 0 ? _c : existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.configurable) !== null && _d !== void 0 ? _d : true;
+    if (!desc.get && !desc.set) {
+        desc.writable = (_f = (_e = desc.writable) !== null && _e !== void 0 ? _e : existingDesc === null || existingDesc === void 0 ? void 0 : existingDesc.writable) !== null && _f !== void 0 ? _f : true;
+    }
+    return Object.defineProperty(instance, property, desc);
+}

rgthree-comfy/web/common/utils_dom.js

@@ -0,0 +1,311 @@
+const DIRECT_ATTRIBUTE_MAP = {
+    cellpadding: 'cellPadding',
+    cellspacing: 'cellSpacing',
+    colspan: 'colSpan',
+    frameborder: 'frameBorder',
+    height: 'height',
+    maxlength: 'maxLength',
+    nonce: 'nonce',
+    role: 'role',
+    rowspan: 'rowSpan',
+    type: 'type',
+    usemap: 'useMap',
+    valign: 'vAlign',
+    width: 'width',
+};
+const RGX_NUMERIC_STYLE_UNIT = 'px';
+const RGX_NUMERIC_STYLE = /^((max|min)?(width|height)|margin|padding|(margin|padding)?(left|top|bottom|right)|fontsize|borderwidth)$/i;
+const RGX_DEFAULT_VALUE_PROP = /input|textarea|select/i;
+function localAssertNotFalsy(input, errorMsg = `Input is not of type.`) {
+    if (input == null) {
+        throw new Error(errorMsg);
+    }
+    return input;
+}
+const RGX_STRING_VALID = '[a-z0-9_-]';
+const RGX_TAG = new RegExp(`^([a-z]${RGX_STRING_VALID}*)(\\.|\\[|\\#|$)`, 'i');
+const RGX_ATTR_ID = new RegExp(`#(${RGX_STRING_VALID}+)`, 'gi');
+const RGX_ATTR_CLASS = new RegExp(`(^|\\S)\\.([a-z0-9_\\-\\.]+)`, 'gi');
+const RGX_STRING_CONTENT_TO_SQUARES = '(.*?)(\\[|\\])';
+const RGX_ATTRS_MAYBE_OPEN = new RegExp(`\\[${RGX_STRING_CONTENT_TO_SQUARES}`, 'gi');
+const RGX_ATTRS_FOLLOW_OPEN = new RegExp(`^${RGX_STRING_CONTENT_TO_SQUARES}`, 'gi');
+export function query(selectors, parent = document) {
+    return Array.from(parent.querySelectorAll(selectors)).filter(n => !!n);
+}
+export function queryOne(selectors, parent = document) {
+    var _a;
+    return (_a = parent.querySelector(selectors)) !== null && _a !== void 0 ? _a : null;
+}
+export function createText(text) {
+    return document.createTextNode(text);
+}
+export function getClosestOrSelf(element, query) {
+    const el = element;
+    return ((el === null || el === void 0 ? void 0 : el.closest) && (el.matches(query) && el || el.closest(query))) || null;
+}
+export function containsOrSelf(parent, contained) {
+    var _a;
+    return parent === contained || ((_a = parent === null || parent === void 0 ? void 0 : parent.contains) === null || _a === void 0 ? void 0 : _a.call(parent, contained)) || false;
+}
+export function createElement(selectorOrMarkup, attrs) {
+    const frag = getHtmlFragment(selectorOrMarkup);
+    let element = frag === null || frag === void 0 ? void 0 : frag.firstElementChild;
+    let selector = "";
+    if (!element) {
+        selector = selectorOrMarkup.replace(/[\r\n]\s*/g, "");
+        const tag = getSelectorTag(selector) || "div";
+        element = document.createElement(tag);
+        selector = selector.replace(RGX_TAG, "$2");
+        selector = selector.replace(RGX_ATTR_ID, '[id="$1"]');
+        selector = selector.replace(RGX_ATTR_CLASS, (match, p1, p2) => `${p1}[class="${p2.replace(/\./g, " ")}"]`);
+    }
+    const selectorAttrs = getSelectorAttributes(selector);
+    if (selectorAttrs) {
+        for (const attr of selectorAttrs) {
+            let matches = attr.substring(1, attr.length - 1).split("=");
+            let key = localAssertNotFalsy(matches.shift());
+            let value = matches.join("=");
+            if (value === undefined) {
+                setAttribute(element, key, true);
+            }
+            else {
+                value = value.replace(/^['"](.*)['"]$/, "$1");
+                setAttribute(element, key, value);
+            }
+        }
+    }
+    if (attrs) {
+        setAttributes(element, attrs);
+    }
+    return element;
+}
+export const $el = createElement;
+function getSelectorTag(str) {
+    return tryMatch(str, RGX_TAG);
+}
+function getSelectorAttributes(selector) {
+    RGX_ATTRS_MAYBE_OPEN.lastIndex = 0;
+    let attrs = [];
+    let result;
+    while (result = RGX_ATTRS_MAYBE_OPEN.exec(selector)) {
+        let attr = result[0];
+        if (attr.endsWith(']')) {
+            attrs.push(attr);
+        }
+        else {
+            attr = result[0]
+                + getOpenAttributesRecursive(selector.substr(RGX_ATTRS_MAYBE_OPEN.lastIndex), 2);
+            RGX_ATTRS_MAYBE_OPEN.lastIndex += (attr.length - result[0].length);
+            attrs.push(attr);
+        }
+    }
+    return attrs;
+}
+function getOpenAttributesRecursive(selectorSubstring, openCount) {
+    let matches = selectorSubstring.match(RGX_ATTRS_FOLLOW_OPEN);
+    let result = '';
+    if (matches && matches.length) {
+        result = matches[0];
+        openCount += result.endsWith(']') ? -1 : 1;
+        if (openCount > 0) {
+            result += getOpenAttributesRecursive(selectorSubstring.substr(result.length), openCount);
+        }
+    }
+    return result;
+}
+function tryMatch(str, rgx, index = 1) {
+    var _a;
+    let found = '';
+    try {
+        found = ((_a = str.match(rgx)) === null || _a === void 0 ? void 0 : _a[index]) || '';
+    }
+    catch (e) {
+        found = '';
+    }
+    return found;
+}
+export function setAttributes(element, data) {
+    let attr;
+    for (attr in data) {
+        if (data.hasOwnProperty(attr)) {
+            setAttribute(element, attr, data[attr]);
+        }
+    }
+}
+function getHtmlFragment(value) {
+    if (value.match(/^\s*<.*?>[\s\S]*<\/[a-z0-9]+>\s*$/)) {
+        return document.createRange().createContextualFragment(value.trim());
+    }
+    return null;
+}
+function getChild(value) {
+    if (value instanceof Node) {
+        return value;
+    }
+    if (typeof value === 'string') {
+        let child = getHtmlFragment(value);
+        if (child) {
+            return child;
+        }
+        if (getSelectorTag(value)) {
+            return createElement(value);
+        }
+        return createText(value);
+    }
+    if (value && typeof value.toElement === 'function') {
+        return value.toElement();
+    }
+    return null;
+}
+export function setAttribute(element, attribute, value) {
+    let isRemoving = value == null;
+    if (attribute === 'default') {
+        attribute = RGX_DEFAULT_VALUE_PROP.test(element.nodeName) ? 'value' : 'text';
+    }
+    if (attribute === 'text') {
+        empty(element).appendChild(createText(value != null ? String(value) : ''));
+    }
+    else if (attribute === 'html') {
+        empty(element).innerHTML += value != null ? String(value) : '';
+    }
+    else if (attribute == 'style') {
+        if (typeof value === 'string') {
+            element.style.cssText = isRemoving ? '' : (value != null ? String(value) : '');
+        }
+        else {
+            for (const [styleKey, styleValue] of Object.entries(value)) {
+                element.style[styleKey] = styleValue;
+            }
+        }
+    }
+    else if (attribute == 'events') {
+        for (const [key, fn] of Object.entries(value)) {
+            addEvent(element, key, fn);
+        }
+    }
+    else if (attribute === 'parent') {
+        value.appendChild(element);
+    }
+    else if (attribute === 'child' || attribute === 'children') {
+        if (typeof value === 'string' && /^\[[^\[\]]+\]$/.test(value)) {
+            const parseable = value.replace(/^\[([^\[\]]+)\]$/, '["$1"]').replace(/,/g, '","');
+            try {
+                const parsed = JSON.parse(parseable);
+                value = parsed;
+            }
+            catch (e) {
+                console.error(e);
+            }
+        }
+        if (attribute === 'children') {
+            empty(element);
+        }
+        let children = value instanceof Array ? value : [value];
+        for (let child of children) {
+            child = getChild(child);
+            if (child instanceof Node) {
+                if (element instanceof HTMLTemplateElement) {
+                    element.content.appendChild(child);
+                }
+                else {
+                    element.appendChild(child);
+                }
+            }
+        }
+    }
+    else if (attribute == 'for') {
+        element.htmlFor = value != null ? String(value) : '';
+        if (isRemoving) {
+            element.removeAttribute('for');
+        }
+    }
+    else if (attribute === 'class' || attribute === 'className' || attribute === 'classes') {
+        element.className = isRemoving ? '' : Array.isArray(value) ? value.join(' ') : String(value);
+    }
+    else if (attribute === 'dataset') {
+        if (typeof value !== 'object') {
+            console.error('Expecting an object for dataset');
+            return;
+        }
+        for (const [key, val] of Object.entries(value)) {
+            element.dataset[key] = String(val);
+        }
+    }
+    else if (attribute.startsWith('on') && typeof value === 'function') {
+        element.addEventListener(attribute.substring(2), value);
+    }
+    else if (['checked', 'disabled', 'readonly', 'required', 'selected'].includes(attribute)) {
+        element[attribute] = !!value;
+        if (!value) {
+            element.removeAttribute(attribute);
+        }
+        else {
+            element.setAttribute(attribute, attribute);
+        }
+    }
+    else if (DIRECT_ATTRIBUTE_MAP.hasOwnProperty(attribute)) {
+        if (isRemoving) {
+            element.removeAttribute(DIRECT_ATTRIBUTE_MAP[attribute]);
+        }
+        else {
+            element.setAttribute(DIRECT_ATTRIBUTE_MAP[attribute], String(value));
+        }
+    }
+    else if (isRemoving) {
+        element.removeAttribute(attribute);
+    }
+    else {
+        let oldVal = element.getAttribute(attribute);
+        if (oldVal !== value) {
+            element.setAttribute(attribute, String(value));
+        }
+    }
+}
+function addEvent(element, key, fn) {
+    element.addEventListener(key, fn);
+}
+function setStyles(element, styles = null) {
+    if (styles) {
+        for (let name in styles) {
+            setStyle(element, name, styles[name]);
+        }
+    }
+    return element;
+}
+function setStyle(element, name, value) {
+    name = (name.indexOf('float') > -1 ? 'cssFloat' : name);
+    if (name.indexOf('-') != -1) {
+        name = name.replace(/-\D/g, (match) => {
+            return match.charAt(1).toUpperCase();
+        });
+    }
+    if (value == String(Number(value)) && RGX_NUMERIC_STYLE.test(name)) {
+        value = value + RGX_NUMERIC_STYLE_UNIT;
+    }
+    if (name === 'display' && typeof value !== 'string') {
+        value = !!value ? null : 'none';
+    }
+    element.style[name] = value === null ? null : String(value);
+    return element;
+}
+;
+export function empty(element) {
+    while (element.firstChild) {
+        element.removeChild(element.firstChild);
+    }
+    return element;
+}
+export function appendChildren(el, children) {
+    children = !Array.isArray(children) ? [children] : children;
+    for (let child of children) {
+        child = getChild(child);
+        if (child instanceof Node) {
+            if (el instanceof HTMLTemplateElement) {
+                el.content.appendChild(child);
+            }
+            else {
+                el.appendChild(child);
+            }
+        }
+    }
+}

rgthree-comfy/web/common/utils_workflow.js

@@ -0,0 +1,55 @@
+import { getResolver } from "./shared_utils.js";
+import { getPngMetadata, getWebpMetadata } from "../../scripts/pnginfo.js";
+function parseWorkflowJson(stringJson) {
+    stringJson = stringJson || "null";
+    stringJson = stringJson.replace(/:\s*NaN/g, ": null");
+    return JSON.parse(stringJson);
+}
+export async function tryToGetWorkflowDataFromEvent(e) {
+    var _a, _b, _c, _d;
+    let work;
+    for (const file of ((_a = e.dataTransfer) === null || _a === void 0 ? void 0 : _a.files) || []) {
+        const data = await tryToGetWorkflowDataFromFile(file);
+        if (data.workflow || data.prompt) {
+            return data;
+        }
+    }
+    const validTypes = ["text/uri-list", "text/x-moz-url"];
+    const match = (((_b = e.dataTransfer) === null || _b === void 0 ? void 0 : _b.types) || []).find((t) => validTypes.find((v) => t === v));
+    if (match) {
+        const uri = (_d = (_c = e.dataTransfer.getData(match)) === null || _c === void 0 ? void 0 : _c.split("\n")) === null || _d === void 0 ? void 0 : _d[0];
+        if (uri) {
+            return tryToGetWorkflowDataFromFile(await (await fetch(uri)).blob());
+        }
+    }
+    return { workflow: null, prompt: null };
+}
+export async function tryToGetWorkflowDataFromFile(file) {
+    var _a;
+    if (file.type === "image/png") {
+        const pngInfo = await getPngMetadata(file);
+        return {
+            workflow: parseWorkflowJson(pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.workflow),
+            prompt: parseWorkflowJson(pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.prompt),
+        };
+    }
+    if (file.type === "image/webp") {
+        const pngInfo = await getWebpMetadata(file);
+        const workflow = parseWorkflowJson((pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.workflow) || (pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.Workflow) || "null");
+        const prompt = parseWorkflowJson((pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.prompt) || (pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.Prompt) || "null");
+        return { workflow, prompt };
+    }
+    if (file.type === "application/json" || ((_a = file.name) === null || _a === void 0 ? void 0 : _a.endsWith(".json"))) {
+        const resolver = getResolver();
+        const reader = new FileReader();
+        reader.onload = async () => {
+            const json = parseWorkflowJson(reader.result);
+            const isApiJson = Object.values(json).every((v) => v.class_type);
+            const prompt = isApiJson ? json : null;
+            const workflow = !isApiJson && !(json === null || json === void 0 ? void 0 : json.templates) ? json : null;
+            return { workflow, prompt };
+        };
+        return resolver.promise;
+    }
+    return { workflow: null, prompt: null };
+}

rgthree-comfy/web/link_fixer/link_page.js

@@ -0,0 +1,195 @@
+import { fixBadLinks } from "../common/link_fixer.js";
+import { getPngMetadata } from "../../scripts/pnginfo.js";
+function wait(ms = 16, value) {
+    return new Promise((resolve) => {
+        setTimeout(() => {
+            resolve(value);
+        }, ms);
+    });
+}
+const logger = {
+    logTo: console,
+    log: (...args) => {
+        logger.logTo === console
+            ? console.log(...args)
+            : (logger.logTo.innerText += args.join(",") + "\n");
+    },
+};
+const findBadLinksLogger = {
+    log: async (...args) => {
+        logger.log(...args);
+    },
+};
+export class LinkPage {
+    constructor() {
+        this.containerEl = document.querySelector(".box");
+        this.figcaptionEl = document.querySelector("figcaption");
+        this.outputeMessageEl = document.querySelector(".output");
+        this.outputImageEl = document.querySelector(".output-image");
+        this.btnFix = document.querySelector(".btn-fix");
+        document.addEventListener("dragover", (e) => {
+            e.preventDefault();
+        }, false);
+        document.addEventListener("drop", (e) => {
+            this.onDrop(e);
+        });
+        this.btnFix.addEventListener("click", (e) => {
+            this.onFixClick(e);
+        });
+    }
+    async onFixClick(e) {
+        if (!this.graphResults || !this.graph) {
+            this.updateUi("⛔ Fix button click without results.");
+            return;
+        }
+        let graphFinalResults = fixBadLinks(this.graph, true);
+        graphFinalResults = fixBadLinks(graphFinalResults.graph, true);
+        if (graphFinalResults.patched || graphFinalResults.deleted) {
+            graphFinalResults = fixBadLinks(graphFinalResults.graph, true);
+        }
+        this.graphFinalResults = graphFinalResults;
+        await this.saveFixedWorkflow();
+        if (graphFinalResults.hasBadLinks) {
+            this.updateUi("⛔ Hmm... Still detecting bad links. Can you file an issue at https://github.com/rgthree/rgthree-comfy/issues with your image/workflow.");
+        }
+        else {
+            this.updateUi("✅ Workflow fixed.<br><br><small>Please load new saved workflow json and double check linking and execution.</small>");
+        }
+    }
+    async onDrop(event) {
+        var _a, _b, _c, _d;
+        if (!event.dataTransfer) {
+            return;
+        }
+        this.reset();
+        event.preventDefault();
+        event.stopPropagation();
+        if (event.dataTransfer.files.length && ((_b = (_a = event.dataTransfer.files) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.type) !== "image/bmp") {
+            await this.handleFile(event.dataTransfer.files[0]);
+            return;
+        }
+        const validTypes = ["text/uri-list", "text/x-moz-url"];
+        const match = [...event.dataTransfer.types].find((t) => validTypes.find((v) => t === v));
+        if (match) {
+            const uri = (_d = (_c = event.dataTransfer.getData(match)) === null || _c === void 0 ? void 0 : _c.split("\n")) === null || _d === void 0 ? void 0 : _d[0];
+            if (uri) {
+                await this.handleFile(await (await fetch(uri)).blob());
+            }
+        }
+    }
+    reset() {
+        this.file = undefined;
+        this.graph = undefined;
+        this.graphResults = undefined;
+        this.graphFinalResults = undefined;
+        this.updateUi();
+    }
+    updateUi(msg) {
+        this.outputeMessageEl.innerHTML = "";
+        if (this.file && !this.containerEl.classList.contains("-has-file")) {
+            this.containerEl.classList.add("-has-file");
+            this.figcaptionEl.innerHTML = this.file.name || this.file.type;
+            if (this.file.type === "application/json") {
+                this.outputImageEl.src = "icon_file_json.png";
+            }
+            else {
+                const reader = new FileReader();
+                reader.onload = () => (this.outputImageEl.src = reader.result);
+                reader.readAsDataURL(this.file);
+            }
+        }
+        else if (!this.file && this.containerEl.classList.contains("-has-file")) {
+            this.containerEl.classList.remove("-has-file");
+            this.outputImageEl.src = "";
+            this.outputImageEl.removeAttribute("src");
+        }
+        if (this.graphResults) {
+            this.containerEl.classList.add("-has-results");
+            if (!this.graphResults.patched && !this.graphResults.deleted) {
+                this.outputeMessageEl.innerHTML = "✅ No bad links detected in the workflow.";
+            }
+            else {
+                this.containerEl.classList.add("-has-fixable-results");
+                this.outputeMessageEl.innerHTML = `⚠️ Found ${this.graphResults.patched} links to fix, and ${this.graphResults.deleted} to be removed.`;
+            }
+        }
+        else {
+            this.containerEl.classList.remove("-has-results");
+            this.containerEl.classList.remove("-has-fixable-results");
+        }
+        if (msg) {
+            this.outputeMessageEl.innerHTML = msg;
+        }
+    }
+    async handleFile(file) {
+        this.file = file;
+        this.updateUi();
+        let workflow = null;
+        if (file.type.startsWith("image/")) {
+            const pngInfo = await getPngMetadata(file);
+            workflow = pngInfo === null || pngInfo === void 0 ? void 0 : pngInfo.workflow;
+        }
+        else if (file.type === "application/json" ||
+            (file instanceof File && file.name.endsWith(".json"))) {
+            workflow = await new Promise((resolve) => {
+                const reader = new FileReader();
+                reader.onload = () => {
+                    resolve(reader.result);
+                };
+                reader.readAsText(file);
+            });
+        }
+        if (!workflow) {
+            this.updateUi("⛔ No workflow found in dropped item.");
+        }
+        else {
+            try {
+                this.graph = JSON.parse(workflow);
+            }
+            catch (e) {
+                this.graph = undefined;
+            }
+            if (!this.graph) {
+                this.updateUi("⛔ Invalid workflow found in dropped item.");
+            }
+            else {
+                this.loadGraphData(this.graph);
+            }
+        }
+    }
+    async loadGraphData(graphData) {
+        this.graphResults = await fixBadLinks(graphData);
+        this.updateUi();
+    }
+    async saveFixedWorkflow() {
+        if (!this.graphFinalResults) {
+            this.updateUi("⛔ Save w/o final graph patched.");
+            return false;
+        }
+        let filename = this.file.name || "workflow.json";
+        let filenames = filename.split(".");
+        filenames.pop();
+        filename = filenames.join(".");
+        filename += "_fixed.json";
+        filename = prompt("Save workflow as:", filename);
+        if (!filename)
+            return false;
+        if (!filename.toLowerCase().endsWith(".json")) {
+            filename += ".json";
+        }
+        const json = JSON.stringify(this.graphFinalResults.graph, null, 2);
+        const blob = new Blob([json], { type: "application/json" });
+        const url = URL.createObjectURL(blob);
+        const anchor = document.createElement("a");
+        anchor.download = filename;
+        anchor.href = url;
+        anchor.style.display = "none";
+        document.body.appendChild(anchor);
+        await wait();
+        anchor.click();
+        await wait();
+        anchor.remove();
+        window.URL.revokeObjectURL(url);
+        return true;
+    }
+}

sd-dynamic-thresholding/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: mcmonkey4eva

sd-dynamic-thresholding/.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

sd-dynamic-thresholding/javascript/active.js

@@ -0,0 +1,68 @@
+let dynthres_update_enabled = function() {
+    return Array.from(arguments);
+};
+
+(function(){
+    let accordions = {};
+    let enabled = {};
+    onUiUpdate(() => {
+        let accordion_id_prefix = "#dynthres_";
+        let extension_checkbox_class = ".dynthres-enabled";
+
+        dynthres_update_enabled = function() {
+            let res = Array.from(arguments);
+            let tabname = res[1] ? "img2img" : "txt2img";
+
+            let checkbox = accordions[tabname]?.querySelector(extension_checkbox_class + ' input');
+            checkbox?.dispatchEvent(new Event('change'));
+
+            return res;
+        };
+
+        function attachEnabledButtonListener(checkbox, accordion) {
+            let span = accordion.querySelector('.label-wrap span');
+            let badge = document.createElement('input');
+            badge.type = "checkbox";
+            badge.checked = checkbox.checked;
+            badge.addEventListener('click', (e) => {
+                checkbox.checked = !checkbox.checked;
+                badge.checked = checkbox.checked;
+                checkbox.dispatchEvent(new Event('change'));
+                e.stopPropagation();
+            });
+
+            badge.className = checkbox.className;
+            badge.classList.add('primary');
+            span.insertBefore(badge, span.firstChild);
+            let space = document.createElement('span');
+            space.innerHTML = " ";
+            span.insertBefore(space, badge.nextSibling);
+
+            checkbox.addEventListener('change', () => {
+                let badge = accordion.querySelector('.label-wrap span input');
+                badge.checked = checkbox.checked;
+            });
+            checkbox.parentNode.style.display = "none";
+        }
+
+        if (Object.keys(accordions).length < 2) {
+            let accordion = gradioApp().querySelector(accordion_id_prefix + 'txt2img');
+            if (accordion) {
+                accordions.txt2img = accordion;
+            }
+            accordion = gradioApp().querySelector(accordion_id_prefix + 'img2img');
+            if (accordion) {
+                accordions.img2img = accordion;
+            }
+        }
+
+        if (Object.keys(accordions).length > 0 && accordions.txt2img && !enabled.txt2img) {
+            enabled.txt2img = accordions.txt2img.querySelector(extension_checkbox_class + ' input');
+            attachEnabledButtonListener(enabled.txt2img, accordions.txt2img);
+        }
+        if (Object.keys(accordions).length > 0 && accordions.img2img && !enabled.img2img) {
+            enabled.img2img = accordions.img2img.querySelector(extension_checkbox_class + ' input');
+            attachEnabledButtonListener(enabled.img2img, accordions.img2img);
+        }
+    });
+})();

sd-dynamic-thresholding/scripts/dynamic_thresholding.py

@@ -0,0 +1,270 @@
+##################
+# Stable Diffusion Dynamic Thresholding (CFG Scale Fix)
+#
+# Author: Alex 'mcmonkey' Goodwin
+# GitHub URL: https://github.com/mcmonkeyprojects/sd-dynamic-thresholding
+# Created: 2022/01/26
+# Last updated: 2023/01/30
+#
+# For usage help, view the README.md file in the extension root, or via the GitHub page.
+#
+##################
+
+import gradio as gr
+import torch, traceback
+import dynthres_core
+from modules import scripts, script_callbacks, sd_samplers, sd_samplers_compvis, sd_samplers_common
+try:
+    import dynthres_unipc
+except Exception as e:
+    print(f"\n\n======\nError! UniPC sampler support failed to load! Is your WebUI up to date?\n(Error: {e})\n======")
+try:
+    from modules.sd_samplers_kdiffusion import CFGDenoiserKDiffusion as cfgdenoisekdiff
+    IS_AUTO_16 = True
+except Exception as e:
+    print(f"\n\n======\nWarning! Using legacy KDiff version! Is your WebUI up to date?\n======")
+    from modules.sd_samplers_kdiffusion import CFGDenoiser as cfgdenoisekdiff
+    IS_AUTO_16 = False
+
+DISABLE_VISIBILITY = True
+
+######################### Data values #########################
+MODES_WITH_VALUE = ["Power Up", "Power Down", "Linear Repeating", "Cosine Repeating", "Sawtooth"]
+
+######################### Script class entrypoint #########################
+class Script(scripts.Script):
+
+    def title(self):
+        return "Dynamic Thresholding (CFG Scale Fix)"
+
+    def show(self, is_img2img):
+        return scripts.AlwaysVisible
+
+    def ui(self, is_img2img):
+        def vis_change(is_vis):
+            return {"visible": is_vis, "__type__": "update"}
+        # "Dynamic Thresholding (CFG Scale Fix)"
+        dtrue = gr.Checkbox(value=True, visible=False)
+        dfalse = gr.Checkbox(value=False, visible=False)
+        with gr.Accordion("Dynamic Thresholding (CFG Scale Fix)", open=False, elem_id="dynthres_" + ("img2img" if is_img2img else "txt2img")):
+            with gr.Row():
+                enabled = gr.Checkbox(value=False, label="Enable Dynamic Thresholding (CFG Scale Fix)", elem_classes=["dynthres-enabled"], elem_id='dynthres_enabled')
+            with gr.Group():
+                gr.HTML(value=f"View <a style=\"border-bottom: 1px #00ffff dotted;\" href=\"https://github.com/mcmonkeyprojects/sd-dynamic-thresholding/wiki/Usage-Tips\">the wiki for usage tips.</a><br><br>", elem_id='dynthres_wiki_link')
+                mimic_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='Mimic CFG Scale', value=7.0, elem_id='dynthres_mimic_scale')
+                with gr.Accordion("Advanced Options", open=False, elem_id='dynthres_advanced_opts'):
+                    with gr.Row():
+                        threshold_percentile = gr.Slider(minimum=90.0, value=100.0, maximum=100.0, step=0.05, label='Top percentile of latents to clamp', elem_id='dynthres_threshold_percentile')
+                        interpolate_phi = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Interpolate Phi", value=1.0, elem_id='dynthres_interpolate_phi')
+                    with gr.Row():
+                        mimic_mode = gr.Dropdown(dynthres_core.DynThresh.Modes, value="Constant", label="Mimic Scale Scheduler", elem_id='dynthres_mimic_mode')
+                        cfg_mode = gr.Dropdown(dynthres_core.DynThresh.Modes, value="Constant", label="CFG Scale Scheduler", elem_id='dynthres_cfg_mode')
+                    mimic_scale_min = gr.Slider(minimum=0.0, maximum=30.0, step=0.5, visible=DISABLE_VISIBILITY, label="Minimum value of the Mimic Scale Scheduler", elem_id='dynthres_mimic_scale_min')
+                    cfg_scale_min = gr.Slider(minimum=0.0, maximum=30.0, step=0.5, visible=DISABLE_VISIBILITY, label="Minimum value of the CFG Scale Scheduler", elem_id='dynthres_cfg_scale_min')
+                    sched_val = gr.Slider(minimum=0.0, maximum=40.0, step=0.5, value=4.0, visible=DISABLE_VISIBILITY, label="Scheduler Value", info="Value unique to the scheduler mode - for Power Up/Down, this is the power. For Linear/Cosine Repeating, this is the number of repeats per image.", elem_id='dynthres_sched_val')
+                    with gr.Row():
+                        separate_feature_channels = gr.Checkbox(value=True, label="Separate Feature Channels", elem_id='dynthres_separate_feature_channels')
+                        scaling_startpoint = gr.Radio(["ZERO", "MEAN"], value="MEAN", label="Scaling Startpoint")
+                        variability_measure = gr.Radio(["STD", "AD"], value="AD", label="Variability Measure")
+        def should_show_scheduler_value(cfg_mode, mimic_mode):
+            sched_vis = cfg_mode in MODES_WITH_VALUE or mimic_mode in MODES_WITH_VALUE or DISABLE_VISIBILITY
+            return vis_change(sched_vis), vis_change(mimic_mode != "Constant" or DISABLE_VISIBILITY), vis_change(cfg_mode != "Constant" or DISABLE_VISIBILITY)
+        cfg_mode.change(should_show_scheduler_value, inputs=[cfg_mode, mimic_mode], outputs=[sched_val, mimic_scale_min, cfg_scale_min])
+        mimic_mode.change(should_show_scheduler_value, inputs=[cfg_mode, mimic_mode], outputs=[sched_val, mimic_scale_min, cfg_scale_min])
+        enabled.change(
+            _js="dynthres_update_enabled",
+            fn=None,
+            inputs=[enabled, dtrue if is_img2img else dfalse],
+            show_progress = False)
+        self.infotext_fields = (
+            (enabled, lambda d: gr.Checkbox.update(value="Dynamic thresholding enabled" in d)),
+            (mimic_scale, "Mimic scale"),
+            (separate_feature_channels, "Separate Feature Channels"),
+            (scaling_startpoint, lambda d: gr.Radio.update(value=d.get("Scaling Startpoint", "MEAN"))),
+            (variability_measure, lambda d: gr.Radio.update(value=d.get("Variability Measure", "AD"))),
+            (interpolate_phi, "Interpolate Phi"),
+            (threshold_percentile, "Threshold percentile"),
+            (mimic_scale_min, "Mimic scale minimum"),
+            (mimic_mode, lambda d: gr.Dropdown.update(value=d.get("Mimic mode", "Constant"))),
+            (cfg_mode, lambda d: gr.Dropdown.update(value=d.get("CFG mode", "Constant"))),
+            (cfg_scale_min, "CFG scale minimum"),
+            (sched_val, "Scheduler value"))
+        return [enabled, mimic_scale, threshold_percentile, mimic_mode, mimic_scale_min, cfg_mode, cfg_scale_min, sched_val, separate_feature_channels, scaling_startpoint, variability_measure, interpolate_phi]
+
+    last_id = 0
+
+    def process_batch(self, p, enabled, mimic_scale, threshold_percentile, mimic_mode, mimic_scale_min, cfg_mode, cfg_scale_min, sched_val, separate_feature_channels, scaling_startpoint, variability_measure, interpolate_phi, batch_number, prompts, seeds, subseeds):
+        enabled = getattr(p, 'dynthres_enabled', enabled)
+        if not enabled:
+            return
+        orig_sampler_name = p.sampler_name
+        orig_latent_sampler_name = getattr(p, 'latent_sampler', None)
+        if orig_sampler_name in ["DDIM", "PLMS"]:
+            raise RuntimeError(f"Cannot use sampler {orig_sampler_name} with Dynamic Thresholding")
+        if orig_latent_sampler_name in ["DDIM", "PLMS"]:
+            raise RuntimeError(f"Cannot use secondary sampler {orig_latent_sampler_name} with Dynamic Thresholding")
+        if 'UniPC' in (orig_sampler_name, orig_latent_sampler_name) and p.enable_hr:
+            raise RuntimeError(f"UniPC does not support Hires Fix. Auto WebUI silently swaps to DDIM for this, which DynThresh does not support. Please swap to a sampler capable of img2img processing for HR Fix to work.")
+        mimic_scale = getattr(p, 'dynthres_mimic_scale', mimic_scale)
+        separate_feature_channels = getattr(p, 'dynthres_separate_feature_channels', separate_feature_channels)
+        scaling_startpoint = getattr(p, 'dynthres_scaling_startpoint', scaling_startpoint)
+        variability_measure = getattr(p, 'dynthres_variability_measure', variability_measure)
+        interpolate_phi = getattr(p, 'dynthres_interpolate_phi', interpolate_phi)
+        threshold_percentile = getattr(p, 'dynthres_threshold_percentile', threshold_percentile)
+        mimic_mode = getattr(p, 'dynthres_mimic_mode', mimic_mode)
+        mimic_scale_min = getattr(p, 'dynthres_mimic_scale_min', mimic_scale_min)
+        cfg_mode = getattr(p, 'dynthres_cfg_mode', cfg_mode)
+        cfg_scale_min = getattr(p, 'dynthres_cfg_scale_min', cfg_scale_min)
+        experiment_mode = getattr(p, 'dynthres_experiment_mode', 0)
+        sched_val = getattr(p, 'dynthres_scheduler_val', sched_val)
+        p.extra_generation_params["Dynamic thresholding enabled"] = True
+        p.extra_generation_params["Mimic scale"] = mimic_scale
+        p.extra_generation_params["Separate Feature Channels"] = separate_feature_channels
+        p.extra_generation_params["Scaling Startpoint"] = scaling_startpoint
+        p.extra_generation_params["Variability Measure"] = variability_measure
+        p.extra_generation_params["Interpolate Phi"] = interpolate_phi
+        p.extra_generation_params["Threshold percentile"] = threshold_percentile
+        p.extra_generation_params["Sampler"] = orig_sampler_name
+        if mimic_mode != "Constant":
+            p.extra_generation_params["Mimic mode"] = mimic_mode
+            p.extra_generation_params["Mimic scale minimum"] = mimic_scale_min
+        if cfg_mode != "Constant":
+            p.extra_generation_params["CFG mode"] = cfg_mode
+            p.extra_generation_params["CFG scale minimum"] = cfg_scale_min
+        if cfg_mode in MODES_WITH_VALUE or mimic_mode in MODES_WITH_VALUE:
+            p.extra_generation_params["Scheduler value"] = sched_val
+        # Note: the ID number is to protect the edge case of multiple simultaneous runs with different settings
+        Script.last_id += 1
+        # Percentage to portion
+        threshold_percentile *= 0.01
+
+        def make_sampler(orig_sampler_name):
+            fixed_sampler_name = f"{orig_sampler_name}_dynthres{Script.last_id}"
+
+            # Make a placeholder sampler
+            sampler = sd_samplers.all_samplers_map[orig_sampler_name]
+            dt_data = dynthres_core.DynThresh(mimic_scale, threshold_percentile, mimic_mode, mimic_scale_min, cfg_mode, cfg_scale_min, sched_val, experiment_mode, p.steps, separate_feature_channels, scaling_startpoint, variability_measure, interpolate_phi)
+            if orig_sampler_name == "UniPC":
+                def unipc_constructor(model):
+                    return CustomVanillaSDSampler(dynthres_unipc.CustomUniPCSampler, model, dt_data)
+                new_sampler = sd_samplers_common.SamplerData(fixed_sampler_name, unipc_constructor, sampler.aliases, sampler.options)
+            else:
+                def new_constructor(model):
+                    result = sampler.constructor(model)
+                    cfg = CustomCFGDenoiser(result if IS_AUTO_16 else result.model_wrap_cfg.inner_model, dt_data)
+                    result.model_wrap_cfg = cfg
+                    return result
+                new_sampler = sd_samplers_common.SamplerData(fixed_sampler_name, new_constructor, sampler.aliases, sampler.options)
+            return fixed_sampler_name, new_sampler
+
+        # Apply for usage
+        p.orig_sampler_name = orig_sampler_name
+        p.orig_latent_sampler_name = orig_latent_sampler_name
+        p.fixed_samplers = []
+
+        if orig_latent_sampler_name:
+            latent_sampler_name, latent_sampler = make_sampler(orig_latent_sampler_name)
+            sd_samplers.all_samplers_map[latent_sampler_name] = latent_sampler
+            p.fixed_samplers.append(latent_sampler_name)
+            p.latent_sampler = latent_sampler_name
+
+        if orig_sampler_name != orig_latent_sampler_name:
+            p.sampler_name, new_sampler = make_sampler(orig_sampler_name)
+            sd_samplers.all_samplers_map[p.sampler_name] = new_sampler
+            p.fixed_samplers.append(p.sampler_name)
+        else:
+            p.sampler_name = p.latent_sampler
+
+        if p.sampler is not None:
+            p.sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model)
+
+    def postprocess_batch(self, p, enabled, mimic_scale, threshold_percentile, mimic_mode, mimic_scale_min, cfg_mode, cfg_scale_min, sched_val, separate_feature_channels, scaling_startpoint, variability_measure, interpolate_phi, batch_number, images):
+        if not enabled or not hasattr(p, 'orig_sampler_name'):
+            return
+        p.sampler_name = p.orig_sampler_name
+        if p.orig_latent_sampler_name:
+            p.latent_sampler = p.orig_latent_sampler_name
+        for added_sampler in p.fixed_samplers:
+            del sd_samplers.all_samplers_map[added_sampler]
+        del p.fixed_samplers
+        del p.orig_sampler_name
+        del p.orig_latent_sampler_name
+
+######################### CompVis Implementation logic #########################
+
+class CustomVanillaSDSampler(sd_samplers_compvis.VanillaStableDiffusionSampler):
+    def __init__(self, constructor, sd_model, dt_data):
+        super().__init__(constructor, sd_model)
+        self.sampler.main_class = dt_data
+
+######################### K-Diffusion Implementation logic #########################
+
+class CustomCFGDenoiser(cfgdenoisekdiff):
+    def __init__(self, model, dt_data):
+        super().__init__(model)
+        self.main_class = dt_data
+
+    def combine_denoised(self, x_out, conds_list, uncond, cond_scale):
+        if isinstance(uncond, dict) and 'crossattn' in uncond:
+            uncond = uncond['crossattn']
+        denoised_uncond = x_out[-uncond.shape[0]:]
+        # conds_list shape is (batch, cond, 2)
+        weights = torch.tensor(conds_list, device=uncond.device).select(2, 1)
+        weights = weights.reshape(*weights.shape, 1, 1, 1)
+        self.main_class.step = self.step
+        if hasattr(self, 'total_steps'):
+            self.main_class.max_steps = self.total_steps
+
+        if self.main_class.experiment_mode >= 4 and self.main_class.experiment_mode <= 5:
+            # https://arxiv.org/pdf/2305.08891.pdf "Rescale CFG". It's not good, but if you want to test it, just set experiment_mode = 4 + phi.
+            denoised = torch.clone(denoised_uncond)
+            fi = self.main_class.experiment_mode - 4.0
+            for i, conds in enumerate(conds_list):
+                for cond_index, weight in conds:
+                    xcfg = (denoised_uncond[i] + (x_out[cond_index] - denoised_uncond[i]) * (cond_scale * weight))
+                    xrescaled = xcfg * (torch.std(x_out[cond_index]) / torch.std(xcfg))
+                    xfinal = fi * xrescaled + (1.0 - fi) * xcfg
+                    denoised[i] = xfinal
+            return denoised
+
+        return self.main_class.dynthresh(x_out[:-uncond.shape[0]], denoised_uncond, cond_scale, weights)
+
+######################### XYZ Plot Script Support logic #########################
+
+def make_axis_options():
+    xyz_grid = [x for x in scripts.scripts_data if x.script_class.__module__ in ("xyz_grid.py", "scripts.xyz_grid")][0].module
+    def apply_mimic_scale(p, x, xs):
+        if x != 0:
+            setattr(p, "dynthres_enabled", True)
+            setattr(p, "dynthres_mimic_scale", x)
+        else:
+            setattr(p, "dynthres_enabled", False)
+    def confirm_scheduler(p, xs):
+        for x in xs:
+            if x not in dynthres_core.DynThresh.Modes:
+                raise RuntimeError(f"Unknown Scheduler: {x}")
+    extra_axis_options = [
+        xyz_grid.AxisOption("[DynThres] Mimic Scale", float, apply_mimic_scale),
+        xyz_grid.AxisOption("[DynThres] Separate Feature Channels", int,
+                            xyz_grid.apply_field("dynthres_separate_feature_channels")),
+        xyz_grid.AxisOption("[DynThres] Scaling Startpoint", str, xyz_grid.apply_field("dynthres_scaling_startpoint"), choices=lambda:['ZERO', 'MEAN']),
+        xyz_grid.AxisOption("[DynThres] Variability Measure", str, xyz_grid.apply_field("dynthres_variability_measure"), choices=lambda:['STD', 'AD']),
+        xyz_grid.AxisOption("[DynThres] Interpolate Phi", float, xyz_grid.apply_field("dynthres_interpolate_phi")),
+        xyz_grid.AxisOption("[DynThres] Threshold Percentile", float, xyz_grid.apply_field("dynthres_threshold_percentile")),
+        xyz_grid.AxisOption("[DynThres] Mimic Scheduler", str, xyz_grid.apply_field("dynthres_mimic_mode"), confirm=confirm_scheduler, choices=lambda: dynthres_core.DynThresh.Modes),
+        xyz_grid.AxisOption("[DynThres] Mimic minimum", float, xyz_grid.apply_field("dynthres_mimic_scale_min")),
+        xyz_grid.AxisOption("[DynThres] CFG Scheduler", str, xyz_grid.apply_field("dynthres_cfg_mode"), confirm=confirm_scheduler, choices=lambda: dynthres_core.DynThresh.Modes),
+        xyz_grid.AxisOption("[DynThres] CFG minimum", float, xyz_grid.apply_field("dynthres_cfg_scale_min")),
+        xyz_grid.AxisOption("[DynThres] Scheduler value", float, xyz_grid.apply_field("dynthres_scheduler_val"))
+    ]
+    if not any("[DynThres]" in x.label for x in xyz_grid.axis_options):
+        xyz_grid.axis_options.extend(extra_axis_options)
+
+def callback_before_ui():
+    try:
+        make_axis_options()
+    except Exception as e:
+        traceback.print_exc()
+        print(f"Failed to add support for X/Y/Z Plot Script because: {e}")
+
+script_callbacks.on_before_ui(callback_before_ui)

sigmas_tools_and_the_golden_scheduler/.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

stable-diffusion-temperature-settings/.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: extraltodeus

stable-diffusion-temperature-settings/.github/workflows/publish.yml

@@ -0,0 +1,22 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - master
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

ultimate-upscale-for-automatic1111/scripts/ultimate-upscale.py

@@ -0,0 +1,569 @@
+import math
+import gradio as gr
+from PIL import Image, ImageDraw, ImageOps
+from modules import processing, shared, images, devices, scripts
+from modules.processing import StableDiffusionProcessing
+from modules.processing import Processed
+from modules.shared import opts, state
+from enum import Enum
+
+elem_id_prefix = "ultimateupscale"
+
+class USDUMode(Enum):
+    LINEAR = 0
+    CHESS = 1
+    NONE = 2
+
+class USDUSFMode(Enum):
+    NONE = 0
+    BAND_PASS = 1
+    HALF_TILE = 2
+    HALF_TILE_PLUS_INTERSECTIONS = 3
+
+class USDUpscaler():
+
+    def __init__(self, p, image, upscaler_index:int, save_redraw, save_seams_fix, tile_width, tile_height) -> None:
+        self.p:StableDiffusionProcessing = p
+        self.image:Image = image
+        self.scale_factor = math.ceil(max(p.width, p.height) / max(image.width, image.height))
+        self.upscaler = shared.sd_upscalers[upscaler_index]
+        self.redraw = USDURedraw()
+        self.redraw.save = save_redraw
+        self.redraw.tile_width = tile_width if tile_width > 0 else tile_height
+        self.redraw.tile_height = tile_height if tile_height > 0 else tile_width
+        self.seams_fix = USDUSeamsFix()
+        self.seams_fix.save = save_seams_fix
+        self.seams_fix.tile_width = tile_width if tile_width > 0 else tile_height
+        self.seams_fix.tile_height = tile_height if tile_height > 0 else tile_width
+        self.initial_info = None
+        self.rows = math.ceil(self.p.height / self.redraw.tile_height)
+        self.cols = math.ceil(self.p.width / self.redraw.tile_width)
+
+    def get_factor(self, num):
+        # Its just return, don't need elif
+        if num == 1:
+            return 2
+        if num % 4 == 0:
+            return 4
+        if num % 3 == 0:
+            return 3
+        if num % 2 == 0:
+            return 2
+        return 0
+
+    def get_factors(self):
+        scales = []
+        current_scale = 1
+        current_scale_factor = self.get_factor(self.scale_factor)
+        while current_scale_factor == 0:
+            self.scale_factor += 1
+            current_scale_factor = self.get_factor(self.scale_factor)
+        while current_scale < self.scale_factor:
+            current_scale_factor = self.get_factor(self.scale_factor // current_scale)
+            scales.append(current_scale_factor)
+            current_scale = current_scale * current_scale_factor
+            if current_scale_factor == 0:
+                break
+        self.scales = enumerate(scales)
+
+    def upscale(self):
+        # Log info
+        print(f"Canva size: {self.p.width}x{self.p.height}")
+        print(f"Image size: {self.image.width}x{self.image.height}")
+        print(f"Scale factor: {self.scale_factor}")
+        # Check upscaler is not empty
+        if self.upscaler.name == "None":
+            self.image = self.image.resize((self.p.width, self.p.height), resample=Image.LANCZOS)
+            return
+        # Get list with scale factors
+        self.get_factors()
+        # Upscaling image over all factors
+        for index, value in self.scales:
+            print(f"Upscaling iteration {index+1} with scale factor {value}")
+            self.image = self.upscaler.scaler.upscale(self.image, value, self.upscaler.data_path)
+        # Resize image to set values
+        self.image = self.image.resize((self.p.width, self.p.height), resample=Image.LANCZOS)
+
+    def setup_redraw(self, redraw_mode, padding, mask_blur):
+        self.redraw.mode = USDUMode(redraw_mode)
+        self.redraw.enabled = self.redraw.mode != USDUMode.NONE
+        self.redraw.padding = padding
+        self.p.mask_blur = mask_blur
+
+    def setup_seams_fix(self, padding, denoise, mask_blur, width, mode):
+        self.seams_fix.padding = padding
+        self.seams_fix.denoise = denoise
+        self.seams_fix.mask_blur = mask_blur
+        self.seams_fix.width = width
+        self.seams_fix.mode = USDUSFMode(mode)
+        self.seams_fix.enabled = self.seams_fix.mode != USDUSFMode.NONE
+
+    def save_image(self):
+        if type(self.p.prompt) != list:
+            images.save_image(self.image, self.p.outpath_samples, "", self.p.seed, self.p.prompt, opts.samples_format, info=self.initial_info, p=self.p)
+        else:
+            images.save_image(self.image, self.p.outpath_samples, "", self.p.seed, self.p.prompt[0], opts.samples_format, info=self.initial_info, p=self.p)
+
+    def calc_jobs_count(self):
+        redraw_job_count = (self.rows * self.cols) if self.redraw.enabled else 0
+        seams_job_count = 0
+        if self.seams_fix.mode == USDUSFMode.BAND_PASS:
+            seams_job_count = self.rows + self.cols - 2
+        elif self.seams_fix.mode == USDUSFMode.HALF_TILE:
+            seams_job_count = self.rows * (self.cols - 1) + (self.rows - 1) * self.cols
+        elif self.seams_fix.mode == USDUSFMode.HALF_TILE_PLUS_INTERSECTIONS:
+            seams_job_count = self.rows * (self.cols - 1) + (self.rows - 1) * self.cols + (self.rows - 1) * (self.cols - 1)
+
+        state.job_count = redraw_job_count + seams_job_count
+
+    def print_info(self):
+        print(f"Tile size: {self.redraw.tile_width}x{self.redraw.tile_height}")
+        print(f"Tiles amount: {self.rows * self.cols}")
+        print(f"Grid: {self.rows}x{self.cols}")
+        print(f"Redraw enabled: {self.redraw.enabled}")
+        print(f"Seams fix mode: {self.seams_fix.mode.name}")
+
+    def add_extra_info(self):
+        self.p.extra_generation_params["Ultimate SD upscale upscaler"] = self.upscaler.name
+        self.p.extra_generation_params["Ultimate SD upscale tile_width"] = self.redraw.tile_width
+        self.p.extra_generation_params["Ultimate SD upscale tile_height"] = self.redraw.tile_height
+        self.p.extra_generation_params["Ultimate SD upscale mask_blur"] = self.p.mask_blur
+        self.p.extra_generation_params["Ultimate SD upscale padding"] = self.redraw.padding
+
+    def process(self):
+        state.begin()
+        self.calc_jobs_count()
+        self.result_images = []
+        if self.redraw.enabled:
+            self.image = self.redraw.start(self.p, self.image, self.rows, self.cols)
+            self.initial_info = self.redraw.initial_info
+        self.result_images.append(self.image)
+        if self.redraw.save:
+            self.save_image()
+
+        if self.seams_fix.enabled:
+            self.image = self.seams_fix.start(self.p, self.image, self.rows, self.cols)
+            self.initial_info = self.seams_fix.initial_info
+            self.result_images.append(self.image)
+            if self.seams_fix.save:
+                self.save_image()
+        state.end()
+
+class USDURedraw():
+
+    def init_draw(self, p, width, height):
+        p.inpaint_full_res = True
+        p.inpaint_full_res_padding = self.padding
+        p.width = math.ceil((self.tile_width+self.padding) / 64) * 64
+        p.height = math.ceil((self.tile_height+self.padding) / 64) * 64
+        mask = Image.new("L", (width, height), "black")
+        draw = ImageDraw.Draw(mask)
+        return mask, draw
+
+    def calc_rectangle(self, xi, yi):
+        x1 = xi * self.tile_width
+        y1 = yi * self.tile_height
+        x2 = xi * self.tile_width + self.tile_width
+        y2 = yi * self.tile_height + self.tile_height
+
+        return x1, y1, x2, y2
+
+    def linear_process(self, p, image, rows, cols):
+        mask, draw = self.init_draw(p, image.width, image.height)
+        for yi in range(rows):
+            for xi in range(cols):
+                if state.interrupted:
+                    break
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="white")
+                p.init_images = [image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="black")
+                if (len(processed.images) > 0):
+                    image = processed.images[0]
+
+        p.width = image.width
+        p.height = image.height
+        self.initial_info = processed.infotext(p, 0)
+
+        return image
+
+    def chess_process(self, p, image, rows, cols):
+        mask, draw = self.init_draw(p, image.width, image.height)
+        tiles = []
+        # calc tiles colors
+        for yi in range(rows):
+            for xi in range(cols):
+                if state.interrupted:
+                    break
+                if xi == 0:
+                    tiles.append([])
+                color = xi % 2 == 0
+                if yi > 0 and yi % 2 != 0:
+                    color = not color
+                tiles[yi].append(color)
+
+        for yi in range(len(tiles)):
+            for xi in range(len(tiles[yi])):
+                if state.interrupted:
+                    break
+                if not tiles[yi][xi]:
+                    tiles[yi][xi] = not tiles[yi][xi]
+                    continue
+                tiles[yi][xi] = not tiles[yi][xi]
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="white")
+                p.init_images = [image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="black")
+                if (len(processed.images) > 0):
+                    image = processed.images[0]
+
+        for yi in range(len(tiles)):
+            for xi in range(len(tiles[yi])):
+                if state.interrupted:
+                    break
+                if not tiles[yi][xi]:
+                    continue
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="white")
+                p.init_images = [image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                draw.rectangle(self.calc_rectangle(xi, yi), fill="black")
+                if (len(processed.images) > 0):
+                    image = processed.images[0]
+
+        p.width = image.width
+        p.height = image.height
+        self.initial_info = processed.infotext(p, 0)
+
+        return image
+
+    def start(self, p, image, rows, cols):
+        self.initial_info = None
+        if self.mode == USDUMode.LINEAR:
+            return self.linear_process(p, image, rows, cols)
+        if self.mode == USDUMode.CHESS:
+            return self.chess_process(p, image, rows, cols)
+
+class USDUSeamsFix():
+
+    def init_draw(self, p):
+        self.initial_info = None
+        p.width = math.ceil((self.tile_width+self.padding) / 64) * 64
+        p.height = math.ceil((self.tile_height+self.padding) / 64) * 64
+
+    def half_tile_process(self, p, image, rows, cols):
+
+        self.init_draw(p)
+        processed = None
+
+        gradient = Image.linear_gradient("L")
+        row_gradient = Image.new("L", (self.tile_width, self.tile_height), "black")
+        row_gradient.paste(gradient.resize(
+            (self.tile_width, self.tile_height//2), resample=Image.BICUBIC), (0, 0))
+        row_gradient.paste(gradient.rotate(180).resize(
+                (self.tile_width, self.tile_height//2), resample=Image.BICUBIC),
+                (0, self.tile_height//2))
+        col_gradient = Image.new("L", (self.tile_width, self.tile_height), "black")
+        col_gradient.paste(gradient.rotate(90).resize(
+            (self.tile_width//2, self.tile_height), resample=Image.BICUBIC), (0, 0))
+        col_gradient.paste(gradient.rotate(270).resize(
+            (self.tile_width//2, self.tile_height), resample=Image.BICUBIC), (self.tile_width//2, 0))
+
+        p.denoising_strength = self.denoise
+        p.mask_blur = self.mask_blur
+
+        for yi in range(rows-1):
+            for xi in range(cols):
+                if state.interrupted:
+                    break
+                p.width = self.tile_width
+                p.height = self.tile_height
+                p.inpaint_full_res = True
+                p.inpaint_full_res_padding = self.padding
+                mask = Image.new("L", (image.width, image.height), "black")
+                mask.paste(row_gradient, (xi*self.tile_width, yi*self.tile_height + self.tile_height//2))
+
+                p.init_images = [image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                if (len(processed.images) > 0):
+                    image = processed.images[0]
+
+        for yi in range(rows):
+            for xi in range(cols-1):
+                if state.interrupted:
+                    break
+                p.width = self.tile_width
+                p.height = self.tile_height
+                p.inpaint_full_res = True
+                p.inpaint_full_res_padding = self.padding
+                mask = Image.new("L", (image.width, image.height), "black")
+                mask.paste(col_gradient, (xi*self.tile_width+self.tile_width//2, yi*self.tile_height))
+
+                p.init_images = [image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                if (len(processed.images) > 0):
+                    image = processed.images[0]
+
+        p.width = image.width
+        p.height = image.height
+        if processed is not None:
+            self.initial_info = processed.infotext(p, 0)
+
+        return image
+
+    def half_tile_process_corners(self, p, image, rows, cols):
+        fixed_image = self.half_tile_process(p, image, rows, cols)
+        processed = None
+        self.init_draw(p)
+        gradient = Image.radial_gradient("L").resize(
+            (self.tile_width, self.tile_height), resample=Image.BICUBIC)
+        gradient = ImageOps.invert(gradient)
+        p.denoising_strength = self.denoise
+        #p.mask_blur = 0
+        p.mask_blur = self.mask_blur
+
+        for yi in range(rows-1):
+            for xi in range(cols-1):
+                if state.interrupted:
+                    break
+                p.width = self.tile_width
+                p.height = self.tile_height
+                p.inpaint_full_res = True
+                p.inpaint_full_res_padding = 0
+                mask = Image.new("L", (fixed_image.width, fixed_image.height), "black")
+                mask.paste(gradient, (xi*self.tile_width + self.tile_width//2,
+                                      yi*self.tile_height + self.tile_height//2))
+
+                p.init_images = [fixed_image]
+                p.image_mask = mask
+                processed = processing.process_images(p)
+                if (len(processed.images) > 0):
+                    fixed_image = processed.images[0]
+
+        p.width = fixed_image.width
+        p.height = fixed_image.height
+        if processed is not None:
+            self.initial_info = processed.infotext(p, 0)
+
+        return fixed_image
+
+    def band_pass_process(self, p, image, cols, rows):
+
+        self.init_draw(p)
+        processed = None
+
+        p.denoising_strength = self.denoise
+        p.mask_blur = 0
+
+        gradient = Image.linear_gradient("L")
+        mirror_gradient = Image.new("L", (256, 256), "black")
+        mirror_gradient.paste(gradient.resize((256, 128), resample=Image.BICUBIC), (0, 0))
+        mirror_gradient.paste(gradient.rotate(180).resize((256, 128), resample=Image.BICUBIC), (0, 128))
+
+        row_gradient = mirror_gradient.resize((image.width, self.width), resample=Image.BICUBIC)
+        col_gradient = mirror_gradient.rotate(90).resize((self.width, image.height), resample=Image.BICUBIC)
+
+        for xi in range(1, rows):
+            if state.interrupted:
+                    break
+            p.width = self.width + self.padding * 2
+            p.height = image.height
+            p.inpaint_full_res = True
+            p.inpaint_full_res_padding = self.padding
+            mask = Image.new("L", (image.width, image.height), "black")
+            mask.paste(col_gradient, (xi * self.tile_width - self.width // 2, 0))
+
+            p.init_images = [image]
+            p.image_mask = mask
+            processed = processing.process_images(p)
+            if (len(processed.images) > 0):
+                image = processed.images[0]
+        for yi in range(1, cols):
+            if state.interrupted:
+                    break
+            p.width = image.width
+            p.height = self.width + self.padding * 2
+            p.inpaint_full_res = True
+            p.inpaint_full_res_padding = self.padding
+            mask = Image.new("L", (image.width, image.height), "black")
+            mask.paste(row_gradient, (0, yi * self.tile_height - self.width // 2))
+
+            p.init_images = [image]
+            p.image_mask = mask
+            processed = processing.process_images(p)
+            if (len(processed.images) > 0):
+                image = processed.images[0]
+
+        p.width = image.width
+        p.height = image.height
+        if processed is not None:
+            self.initial_info = processed.infotext(p, 0)
+
+        return image
+
+    def start(self, p, image, rows, cols):
+        if USDUSFMode(self.mode) == USDUSFMode.BAND_PASS:
+            return self.band_pass_process(p, image, rows, cols)
+        elif USDUSFMode(self.mode) == USDUSFMode.HALF_TILE:
+            return self.half_tile_process(p, image, rows, cols)
+        elif USDUSFMode(self.mode) == USDUSFMode.HALF_TILE_PLUS_INTERSECTIONS:
+            return self.half_tile_process_corners(p, image, rows, cols)
+        else:
+            return image
+
+class Script(scripts.Script):
+    def title(self):
+        return "Ultimate SD upscale"
+
+    def show(self, is_img2img):
+        return is_img2img
+
+    def ui(self, is_img2img):
+
+        target_size_types = [
+            "From img2img2 settings",
+            "Custom size",
+            "Scale from image size"
+        ]
+
+        seams_fix_types = [
+            "None",
+            "Band pass",
+            "Half tile offset pass",
+            "Half tile offset pass + intersections"
+        ]
+
+        redrow_modes = [
+            "Linear",
+            "Chess",
+            "None"
+        ]
+
+        info = gr.HTML(
+            "<p style=\"margin-bottom:0.75em\">Will upscale the image depending on the selected target size type</p>")
+
+        with gr.Row():
+            target_size_type = gr.Dropdown(label="Target size type", elem_id=f"{elem_id_prefix}_target_size_type", choices=[k for k in target_size_types], type="index",
+                                  value=next(iter(target_size_types)))
+
+            custom_width = gr.Slider(label='Custom width', elem_id=f"{elem_id_prefix}_custom_width", minimum=64, maximum=8192, step=64, value=2048, visible=False, interactive=True)
+            custom_height = gr.Slider(label='Custom height', elem_id=f"{elem_id_prefix}_custom_height", minimum=64, maximum=8192, step=64, value=2048, visible=False, interactive=True)
+            custom_scale = gr.Slider(label='Scale', elem_id=f"{elem_id_prefix}_custom_scale", minimum=1, maximum=16, step=0.01, value=2, visible=False, interactive=True)
+
+        gr.HTML("<p style=\"margin-bottom:0.75em\">Redraw options:</p>")
+        with gr.Row():
+            upscaler_index = gr.Radio(label='Upscaler', elem_id=f"{elem_id_prefix}_upscaler_index", choices=[x.name for x in shared.sd_upscalers],
+                                value=shared.sd_upscalers[0].name, type="index")
+        with gr.Row():
+            redraw_mode = gr.Dropdown(label="Type", elem_id=f"{elem_id_prefix}_redraw_mode", choices=[k for k in redrow_modes], type="index", value=next(iter(redrow_modes)))
+            tile_width = gr.Slider(elem_id=f"{elem_id_prefix}_tile_width", minimum=0, maximum=2048, step=64, label='Tile width', value=512)
+            tile_height = gr.Slider(elem_id=f"{elem_id_prefix}_tile_height", minimum=0, maximum=2048, step=64, label='Tile height', value=0)
+            mask_blur = gr.Slider(elem_id=f"{elem_id_prefix}_mask_blur", label='Mask blur', minimum=0, maximum=64, step=1, value=8)
+            padding = gr.Slider(elem_id=f"{elem_id_prefix}_padding", label='Padding', minimum=0, maximum=512, step=1, value=32)
+        gr.HTML("<p style=\"margin-bottom:0.75em\">Seams fix:</p>")
+        with gr.Row():
+            seams_fix_type = gr.Dropdown(label="Type", elem_id=f"{elem_id_prefix}_seams_fix_type", choices=[k for k in seams_fix_types], type="index", value=next(iter(seams_fix_types)))
+            seams_fix_denoise = gr.Slider(label='Denoise', elem_id=f"{elem_id_prefix}_seams_fix_denoise", minimum=0, maximum=1, step=0.01, value=0.35, visible=False, interactive=True)
+            seams_fix_width = gr.Slider(label='Width', elem_id=f"{elem_id_prefix}_seams_fix_width", minimum=0, maximum=128, step=1, value=64, visible=False, interactive=True)
+            seams_fix_mask_blur = gr.Slider(label='Mask blur', elem_id=f"{elem_id_prefix}_seams_fix_mask_blur", minimum=0, maximum=64, step=1, value=4, visible=False, interactive=True)
+            seams_fix_padding = gr.Slider(label='Padding', elem_id=f"{elem_id_prefix}_seams_fix_padding", minimum=0, maximum=128, step=1, value=16, visible=False, interactive=True)
+        gr.HTML("<p style=\"margin-bottom:0.75em\">Save options:</p>")
+        with gr.Row():
+            save_upscaled_image = gr.Checkbox(label="Upscaled", elem_id=f"{elem_id_prefix}_save_upscaled_image", value=True)
+            save_seams_fix_image = gr.Checkbox(label="Seams fix", elem_id=f"{elem_id_prefix}_save_seams_fix_image", value=False)
+
+        def select_fix_type(fix_index):
+            all_visible = fix_index != 0
+            mask_blur_visible = fix_index == 2 or fix_index == 3
+            width_visible = fix_index == 1
+
+            return [gr.update(visible=all_visible),
+                    gr.update(visible=width_visible),
+                    gr.update(visible=mask_blur_visible),
+                    gr.update(visible=all_visible)]
+
+        seams_fix_type.change(
+            fn=select_fix_type,
+            inputs=seams_fix_type,
+            outputs=[seams_fix_denoise, seams_fix_width, seams_fix_mask_blur, seams_fix_padding]
+        )
+
+        def select_scale_type(scale_index):
+            is_custom_size = scale_index == 1
+            is_custom_scale = scale_index == 2
+
+            return [gr.update(visible=is_custom_size),
+                    gr.update(visible=is_custom_size),
+                    gr.update(visible=is_custom_scale),
+                    ]
+
+        target_size_type.change(
+            fn=select_scale_type,
+            inputs=target_size_type,
+            outputs=[custom_width, custom_height, custom_scale]
+        )
+
+        def init_field(scale_name):
+            try:
+                scale_index = target_size_types.index(scale_name)
+                custom_width.visible = custom_height.visible = scale_index == 1
+                custom_scale.visible = scale_index == 2
+            except:
+                pass
+
+        target_size_type.init_field = init_field
+
+        return [info, tile_width, tile_height, mask_blur, padding, seams_fix_width, seams_fix_denoise, seams_fix_padding,
+                upscaler_index, save_upscaled_image, redraw_mode, save_seams_fix_image, seams_fix_mask_blur,
+                seams_fix_type, target_size_type, custom_width, custom_height, custom_scale]
+
+    def run(self, p, _, tile_width, tile_height, mask_blur, padding, seams_fix_width, seams_fix_denoise, seams_fix_padding,
+            upscaler_index, save_upscaled_image, redraw_mode, save_seams_fix_image, seams_fix_mask_blur,
+            seams_fix_type, target_size_type, custom_width, custom_height, custom_scale):
+
+        # Init
+        processing.fix_seed(p)
+        devices.torch_gc()
+
+        p.do_not_save_grid = True
+        p.do_not_save_samples = True
+        p.inpaint_full_res = False
+
+        p.inpainting_fill = 1
+        p.n_iter = 1
+        p.batch_size = 1
+
+        seed = p.seed
+
+        # Init image
+        init_img = p.init_images[0]
+        if init_img == None:
+            return Processed(p, [], seed, "Empty image")
+        init_img = images.flatten(init_img, opts.img2img_background_color)
+
+        #override size
+        if target_size_type == 1:
+            p.width = custom_width
+            p.height = custom_height
+        if target_size_type == 2:
+            p.width = math.ceil((init_img.width * custom_scale) / 64) * 64
+            p.height = math.ceil((init_img.height * custom_scale) / 64) * 64
+
+        # Upscaling
+        upscaler = USDUpscaler(p, init_img, upscaler_index, save_upscaled_image, save_seams_fix_image, tile_width, tile_height)
+        upscaler.upscale()
+        
+        # Drawing
+        upscaler.setup_redraw(redraw_mode, padding, mask_blur)
+        upscaler.setup_seams_fix(seams_fix_padding, seams_fix_denoise, seams_fix_mask_blur, seams_fix_width, seams_fix_type)
+        upscaler.print_info()
+        upscaler.add_extra_info()
+        upscaler.process()
+        result_images = upscaler.result_images
+
+        return Processed(p, result_images, seed, upscaler.initial_info if upscaler.initial_info is not None else "")
+

was-node-suite-comfyui/.github/workflows/publish_action.yml

@@ -0,0 +1,20 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} ## Add your own personal access token to your Github Repository secrets and reference it here.

was-node-suite-comfyui/modules/BLIP/blip_med.py

@@ -0,0 +1,955 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+ * Based on huggingface code base
+ * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert
+'''
+
+import math
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import torch
+from torch import Tensor, device, dtype, nn
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+import torch.nn.functional as F
+
+from transformers.activations import ACT2FN
+from transformers.file_utils import (
+    ModelOutput,
+)
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPastAndCrossAttentions,
+    BaseModelOutputWithPoolingAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    NextSentencePredictorOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+)
+from transformers.modeling_utils import (
+    PreTrainedModel,
+    apply_chunking_to_forward,
+    find_pruneable_heads_and_indices,
+    prune_linear_layer,
+)
+from transformers.utils import logging
+from transformers.models.bert.configuration_bert import BertConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class BertEmbeddings(nn.Module):
+    """Construct the embeddings from word and position embeddings."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+
+        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
+        # any TensorFlow checkpoint file
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        
+        self.config = config
+
+    def forward(
+        self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0
+    ):
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            input_shape = inputs_embeds.size()[:-1]
+
+        seq_length = input_shape[1]
+
+        if position_ids is None:
+            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+
+        embeddings = inputs_embeds
+
+        if self.position_embedding_type == "absolute":
+            position_embeddings = self.position_embeddings(position_ids)
+            embeddings += position_embeddings
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+class BertSelfAttention(nn.Module):
+    def __init__(self, config, is_cross_attention):
+        super().__init__()
+        self.config = config
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                "The hidden size (%d) is not a multiple of the number of attention "
+                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
+            )
+        
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        if is_cross_attention:
+            self.key = nn.Linear(config.encoder_width, self.all_head_size)
+            self.value = nn.Linear(config.encoder_width, self.all_head_size)
+        else:
+            self.key = nn.Linear(config.hidden_size, self.all_head_size)
+            self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
+        self.save_attention = False   
+            
+    def save_attn_gradients(self, attn_gradients):
+        self.attn_gradients = attn_gradients
+        
+    def get_attn_gradients(self):
+        return self.attn_gradients
+    
+    def save_attention_map(self, attention_map):
+        self.attention_map = attention_map
+        
+    def get_attention_map(self):
+        return self.attention_map
+    
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        mixed_query_layer = self.query(hidden_states)
+
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
+            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
+            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+        
+        if is_cross_attention and self.save_attention:
+            self.save_attention_map(attention_probs)
+            attention_probs.register_hook(self.save_attn_gradients)         
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs_dropped = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs_dropped = attention_probs_dropped * head_mask
+
+        context_layer = torch.matmul(attention_probs_dropped, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
+
+        outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class BertSelfOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertAttention(nn.Module):
+    def __init__(self, config, is_cross_attention=False):
+        super().__init__()
+        self.self = BertSelfAttention(config, is_cross_attention)
+        self.output = BertSelfOutput(config)
+        self.pruned_heads = set()
+
+    def prune_heads(self, heads):
+        if len(heads) == 0:
+            return
+        heads, index = find_pruneable_heads_and_indices(
+            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
+        )
+
+        # Prune linear layers
+        self.self.query = prune_linear_layer(self.self.query, index)
+        self.self.key = prune_linear_layer(self.self.key, index)
+        self.self.value = prune_linear_layer(self.self.value, index)
+        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
+
+        # Update hyper params and store pruned heads
+        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
+        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
+        self.pruned_heads = self.pruned_heads.union(heads)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        self_outputs = self.self(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            encoder_hidden_states,
+            encoder_attention_mask,
+            past_key_value,
+            output_attentions,
+        )
+        attention_output = self.output(self_outputs[0], hidden_states)
+        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+class BertIntermediate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        return hidden_states
+
+
+class BertOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states, input_tensor):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class BertLayer(nn.Module):
+    def __init__(self, config, layer_num):
+        super().__init__()
+        self.config = config
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = BertAttention(config)      
+        self.layer_num = layer_num          
+        if self.config.add_cross_attention:
+            self.crossattention = BertAttention(config, is_cross_attention=self.config.add_cross_attention)
+        self.intermediate = BertIntermediate(config)
+        self.output = BertOutput(config)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+        mode=None,
+    ):
+        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
+        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
+        self_attention_outputs = self.attention(
+            hidden_states,
+            attention_mask,
+            head_mask,
+            output_attentions=output_attentions,
+            past_key_value=self_attn_past_key_value,
+        )
+        attention_output = self_attention_outputs[0]
+
+        outputs = self_attention_outputs[1:-1]
+        present_key_value = self_attention_outputs[-1]
+
+        if mode=='multimodal':
+            assert encoder_hidden_states is not None, "encoder_hidden_states must be given for cross-attention layers"
+
+            cross_attention_outputs = self.crossattention(
+                attention_output,
+                attention_mask,
+                head_mask,
+                encoder_hidden_states,
+                encoder_attention_mask,
+                output_attentions=output_attentions,
+            )
+            attention_output = cross_attention_outputs[0]
+            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights                               
+        layer_output = apply_chunking_to_forward(
+            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
+        )
+        outputs = (layer_output,) + outputs
+
+        outputs = outputs + (present_key_value,)
+
+        return outputs
+
+    def feed_forward_chunk(self, attention_output):
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        return layer_output
+
+
+class BertEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layer = nn.ModuleList([BertLayer(config,i) for i in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=False,
+        output_hidden_states=False,
+        return_dict=True,
+        mode='multimodal',
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+
+        next_decoder_cache = () if use_cache else None
+               
+        for i in range(self.config.num_hidden_layers):
+            layer_module = self.layer[i]
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_head_mask = head_mask[i] if head_mask is not None else None
+            past_key_value = past_key_values[i] if past_key_values is not None else None
+
+            if self.gradient_checkpointing and self.training:
+
+                if use_cache:
+                    logger.warn(
+                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                    )
+                    use_cache = False
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        return module(*inputs, past_key_value, output_attentions)
+
+                    return custom_forward
+
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(layer_module),
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    mode=mode,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask,
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    past_key_value,
+                    output_attentions,
+                    mode=mode,
+                )
+
+            hidden_states = layer_outputs[0]
+            if use_cache:
+                next_decoder_cache += (layer_outputs[-1],)
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    next_decoder_cache,
+                    all_hidden_states,
+                    all_self_attentions,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=next_decoder_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+
+class BertPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+class BertPredictionHeadTransform(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        if isinstance(config.hidden_act, str):
+            self.transform_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.transform_act_fn = config.hidden_act
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.transform_act_fn(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        return hidden_states
+
+
+class BertLMPredictionHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.transform = BertPredictionHeadTransform(config)
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
+    def forward(self, hidden_states):
+        hidden_states = self.transform(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+        return hidden_states
+
+
+class BertOnlyMLMHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.predictions = BertLMPredictionHead(config)
+
+    def forward(self, sequence_output):
+        prediction_scores = self.predictions(sequence_output)
+        return prediction_scores
+
+
+class BertPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = BertConfig
+    base_model_prefix = "bert"
+    _keys_to_ignore_on_load_missing = [r"position_ids"]
+
+    def _init_weights(self, module):
+        """ Initialize the weights """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+
+class BertModel(BertPreTrainedModel):
+    """
+    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
+    cross-attention is added between the self-attention layers, following the architecture described in `Attention is
+    all you need <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
+    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+    argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an
+    input to the forward pass.
+    """
+
+    def __init__(self, config, add_pooling_layer=True):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = BertEmbeddings(config)
+        
+        self.encoder = BertEncoder(config)
+
+        self.pooler = BertPooler(config) if add_pooling_layer else None
+
+        self.init_weights()
+ 
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
+        class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    
+    def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor:
+        """
+        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
+
+        Arguments:
+            attention_mask (:obj:`torch.Tensor`):
+                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
+            input_shape (:obj:`Tuple[int]`):
+                The shape of the input to the model.
+            device: (:obj:`torch.device`):
+                The device of the input to the model.
+
+        Returns:
+            :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`.
+        """
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        if attention_mask.dim() == 3:
+            extended_attention_mask = attention_mask[:, None, :, :]
+        elif attention_mask.dim() == 2:
+            # Provided a padding mask of dimensions [batch_size, seq_length]
+            # - if the model is a decoder, apply a causal mask in addition to the padding mask
+            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
+            if is_decoder:
+                batch_size, seq_length = input_shape
+
+                seq_ids = torch.arange(seq_length, device=device)
+                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
+                # in case past_key_values are used we need to add a prefix ones mask to the causal mask
+                # causal and attention masks must have same type with pytorch version < 1.3
+                causal_mask = causal_mask.to(attention_mask.dtype)
+   
+                if causal_mask.shape[1] < attention_mask.shape[1]:
+                    prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
+                    causal_mask = torch.cat(
+                        [
+                            torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
+                            causal_mask,
+                        ],
+                        axis=-1,
+                    )                     
+
+                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
+            else:
+                extended_attention_mask = attention_mask[:, None, None, :]
+        else:
+            raise ValueError(
+                "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
+                    input_shape, attention_mask.shape
+                )
+            )
+
+        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+        # masked positions, this operation will create a tensor which is 0.0 for
+        # positions we want to attend and -10000.0 for masked positions.
+        # Since we are adding it to the raw scores before the softmax, this is
+        # effectively the same as removing these entirely.
+        extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+        return extended_attention_mask
+    
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        is_decoder=False,
+        mode='multimodal',
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if is_decoder:
+            use_cache = use_cache if use_cache is not None else self.config.use_cache
+        else:
+            use_cache = False
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            batch_size, seq_length = input_shape
+            device = input_ids.device
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size, seq_length = input_shape
+            device = inputs_embeds.device
+        elif encoder_embeds is not None:    
+            input_shape = encoder_embeds.size()[:-1]
+            batch_size, seq_length = input_shape 
+            device = encoder_embeds.device
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds")
+
+        # past_key_values_length
+        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0
+
+        if attention_mask is None:
+            attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device)
+            
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, 
+                                                                                 device, is_decoder)
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if encoder_hidden_states is not None:
+            if type(encoder_hidden_states) == list:
+                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
+            else:
+                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            
+            if type(encoder_attention_mask) == list:
+                encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
+            elif encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+            else:    
+                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_extended_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+        
+        if encoder_embeds is None:
+            embedding_output = self.embeddings(
+                input_ids=input_ids,
+                position_ids=position_ids,
+                inputs_embeds=inputs_embeds,
+                past_key_values_length=past_key_values_length,
+            )
+        else:
+            embedding_output = encoder_embeds
+            
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_mask=extended_attention_mask,
+            head_mask=head_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_extended_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            mode=mode,
+        )
+        sequence_output = encoder_outputs[0]
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        if not return_dict:
+            return (sequence_output, pooled_output) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPoolingAndCrossAttentions(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            past_key_values=encoder_outputs.past_key_values,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+            cross_attentions=encoder_outputs.cross_attentions,
+        )
+
+
+
+class BertLMHeadModel(BertPreTrainedModel):
+
+    _keys_to_ignore_on_load_unexpected = [r"pooler"]
+    _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.bert = BertModel(config, add_pooling_layer=False)
+        self.cls = BertOnlyMLMHead(config)
+
+        self.init_weights()
+
+    def get_output_embeddings(self):
+        return self.cls.predictions.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.cls.predictions.decoder = new_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        labels=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        return_logits=False,            
+        is_decoder=True,
+        reduction='mean',
+        mode='multimodal', 
+    ):
+        r"""
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
+            the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
+            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are
+            ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]``
+        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
+            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
+            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
+            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
+            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
+        use_cache (:obj:`bool`, `optional`):
+            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
+            decoding (see :obj:`past_key_values`).
+        Returns:
+        Example::
+            >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig
+            >>> import torch
+            >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
+            >>> config = BertConfig.from_pretrained("bert-base-cased")
+            >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config)
+            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+            >>> outputs = model(**inputs)
+            >>> prediction_logits = outputs.logits
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        if labels is not None:
+            use_cache = False
+
+        outputs = self.bert(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            is_decoder=is_decoder,
+            mode=mode,
+        )
+        
+        sequence_output = outputs[0]
+        prediction_scores = self.cls(sequence_output)
+        
+        if return_logits:
+            return prediction_scores[:, :-1, :].contiguous()  
+
+        lm_loss = None
+        if labels is not None:
+            # we are doing next-token prediction; shift prediction scores and input ids by one
+            shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
+            labels = labels[:, 1:].contiguous()
+            loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) 
+            lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
+            if reduction=='none':
+                lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1)               
+
+        if not return_dict:
+            output = (prediction_scores,) + outputs[2:]
+            return ((lm_loss,) + output) if lm_loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=lm_loss,
+            logits=prediction_scores,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            cross_attentions=outputs.cross_attentions,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, **model_kwargs):
+        input_shape = input_ids.shape
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is None:
+            attention_mask = input_ids.new_ones(input_shape)
+
+        # cut decoder_input_ids if past is used
+        if past is not None:
+            input_ids = input_ids[:, -1:]
+
+        return {
+            "input_ids": input_ids, 
+            "attention_mask": attention_mask, 
+            "past_key_values": past,
+            "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
+            "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
+            "is_decoder": True,
+        }
+
+    def _reorder_cache(self, past, beam_idx):
+        reordered_past = ()
+        for layer_past in past:
+            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
+        return reordered_past

was-node-suite-comfyui/modules/BLIP/blip_module.py

@@ -0,0 +1,423 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+'''
+import warnings
+warnings.filterwarnings("ignore")
+
+from .blip_vit import VisionTransformer, interpolate_pos_embed
+from .blip_med import BertConfig, BertModel, BertLMHeadModel
+from transformers import BertTokenizer
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+import os
+from urllib.parse import urlparse
+from timm.models.hub import download_cached_file
+import numpy as np
+
+from pathlib import Path
+LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
+
+# BLIP
+
+class BLIP_Base(nn.Module):
+    def __init__(self,                 
+                 med_config = Path(LOCAL_PATH, 'blip_configs/med_config.json'),  
+                 image_size = 224,
+                 vit = 'base',
+                 vit_grad_ckpt = False,
+                 vit_ckpt_layer = 0,                 
+                 ):
+        """
+        Args:
+            med_config (str): path for the mixture of encoder-decoder model's configuration file
+            image_size (int): input image size
+            vit (str): model size of vision transformer
+        """               
+        super().__init__()
+        
+        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
+        self.tokenizer = init_tokenizer()   
+        med_config = BertConfig.from_json_file(med_config)
+        med_config.encoder_width = vision_width
+        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False)  
+
+        
+    def forward(self, image, caption, mode):
+        
+        assert mode in ['image', 'text', 'multimodal'], "mode parameter must be image, text, or multimodal"
+        text = self.tokenizer(caption, return_tensors="pt").to(image.device) 
+        
+        if mode=='image':    
+            # return image features
+            image_embeds = self.visual_encoder(image)             
+            return image_embeds
+        
+        elif mode=='text':
+            # return text features
+            text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
+                                            return_dict = True, mode = 'text')  
+            return text_output.last_hidden_state
+        
+        elif mode=='multimodal':
+            # return multimodel features
+            image_embeds = self.visual_encoder(image)    
+            image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)      
+            
+            text.input_ids[:,0] = self.tokenizer.enc_token_id
+            output = self.text_encoder(text.input_ids,
+                                       attention_mask = text.attention_mask,
+                                       encoder_hidden_states = image_embeds,
+                                       encoder_attention_mask = image_atts,      
+                                       return_dict = True,
+                                      )              
+            return output.last_hidden_state
+        
+        
+        
+class BLIP_Decoder(nn.Module):
+    def __init__(self,                 
+                 med_config = Path(LOCAL_PATH, 'blip_configs/med_config.json'),  
+                 image_size = 384,
+                 vit = 'base',
+                 vit_grad_ckpt = False,
+                 vit_ckpt_layer = 0,
+                 prompt = 'a picture of ',
+                 ):
+        """
+        Args:
+            med_config (str): path for the mixture of encoder-decoder model's configuration file
+            image_size (int): input image size
+            vit (str): model size of vision transformer
+        """            
+        super().__init__()
+        
+        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
+        self.tokenizer = init_tokenizer()   
+        med_config = BertConfig.from_json_file(med_config)
+        med_config.encoder_width = vision_width
+        self.text_decoder = BertLMHeadModel(config=med_config)    
+        
+        self.prompt = prompt
+        self.prompt_length = len(self.tokenizer(self.prompt).input_ids)-1
+
+        
+    def forward(self, image, caption):
+        
+        image_embeds = self.visual_encoder(image) 
+        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
+        
+        text = self.tokenizer(caption, padding='longest', truncation=True, max_length=40, return_tensors="pt").to(image.device) 
+        
+        text.input_ids[:,0] = self.tokenizer.bos_token_id
+        
+        decoder_targets = text.input_ids.masked_fill(text.input_ids == self.tokenizer.pad_token_id, -100)         
+        decoder_targets[:,:self.prompt_length] = -100
+     
+        decoder_output = self.text_decoder(text.input_ids, 
+                                           attention_mask = text.attention_mask, 
+                                           encoder_hidden_states = image_embeds,
+                                           encoder_attention_mask = image_atts,                  
+                                           labels = decoder_targets,
+                                           return_dict = True,   
+                                          )   
+        loss_lm = decoder_output.loss
+        
+        return loss_lm
+        
+    def generate(self, image, sample=False, num_beams=3, max_length=30, min_length=10, top_p=0.9, repetition_penalty=1.0):
+        image_embeds = self.visual_encoder(image)
+
+        if not sample:
+            image_embeds = image_embeds.repeat_interleave(num_beams,dim=0)
+            
+        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
+        model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask":image_atts}
+        
+        prompt = [self.prompt] * image.size(0)
+        input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.to(image.device) 
+        input_ids[:,0] = self.tokenizer.bos_token_id
+        input_ids = input_ids[:, :-1] 
+
+        if sample:
+            #nucleus sampling
+            outputs = self.text_decoder.generate(input_ids=input_ids,
+                                                  max_length=max_length,
+                                                  min_length=min_length,
+                                                  do_sample=True,
+                                                  top_p=top_p,
+                                                  num_return_sequences=1,
+                                                  eos_token_id=self.tokenizer.sep_token_id,
+                                                  pad_token_id=self.tokenizer.pad_token_id, 
+                                                  repetition_penalty=1.1,                                            
+                                                  **model_kwargs)
+        else:
+            #beam search
+            outputs = self.text_decoder.generate(input_ids=input_ids,
+                                                  max_length=max_length,
+                                                  min_length=min_length,
+                                                  num_beams=num_beams,
+                                                  eos_token_id=self.tokenizer.sep_token_id,
+                                                  pad_token_id=self.tokenizer.pad_token_id,     
+                                                  repetition_penalty=repetition_penalty,
+                                                  **model_kwargs)            
+            
+        captions = []    
+        for output in outputs:
+            caption = self.tokenizer.decode(output, skip_special_tokens=True)    
+            captions.append(caption[len(self.prompt):])
+        return captions
+    
+
+def blip_decoder(pretrained='',**kwargs):
+    model = BLIP_Decoder(**kwargs)
+    if pretrained:
+        model,msg = load_checkpoint(model,pretrained)
+        assert(len(msg.missing_keys)==0)
+    return model    
+    
+def blip_feature_extractor(pretrained='',**kwargs):
+    model = BLIP_Base(**kwargs)
+    if pretrained:
+        model,msg = load_checkpoint(model,pretrained)
+        assert(len(msg.missing_keys)==0)
+    return model        
+
+def init_tokenizer():
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    tokenizer.add_special_tokens({'bos_token':'[DEC]'})
+    tokenizer.add_special_tokens({'additional_special_tokens':['[ENC]']})       
+    tokenizer.enc_token_id = tokenizer.additional_special_tokens_ids[0]  
+    return tokenizer
+
+
+def create_vit(vit, image_size, use_grad_checkpointing=False, ckpt_layer=0, drop_path_rate=0):
+        
+    assert vit in ['base', 'large'], "vit parameter must be base or large"
+    if vit=='base':
+        vision_width = 768
+        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=12, 
+                                           num_heads=12, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
+                                           drop_path_rate=0 or drop_path_rate
+                                          )   
+    elif vit=='large':
+        vision_width = 1024
+        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=24, 
+                                           num_heads=16, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
+                                           drop_path_rate=0.1 or drop_path_rate
+                                          )   
+    return visual_encoder, vision_width
+
+def is_url(url_or_filename):
+    parsed = urlparse(url_or_filename)
+    return parsed.scheme in ("http", "https")
+
+def load_checkpoint(model,url_or_filename):
+    if is_url(url_or_filename):
+        cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
+        checkpoint = torch.load(cached_file, map_location='cpu') 
+    elif os.path.isfile(url_or_filename):        
+        checkpoint = torch.load(url_or_filename, map_location='cpu') 
+    else:
+        raise RuntimeError('checkpoint url or path is invalid')
+        
+    state_dict = checkpoint['model']
+    
+    state_dict['visual_encoder.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) 
+    if 'visual_encoder_m.pos_embed' in model.state_dict().keys():
+        state_dict['visual_encoder_m.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],
+                                                                         model.visual_encoder_m)    
+    for key in model.state_dict().keys():
+        if key in state_dict.keys():
+            if state_dict[key].shape!=model.state_dict()[key].shape:
+                del state_dict[key]
+    
+    msg = model.load_state_dict(state_dict,strict=False)
+    print('load checkpoint from %s'%url_or_filename)  
+    return model,msg
+    
+# BLIP VQA
+
+class BLIP_VQA(nn.Module):
+    def __init__(self,                 
+                 med_config = Path(LOCAL_PATH, 'blip_configs/med_config.json'),  
+                 image_size = 480,
+                 vit = 'base',
+                 vit_grad_ckpt = False,
+                 vit_ckpt_layer = 0,                   
+                 ):
+        """
+        Args:
+            med_config (str): path for the mixture of encoder-decoder model's configuration file
+            image_size (int): input image size
+            vit (str): model size of vision transformer
+        """               
+        super().__init__()
+        
+        self.visual_encoder, vision_width = create_vit(vit, image_size, vit_grad_ckpt, vit_ckpt_layer, drop_path_rate=0.1)
+        self.tokenizer = init_tokenizer()  
+        
+        encoder_config = BertConfig.from_json_file(med_config)
+        encoder_config.encoder_width = vision_width
+        self.text_encoder = BertModel(config=encoder_config, add_pooling_layer=False) 
+        
+        decoder_config = BertConfig.from_json_file(med_config)        
+        self.text_decoder = BertLMHeadModel(config=decoder_config)          
+
+
+    def forward(self, image, question, answer=None, n=None, weights=None, train=True, inference='rank', k_test=128):
+        
+        image_embeds = self.visual_encoder(image) 
+        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
+        
+        question = self.tokenizer(question, padding='longest', truncation=True, max_length=35, 
+                                  return_tensors="pt").to(image.device) 
+        question.input_ids[:,0] = self.tokenizer.enc_token_id
+        
+        if train:               
+            '''
+            n: number of answers for each question
+            weights: weight for each answer
+            '''                     
+            answer = self.tokenizer(answer, padding='longest', return_tensors="pt").to(image.device) 
+            answer.input_ids[:,0] = self.tokenizer.bos_token_id
+            answer_targets = answer.input_ids.masked_fill(answer.input_ids == self.tokenizer.pad_token_id, -100)      
+
+            question_output = self.text_encoder(question.input_ids, 
+                                                attention_mask = question.attention_mask, 
+                                                encoder_hidden_states = image_embeds,
+                                                encoder_attention_mask = image_atts,                             
+                                                return_dict = True)    
+
+            question_states = []                
+            question_atts = []  
+            for b, n in enumerate(n):
+                question_states += [question_output.last_hidden_state[b]]*n
+                question_atts += [question.attention_mask[b]]*n                
+            question_states = torch.stack(question_states,0)    
+            question_atts = torch.stack(question_atts,0)     
+
+            answer_output = self.text_decoder(answer.input_ids, 
+                                              attention_mask = answer.attention_mask, 
+                                              encoder_hidden_states = question_states,
+                                              encoder_attention_mask = question_atts,                  
+                                              labels = answer_targets,
+                                              return_dict = True,   
+                                              reduction = 'none',
+                                             )      
+            
+            loss = weights * answer_output.loss
+            loss = loss.sum()/image.size(0)
+
+            return loss
+            
+
+        else: 
+            question_output = self.text_encoder(question.input_ids, 
+                                                attention_mask = question.attention_mask, 
+                                                encoder_hidden_states = image_embeds,
+                                                encoder_attention_mask = image_atts,                                    
+                                                return_dict = True) 
+            
+            if inference=='generate':
+                num_beams = 3
+                question_states = question_output.last_hidden_state.repeat_interleave(num_beams,dim=0)
+                question_atts = torch.ones(question_states.size()[:-1],dtype=torch.long).to(question_states.device)
+                model_kwargs = {"encoder_hidden_states": question_states, "encoder_attention_mask":question_atts}
+                
+                bos_ids = torch.full((image.size(0),1),fill_value=self.tokenizer.bos_token_id,device=image.device)
+                
+                outputs = self.text_decoder.generate(input_ids=bos_ids,
+                                                     max_length=10,
+                                                     min_length=1,
+                                                     num_beams=num_beams,
+                                                     eos_token_id=self.tokenizer.sep_token_id,
+                                                     pad_token_id=self.tokenizer.pad_token_id, 
+                                                     **model_kwargs)
+                
+                answers = []    
+                for output in outputs:
+                    answer = self.tokenizer.decode(output, skip_special_tokens=True)    
+                    answers.append(answer)
+                return answers
+            
+            elif inference=='rank':
+                max_ids = self.rank_answer(question_output.last_hidden_state, question.attention_mask, 
+                                           answer.input_ids, answer.attention_mask, k_test) 
+                return max_ids
+ 
+                
+                
+    def rank_answer(self, question_states, question_atts, answer_ids, answer_atts, k):
+        
+        num_ques = question_states.size(0)
+        start_ids = answer_ids[0,0].repeat(num_ques,1) # bos token
+        
+        start_output = self.text_decoder(start_ids, 
+                                         encoder_hidden_states = question_states,
+                                         encoder_attention_mask = question_atts,                                      
+                                         return_dict = True,
+                                         reduction = 'none')              
+        logits = start_output.logits[:,0,:] # first token's logit
+        
+        # topk_probs: top-k probability 
+        # topk_ids: [num_question, k]        
+        answer_first_token = answer_ids[:,1]
+        prob_first_token = F.softmax(logits,dim=1).index_select(dim=1, index=answer_first_token) 
+        topk_probs, topk_ids = prob_first_token.topk(k,dim=1) 
+        
+        # answer input: [num_question*k, answer_len]                 
+        input_ids = []
+        input_atts = []
+        for b, topk_id in enumerate(topk_ids):
+            input_ids.append(answer_ids.index_select(dim=0, index=topk_id))
+            input_atts.append(answer_atts.index_select(dim=0, index=topk_id))
+        input_ids = torch.cat(input_ids,dim=0)  
+        input_atts = torch.cat(input_atts,dim=0)  
+
+        targets_ids = input_ids.masked_fill(input_ids == self.tokenizer.pad_token_id, -100)
+
+        # repeat encoder's output for top-k answers
+        question_states = tile(question_states, 0, k)
+        question_atts = tile(question_atts, 0, k)
+        
+        output = self.text_decoder(input_ids, 
+                                   attention_mask = input_atts, 
+                                   encoder_hidden_states = question_states,
+                                   encoder_attention_mask = question_atts,     
+                                   labels = targets_ids,
+                                   return_dict = True, 
+                                   reduction = 'none')   
+        
+        log_probs_sum = -output.loss
+        log_probs_sum = log_probs_sum.view(num_ques,k)
+
+        max_topk_ids = log_probs_sum.argmax(dim=1) 
+        max_ids = topk_ids[max_topk_ids>=0,max_topk_ids]
+
+        return max_ids
+    
+    
+def blip_vqa(pretrained='',**kwargs):
+    model = BLIP_VQA(**kwargs)
+    if pretrained:
+        model,msg = load_checkpoint(model,pretrained)
+#         assert(len(msg.missing_keys)==0)
+    return model  
+
+
+def tile(x, dim, n_tile):
+    init_dim = x.size(dim)
+    repeat_idx = [1] * x.dim()
+    repeat_idx[dim] = n_tile
+    x = x.repeat(*(repeat_idx))
+    order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)]))
+    return torch.index_select(x, dim, order_index.to(x.device))    
+        
+        

was-node-suite-comfyui/modules/BLIP/blip_module_license.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2022, Salesforce.com, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

was-node-suite-comfyui/modules/BLIP/blip_vit.py

@@ -0,0 +1,305 @@
+'''
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ * By Junnan Li
+ * Based on timm code base
+ * https://github.com/rwightman/pytorch-image-models/tree/master/timm
+'''
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from functools import partial
+
+from timm.models.vision_transformer import _cfg, PatchEmbed
+from timm.models.registry import register_model
+from timm.models.layers import trunc_normal_, DropPath
+from timm.models.helpers import named_apply, adapt_input_conv
+
+from fairscale.nn.checkpoint.checkpoint_activations import checkpoint_wrapper
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights
+        self.scale = qk_scale or head_dim ** -0.5
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.attn_gradients = None
+        self.attention_map = None
+        
+    def save_attn_gradients(self, attn_gradients):
+        self.attn_gradients = attn_gradients
+        
+    def get_attn_gradients(self):
+        return self.attn_gradients
+    
+    def save_attention_map(self, attention_map):
+        self.attention_map = attention_map
+        
+    def get_attention_map(self):
+        return self.attention_map
+    
+    def forward(self, x, register_hook=False):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)
+
+        attn = (q @ k.transpose(-2, -1)) * self.scale
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+                
+        if register_hook:
+            self.save_attention_map(attn)
+            attn.register_hook(self.save_attn_gradients)        
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class Block(nn.Module):
+
+    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_grad_checkpointing=False):
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        if use_grad_checkpointing:
+            self.attn = checkpoint_wrapper(self.attn)
+            self.mlp = checkpoint_wrapper(self.mlp)
+
+    def forward(self, x, register_hook=False):
+        x = x + self.drop_path(self.attn(self.norm1(x), register_hook=register_hook))
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+        return x
+
+    
+class VisionTransformer(nn.Module):
+    """ Vision Transformer
+    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`  -
+        https://arxiv.org/abs/2010.11929
+    """
+    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
+                 num_heads=12, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None,
+                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None, 
+                 use_grad_checkpointing=False, ckpt_layer=0):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            num_classes (int): number of classes for classification head
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            qk_scale (float): override default qk scale of head_dim ** -0.5 if set
+            representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
+            drop_rate (float): dropout rate
+            attn_drop_rate (float): attention dropout rate
+            drop_path_rate (float): stochastic depth rate
+            norm_layer: (nn.Module): normalization layer
+        """
+        super().__init__()
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
+
+        self.patch_embed = PatchEmbed(
+            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+        self.blocks = nn.ModuleList([
+            Block(
+                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
+                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,
+                use_grad_checkpointing=(use_grad_checkpointing and i>=depth-ckpt_layer)
+            )
+            for i in range(depth)])
+        self.norm = norm_layer(embed_dim)
+
+        trunc_normal_(self.pos_embed, std=.02)
+        trunc_normal_(self.cls_token, std=.02)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'pos_embed', 'cls_token'}
+
+    def forward(self, x, register_blk=-1):
+        B = x.shape[0]
+        x = self.patch_embed(x)
+
+        cls_tokens = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
+        x = torch.cat((cls_tokens, x), dim=1)
+  
+        x = x + self.pos_embed[:,:x.size(1),:]
+        x = self.pos_drop(x)
+
+        for i,blk in enumerate(self.blocks):
+            x = blk(x, register_blk==i)
+        x = self.norm(x)
+        
+        return x
+
+    @torch.jit.ignore()
+    def load_pretrained(self, checkpoint_path, prefix=''):
+        _load_weights(self, checkpoint_path, prefix)
+        
+
+@torch.no_grad()
+def _load_weights(model: VisionTransformer, checkpoint_path: str, prefix: str = ''):
+    """ Load weights from .npz checkpoints for official Google Brain Flax implementation
+    """
+    import numpy as np
+
+    def _n2p(w, t=True):
+        if w.ndim == 4 and w.shape[0] == w.shape[1] == w.shape[2] == 1:
+            w = w.flatten()
+        if t:
+            if w.ndim == 4:
+                w = w.transpose([3, 2, 0, 1])
+            elif w.ndim == 3:
+                w = w.transpose([2, 0, 1])
+            elif w.ndim == 2:
+                w = w.transpose([1, 0])
+        return torch.from_numpy(w)
+
+    w = np.load(checkpoint_path)
+    if not prefix and 'opt/target/embedding/kernel' in w:
+        prefix = 'opt/target/'
+
+    if hasattr(model.patch_embed, 'backbone'):
+        # hybrid
+        backbone = model.patch_embed.backbone
+        stem_only = not hasattr(backbone, 'stem')
+        stem = backbone if stem_only else backbone.stem
+        stem.conv.weight.copy_(adapt_input_conv(stem.conv.weight.shape[1], _n2p(w[f'{prefix}conv_root/kernel'])))
+        stem.norm.weight.copy_(_n2p(w[f'{prefix}gn_root/scale']))
+        stem.norm.bias.copy_(_n2p(w[f'{prefix}gn_root/bias']))
+        if not stem_only:
+            for i, stage in enumerate(backbone.stages):
+                for j, block in enumerate(stage.blocks):
+                    bp = f'{prefix}block{i + 1}/unit{j + 1}/'
+                    for r in range(3):
+                        getattr(block, f'conv{r + 1}').weight.copy_(_n2p(w[f'{bp}conv{r + 1}/kernel']))
+                        getattr(block, f'norm{r + 1}').weight.copy_(_n2p(w[f'{bp}gn{r + 1}/scale']))
+                        getattr(block, f'norm{r + 1}').bias.copy_(_n2p(w[f'{bp}gn{r + 1}/bias']))
+                    if block.downsample is not None:
+                        block.downsample.conv.weight.copy_(_n2p(w[f'{bp}conv_proj/kernel']))
+                        block.downsample.norm.weight.copy_(_n2p(w[f'{bp}gn_proj/scale']))
+                        block.downsample.norm.bias.copy_(_n2p(w[f'{bp}gn_proj/bias']))
+        embed_conv_w = _n2p(w[f'{prefix}embedding/kernel'])
+    else:
+        embed_conv_w = adapt_input_conv(
+            model.patch_embed.proj.weight.shape[1], _n2p(w[f'{prefix}embedding/kernel']))
+    model.patch_embed.proj.weight.copy_(embed_conv_w)
+    model.patch_embed.proj.bias.copy_(_n2p(w[f'{prefix}embedding/bias']))
+    model.cls_token.copy_(_n2p(w[f'{prefix}cls'], t=False))
+    pos_embed_w = _n2p(w[f'{prefix}Transformer/posembed_input/pos_embedding'], t=False)
+    if pos_embed_w.shape != model.pos_embed.shape:
+        pos_embed_w = resize_pos_embed(  # resize pos embedding when different size from pretrained weights
+            pos_embed_w, model.pos_embed, getattr(model, 'num_tokens', 1), model.patch_embed.grid_size)
+    model.pos_embed.copy_(pos_embed_w)
+    model.norm.weight.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/scale']))
+    model.norm.bias.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/bias']))
+#     if isinstance(model.head, nn.Linear) and model.head.bias.shape[0] == w[f'{prefix}head/bias'].shape[-1]:
+#         model.head.weight.copy_(_n2p(w[f'{prefix}head/kernel']))
+#         model.head.bias.copy_(_n2p(w[f'{prefix}head/bias']))
+#     if isinstance(getattr(model.pre_logits, 'fc', None), nn.Linear) and f'{prefix}pre_logits/bias' in w:
+#         model.pre_logits.fc.weight.copy_(_n2p(w[f'{prefix}pre_logits/kernel']))
+#         model.pre_logits.fc.bias.copy_(_n2p(w[f'{prefix}pre_logits/bias']))
+    for i, block in enumerate(model.blocks.children()):
+        block_prefix = f'{prefix}Transformer/encoderblock_{i}/'
+        mha_prefix = block_prefix + 'MultiHeadDotProductAttention_1/'
+        block.norm1.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/scale']))
+        block.norm1.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/bias']))
+        block.attn.qkv.weight.copy_(torch.cat([
+            _n2p(w[f'{mha_prefix}{n}/kernel'], t=False).flatten(1).T for n in ('query', 'key', 'value')]))
+        block.attn.qkv.bias.copy_(torch.cat([
+            _n2p(w[f'{mha_prefix}{n}/bias'], t=False).reshape(-1) for n in ('query', 'key', 'value')]))
+        block.attn.proj.weight.copy_(_n2p(w[f'{mha_prefix}out/kernel']).flatten(1))
+        block.attn.proj.bias.copy_(_n2p(w[f'{mha_prefix}out/bias']))
+        for r in range(2):
+            getattr(block.mlp, f'fc{r + 1}').weight.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/kernel']))
+            getattr(block.mlp, f'fc{r + 1}').bias.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/bias']))
+        block.norm2.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/scale']))
+        block.norm2.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/bias']))
+
+            
+def interpolate_pos_embed(pos_embed_checkpoint, visual_encoder):        
+    # interpolate position embedding
+    embedding_size = pos_embed_checkpoint.shape[-1]
+    num_patches = visual_encoder.patch_embed.num_patches
+    num_extra_tokens = visual_encoder.pos_embed.shape[-2] - num_patches
+    # height (== width) for the checkpoint position embedding
+    orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
+    # height (== width) for the new position embedding
+    new_size = int(num_patches ** 0.5)
+
+    if orig_size!=new_size:
+        # class_token and dist_token are kept unchanged
+        extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+        # only the position tokens are interpolated
+        pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+        pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+        pos_tokens = torch.nn.functional.interpolate(
+            pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+        pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+        new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+        print('reshape position embedding from %d to %d'%(orig_size ** 2,new_size ** 2))
+        
+        return new_pos_embed    
+    else:
+        return pos_embed_checkpoint

was-node-suite-comfyui/repos/SAM/demo/README.md

@@ -0,0 +1,126 @@
+## Segment Anything Simple Web demo
+
+This **front-end only** React based web demo shows how to load a fixed image and corresponding `.npy` file of the SAM image embedding, and run the SAM ONNX model in the browser using Web Assembly with mulithreading enabled by `SharedArrayBuffer`, Web Worker, and SIMD128.
+
+<img src="https://github.com/facebookresearch/segment-anything/raw/main/assets/minidemo.gif" width="500"/>
+
+## Run the app
+
+Install Yarn
+
+```
+npm install --g yarn
+```
+
+Build and run:
+
+```
+yarn && yarn start
+```
+
+Navigate to [`http://localhost:8081/`](http://localhost:8081/)
+
+Move your cursor around to see the mask prediction update in real time.
+
+## Export the image embedding
+
+In the [ONNX Model Example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb) upload the image of your choice and generate and save corresponding embedding.
+
+Initialize the predictor:
+
+```python
+checkpoint = "sam_vit_h_4b8939.pth"
+model_type = "vit_h"
+sam = sam_model_registry[model_type](checkpoint=checkpoint)
+sam.to(device='cuda')
+predictor = SamPredictor(sam)
+```
+
+Set the new image and export the embedding:
+
+```
+image = cv2.imread('src/assets/dogs.jpg')
+predictor.set_image(image)
+image_embedding = predictor.get_image_embedding().cpu().numpy()
+np.save("dogs_embedding.npy", image_embedding)
+```
+
+Save the new image and embedding in `src/assets/data`.
+
+## Export the ONNX model
+
+You also need to export the quantized ONNX model from the [ONNX Model Example notebook](https://github.com/facebookresearch/segment-anything/blob/main/notebooks/onnx_model_example.ipynb).
+
+Run the cell in the notebook which saves the `sam_onnx_quantized_example.onnx` file, download it and copy it to the path `/model/sam_onnx_quantized_example.onnx`.
+
+Here is a snippet of the export/quantization code:
+
+```
+onnx_model_path = "sam_onnx_example.onnx"
+onnx_model_quantized_path = "sam_onnx_quantized_example.onnx"
+quantize_dynamic(
+    model_input=onnx_model_path,
+    model_output=onnx_model_quantized_path,
+    optimize_model=True,
+    per_channel=False,
+    reduce_range=False,
+    weight_type=QuantType.QUInt8,
+)
+```
+
+**NOTE: if you change the ONNX model by using a new checkpoint you need to also re-export the embedding.**
+
+## Update the image, embedding, model in the app
+
+Update the following file paths at the top of`App.tsx`:
+
+```py
+const IMAGE_PATH = "/assets/data/dogs.jpg";
+const IMAGE_EMBEDDING = "/assets/data/dogs_embedding.npy";
+const MODEL_DIR = "/model/sam_onnx_quantized_example.onnx";
+```
+
+## ONNX multithreading with SharedArrayBuffer
+
+To use multithreading, the appropriate headers need to be set to create a cross origin isolation state which will enable use of `SharedArrayBuffer` (see this [blog post](https://cloudblogs.microsoft.com/opensource/2021/09/02/onnx-runtime-web-running-your-machine-learning-model-in-browser/) for more details)
+
+The headers below are set in `configs/webpack/dev.js`:
+
+```js
+headers: {
+    "Cross-Origin-Opener-Policy": "same-origin",
+    "Cross-Origin-Embedder-Policy": "credentialless",
+}
+```
+
+## Structure of the app
+
+**`App.tsx`**
+
+- Initializes ONNX model
+- Loads image embedding and image
+- Runs the ONNX model based on input prompts
+
+**`Stage.tsx`**
+
+- Handles mouse move interaction to update the ONNX model prompt
+
+**`Tool.tsx`**
+
+- Renders the image and the mask prediction
+
+**`helpers/maskUtils.tsx`**
+
+- Conversion of ONNX model output from array to an HTMLImageElement
+
+**`helpers/onnxModelAPI.tsx`**
+
+- Formats the inputs for the ONNX model
+
+**`helpers/scaleHelper.tsx`**
+
+- Handles image scaling logic for SAM (longest size 1024)
+
+**`hooks/`**
+
+- Handle shared state for the app

was-node-suite-comfyui/repos/SAM/demo/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "segment-anything-mini-demo",
+  "version": "0.1.0",
+  "license": "MIT",
+  "scripts": {
+    "build": "yarn run clean-dist && webpack --config=configs/webpack/prod.js && mv dist/*.wasm dist/js",
+    "clean-dist": "rimraf dist/*",
+    "lint": "eslint './src/**/*.{js,ts,tsx}' --quiet",
+    "start": "yarn run start-dev",
+    "test": "yarn run start-model-test",
+    "start-dev": "webpack serve --config=configs/webpack/dev.js"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.18.13",
+    "@babel/preset-env": "^7.18.10",
+    "@babel/preset-react": "^7.18.6",
+    "@babel/preset-typescript": "^7.18.6",
+    "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
+    "@testing-library/react": "^13.3.0",
+    "@types/node": "^18.7.13",
+    "@types/react": "^18.0.17",
+    "@types/react-dom": "^18.0.6",
+    "@types/underscore": "^1.11.4",
+    "@typescript-eslint/eslint-plugin": "^5.35.1",
+    "@typescript-eslint/parser": "^5.35.1",
+    "babel-loader": "^8.2.5",
+    "copy-webpack-plugin": "^11.0.0",
+    "css-loader": "^6.7.1",
+    "dotenv": "^16.0.2",
+    "dotenv-webpack": "^8.0.1",
+    "eslint": "^8.22.0",
+    "eslint-plugin-react": "^7.31.0",
+    "file-loader": "^6.2.0",
+    "fork-ts-checker-webpack-plugin": "^7.2.13",
+    "friendly-errors-webpack-plugin": "^1.7.0",
+    "html-webpack-plugin": "^5.5.0",
+    "image-webpack-loader": "^8.1.0",
+    "postcss-loader": "^7.0.1",
+    "postcss-preset-env": "^7.8.0",
+    "process": "^0.11.10",
+    "rimraf": "^3.0.2",
+    "sass": "^1.54.5",
+    "sass-loader": "^13.0.2",
+    "style-loader": "^3.3.1",
+    "tailwindcss": "^3.1.8",
+    "ts-loader": "^9.3.1",
+    "typescript": "^4.8.2",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0",
+    "webpack-dev-server": "^4.10.0",
+    "webpack-dotenv-plugin": "^2.1.0",
+    "webpack-merge": "^5.8.0"
+  },
+  "dependencies": {
+    "npyjs": "^0.4.0",
+    "onnxruntime-web": "^1.14.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "underscore": "^1.13.6",
+    "react-refresh": "^0.14.0"
+  }
+}

was-node-suite-comfyui/repos/SAM/demo/postcss.config.js

@@ -0,0 +1,10 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+const tailwindcss = require("tailwindcss");
+module.exports = {
+  plugins: ["postcss-preset-env", 'tailwindcss/nesting', tailwindcss],
+};

was-node-suite-comfyui/repos/SAM/demo/src/components/helpers/Interfaces.tsx

@@ -0,0 +1,29 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { Tensor } from "onnxruntime-web";
+
+export interface modelScaleProps {
+  samScale: number;
+  height: number;
+  width: number;
+}
+
+export interface modelInputProps {
+  x: number;
+  y: number;
+  clickType: number;
+}
+
+export interface modeDataProps {
+  clicks?: Array<modelInputProps>;
+  tensor: Tensor;
+  modelScale: modelScaleProps;
+}
+
+export interface ToolProps {
+  handleMouseMove: (e: any) => void;
+}

was-node-suite-comfyui/repos/SAM/demo/src/components/helpers/maskUtils.tsx

@@ -0,0 +1,47 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+// Convert the onnx model mask prediction to ImageData
+function arrayToImageData(input: any, width: number, height: number) {
+  const [r, g, b, a] = [0, 114, 189, 255]; // the masks's blue color
+  const arr = new Uint8ClampedArray(4 * width * height).fill(0);
+  for (let i = 0; i < input.length; i++) {
+
+    // Threshold the onnx model mask prediction at 0.0
+    // This is equivalent to thresholding the mask using predictor.model.mask_threshold
+    // in python
+    if (input[i] > 0.0) {
+      arr[4 * i + 0] = r;
+      arr[4 * i + 1] = g;
+      arr[4 * i + 2] = b;
+      arr[4 * i + 3] = a;
+    }
+  }
+  return new ImageData(arr, height, width);
+}
+
+// Use a Canvas element to produce an image from ImageData
+function imageDataToImage(imageData: ImageData) {
+  const canvas = imageDataToCanvas(imageData);
+  const image = new Image();
+  image.src = canvas.toDataURL();
+  return image;
+}
+
+// Canvas elements can be created from ImageData
+function imageDataToCanvas(imageData: ImageData) {
+  const canvas = document.createElement("canvas");
+  const ctx = canvas.getContext("2d");
+  canvas.width = imageData.width;
+  canvas.height = imageData.height;
+  ctx?.putImageData(imageData, 0, 0);
+  return canvas;
+}
+
+// Convert the onnx model mask output to an HTMLImageElement
+export function onnxMaskToImage(input: any, width: number, height: number) {
+  return imageDataToImage(arrayToImageData(input, width, height));
+}

was-node-suite-comfyui/repos/SAM/demo/src/components/helpers/onnxModelAPI.tsx

@@ -0,0 +1,71 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { Tensor } from "onnxruntime-web";
+import { modeDataProps } from "./Interfaces";
+
+const modelData = ({ clicks, tensor, modelScale }: modeDataProps) => {
+  const imageEmbedding = tensor;
+  let pointCoords;
+  let pointLabels;
+  let pointCoordsTensor;
+  let pointLabelsTensor;
+
+  // Check there are input click prompts
+  if (clicks) {
+    let n = clicks.length;
+
+    // If there is no box input, a single padding point with 
+    // label -1 and coordinates (0.0, 0.0) should be concatenated
+    // so initialize the array to support (n + 1) points.
+    pointCoords = new Float32Array(2 * (n + 1));
+    pointLabels = new Float32Array(n + 1);
+
+    // Add clicks and scale to what SAM expects
+    for (let i = 0; i < n; i++) {
+      pointCoords[2 * i] = clicks[i].x * modelScale.samScale;
+      pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale;
+      pointLabels[i] = clicks[i].clickType;
+    }
+
+    // Add in the extra point/label when only clicks and no box
+    // The extra point is at (0, 0) with label -1
+    pointCoords[2 * n] = 0.0;
+    pointCoords[2 * n + 1] = 0.0;
+    pointLabels[n] = -1.0;
+
+    // Create the tensor
+    pointCoordsTensor = new Tensor("float32", pointCoords, [1, n + 1, 2]);
+    pointLabelsTensor = new Tensor("float32", pointLabels, [1, n + 1]);
+  }
+  const imageSizeTensor = new Tensor("float32", [
+    modelScale.height,
+    modelScale.width,
+  ]);
+
+  if (pointCoordsTensor === undefined || pointLabelsTensor === undefined)
+    return;
+
+  // There is no previous mask, so default to an empty tensor
+  const maskInput = new Tensor(
+    "float32",
+    new Float32Array(256 * 256),
+    [1, 1, 256, 256]
+  );
+  // There is no previous mask, so default to 0
+  const hasMaskInput = new Tensor("float32", [0]);
+
+  return {
+    image_embeddings: imageEmbedding,
+    point_coords: pointCoordsTensor,
+    point_labels: pointLabelsTensor,
+    orig_im_size: imageSizeTensor,
+    mask_input: maskInput,
+    has_mask_input: hasMaskInput,
+  };
+};
+
+export { modelData };

was-node-suite-comfyui/repos/SAM/demo/src/components/helpers/scaleHelper.tsx

@@ -0,0 +1,18 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+// Helper function for handling image scaling needed for SAM
+const handleImageScale = (image: HTMLImageElement) => {
+  // Input images to SAM must be resized so the longest side is 1024
+  const LONG_SIDE_LENGTH = 1024;
+  let w = image.naturalWidth;
+  let h = image.naturalHeight;
+  const samScale = LONG_SIDE_LENGTH / Math.max(h, w);
+  return { height: h, width: w, samScale };
+};
+
+export { handleImageScale };

was-node-suite-comfyui/repos/SAM/demo/src/components/hooks/context.tsx

@@ -0,0 +1,31 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import React, { useState } from "react";
+import { modelInputProps } from "../helpers/Interfaces";
+import AppContext from "./createContext";
+
+const AppContextProvider = (props: {
+  children: React.ReactElement<any, string | React.JSXElementConstructor<any>>;
+}) => {
+  const [clicks, setClicks] = useState<Array<modelInputProps> | null>(null);
+  const [image, setImage] = useState<HTMLImageElement | null>(null);
+  const [maskImg, setMaskImg] = useState<HTMLImageElement | null>(null);
+
+  return (
+    <AppContext.Provider
+      value={{
+        clicks: [clicks, setClicks],
+        image: [image, setImage],
+        maskImg: [maskImg, setMaskImg],
+      }}
+    >
+      {props.children}
+    </AppContext.Provider>
+  );
+};
+
+export default AppContextProvider;

was-node-suite-comfyui/repos/SAM/demo/src/components/hooks/createContext.tsx

@@ -0,0 +1,27 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// All rights reserved.
+
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
+import { createContext } from "react";
+import { modelInputProps } from "../helpers/Interfaces";
+
+interface contextProps {
+  clicks: [
+    clicks: modelInputProps[] | null,
+    setClicks: (e: modelInputProps[] | null) => void
+  ];
+  image: [
+    image: HTMLImageElement | null,
+    setImage: (e: HTMLImageElement | null) => void
+  ];
+  maskImg: [
+    maskImg: HTMLImageElement | null,
+    setMaskImg: (e: HTMLImageElement | null) => void
+  ];
+}
+
+const AppContext = createContext<contextProps | null>(null);
+
+export default AppContext;