Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Commit
·
7ab7ea8
1
Parent(s):
d01d11a
dataset fix and tag fix
Browse files- ui/src/app/api/hf-jobs/route.ts +40 -13
- ui/src/app/jobs/new/SimplifiedJob.tsx +161 -0
ui/src/app/api/hf-jobs/route.ts
CHANGED
|
@@ -598,28 +598,55 @@ def generate_model_card_readme(repo_id: str, config: dict, model_name: str, cura
|
|
| 598 |
tags = []
|
| 599 |
lower_arch = (arch or "").lower()
|
| 600 |
lower_model_name = (model_config.get("name_or_path", "") or "").lower()
|
|
|
|
| 601 |
|
| 602 |
instruction_arches = {'flux_kontext', 'hidream_e1', 'qwen_image_edit'}
|
| 603 |
is_instruction = lower_arch in instruction_arches or 'kontext' in lower_model_name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 604 |
|
| 605 |
datasets_config = config.get('config', {}).get('process', [{}])[0].get('datasets', [])
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 612 |
)
|
| 613 |
|
| 614 |
-
if
|
| 615 |
-
tags.append("image-to-image")
|
| 616 |
-
elif is_instruction:
|
| 617 |
tags.append("image-to-image")
|
| 618 |
elif is_video:
|
| 619 |
-
|
| 620 |
-
tags.append("image-to-video")
|
| 621 |
-
else:
|
| 622 |
-
tags.append("text-to-video")
|
| 623 |
else:
|
| 624 |
tags.append("text-to-image")
|
| 625 |
|
|
|
|
| 598 |
tags = []
|
| 599 |
lower_arch = (arch or "").lower()
|
| 600 |
lower_model_name = (model_config.get("name_or_path", "") or "").lower()
|
| 601 |
+
base_model_lower = (base_model or "").lower()
|
| 602 |
|
| 603 |
instruction_arches = {'flux_kontext', 'hidream_e1', 'qwen_image_edit'}
|
| 604 |
is_instruction = lower_arch in instruction_arches or 'kontext' in lower_model_name
|
| 605 |
+
is_edit_model = (
|
| 606 |
+
'edit' in lower_arch
|
| 607 |
+
or 'image_edit' in lower_arch
|
| 608 |
+
or 'edit' in lower_model_name
|
| 609 |
+
or 'image-edit' in lower_model_name
|
| 610 |
+
or 'edit' in base_model_lower
|
| 611 |
+
or 'image-edit' in base_model_lower
|
| 612 |
+
)
|
| 613 |
|
| 614 |
datasets_config = config.get('config', {}).get('process', [{}])[0].get('datasets', [])
|
| 615 |
+
sample_config = process_config.get('sample', {}) if isinstance(process_config, dict) else {}
|
| 616 |
+
sample_num_frames = sample_config.get('num_frames') if isinstance(sample_config, dict) else None
|
| 617 |
+
|
| 618 |
+
video_indicators = [
|
| 619 |
+
'video' in lower_arch,
|
| 620 |
+
'video' in lower_model_name,
|
| 621 |
+
'video' in base_model_lower,
|
| 622 |
+
't2v' in lower_arch,
|
| 623 |
+
't2v' in lower_model_name,
|
| 624 |
+
't2v' in base_model_lower,
|
| 625 |
+
'i2v' in lower_arch,
|
| 626 |
+
'i2v' in lower_model_name,
|
| 627 |
+
'i2v' in base_model_lower,
|
| 628 |
+
process_config.get('type') == 'video' if isinstance(process_config, dict) else False,
|
| 629 |
+
any(isinstance(dataset, dict) and dataset.get('do_i2v') for dataset in datasets_config),
|
| 630 |
+
isinstance(sample_num_frames, (int, float)) and sample_num_frames and sample_num_frames > 1,
|
| 631 |
+
lower_arch.startswith('wan'),
|
| 632 |
+
'wan' in lower_model_name,
|
| 633 |
+
'wan' in base_model_lower,
|
| 634 |
+
]
|
| 635 |
+
is_video = any(video_indicators)
|
| 636 |
+
is_i2v = any(
|
| 637 |
+
indicator
|
| 638 |
+
for indicator in [
|
| 639 |
+
'i2v' in lower_arch,
|
| 640 |
+
'i2v' in lower_model_name,
|
| 641 |
+
'i2v' in base_model_lower,
|
| 642 |
+
any(isinstance(dataset, dict) and dataset.get('do_i2v') for dataset in datasets_config),
|
| 643 |
+
]
|
| 644 |
)
|
| 645 |
|
| 646 |
+
if is_instruction or is_edit_model:
|
|
|
|
|
|
|
| 647 |
tags.append("image-to-image")
|
| 648 |
elif is_video:
|
| 649 |
+
tags.append("image-to-video" if is_i2v else "text-to-video")
|
|
|
|
|
|
|
|
|
|
| 650 |
else:
|
| 651 |
tags.append("text-to-image")
|
| 652 |
|
ui/src/app/jobs/new/SimplifiedJob.tsx
CHANGED
|
@@ -13,6 +13,7 @@ import { Button } from '@headlessui/react';
|
|
| 13 |
import { ChevronDown, ChevronUp, Upload } from 'lucide-react';
|
| 14 |
import { apiClient } from '@/utils/api';
|
| 15 |
import { addUserDataset, updateUserDatasetPath } from '@/utils/storage/datasetStorage';
|
|
|
|
| 16 |
|
| 17 |
type DatasetMode = 'upload' | 'existing';
|
| 18 |
|
|
@@ -53,6 +54,12 @@ const buildDatasetName = (base: string, suffix: string) => {
|
|
| 53 |
return `${slug}${suffix}`;
|
| 54 |
};
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
type SimplifiedJobProps = {
|
| 57 |
jobConfig: JobConfig;
|
| 58 |
setJobConfig: (value: any, key: string) => void;
|
|
@@ -135,6 +142,8 @@ export default function SimplifiedJob({
|
|
| 135 |
const [trainUploading, setTrainUploading] = useState(false);
|
| 136 |
const [trainUploadInfo, setTrainUploadInfo] = useState<string | null>(null);
|
| 137 |
const [trainUploadError, setTrainUploadError] = useState<string | null>(null);
|
|
|
|
|
|
|
| 138 |
|
| 139 |
const [controlDatasetMode, setControlDatasetMode] = useState<DatasetMode>('upload');
|
| 140 |
const [controlModeTouched, setControlModeTouched] = useState(hasCustomControlPath);
|
|
@@ -152,6 +161,8 @@ export default function SimplifiedJob({
|
|
| 152 |
const [controlUploading, setControlUploading] = useState(false);
|
| 153 |
const [controlUploadInfo, setControlUploadInfo] = useState<string | null>(null);
|
| 154 |
const [controlUploadError, setControlUploadError] = useState<string | null>(null);
|
|
|
|
|
|
|
| 155 |
|
| 156 |
const modelArch = useMemo(() => {
|
| 157 |
return modelArchs.find(arch => arch.name === process.model.arch);
|
|
@@ -486,6 +497,80 @@ export default function SimplifiedJob({
|
|
| 486 |
return datasetOptions;
|
| 487 |
}, [datasetOptions]);
|
| 488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
return (
|
| 490 |
<form onSubmit={handleSubmit} className="space-y-8">
|
| 491 |
<Card title="Training Setup">
|
|
@@ -685,6 +770,44 @@ export default function SimplifiedJob({
|
|
| 685 |
</div>
|
| 686 |
)}
|
| 687 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 688 |
<div className="grid grid-cols-1 md:grid-cols-2 gap-6 mt-6">
|
| 689 |
<NumberInput
|
| 690 |
label="Dataset Weight"
|
|
@@ -771,6 +894,44 @@ export default function SimplifiedJob({
|
|
| 771 |
/>
|
| 772 |
</div>
|
| 773 |
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 774 |
</div>
|
| 775 |
)}
|
| 776 |
</Card>
|
|
|
|
| 13 |
import { ChevronDown, ChevronUp, Upload } from 'lucide-react';
|
| 14 |
import { apiClient } from '@/utils/api';
|
| 15 |
import { addUserDataset, updateUserDatasetPath } from '@/utils/storage/datasetStorage';
|
| 16 |
+
import DatasetImageCard from '@/components/DatasetImageCard';
|
| 17 |
|
| 18 |
type DatasetMode = 'upload' | 'existing';
|
| 19 |
|
|
|
|
| 54 |
return `${slug}${suffix}`;
|
| 55 |
};
|
| 56 |
|
| 57 |
+
const extractFolderName = (value?: string | null) => {
|
| 58 |
+
if (!value) return '';
|
| 59 |
+
const parts = value.split(/[\\/]+/).filter(Boolean);
|
| 60 |
+
return parts.length ? parts[parts.length - 1] : '';
|
| 61 |
+
};
|
| 62 |
+
|
| 63 |
type SimplifiedJobProps = {
|
| 64 |
jobConfig: JobConfig;
|
| 65 |
setJobConfig: (value: any, key: string) => void;
|
|
|
|
| 142 |
const [trainUploading, setTrainUploading] = useState(false);
|
| 143 |
const [trainUploadInfo, setTrainUploadInfo] = useState<string | null>(null);
|
| 144 |
const [trainUploadError, setTrainUploadError] = useState<string | null>(null);
|
| 145 |
+
const [trainImages, setTrainImages] = useState<{ img_path: string }[]>([]);
|
| 146 |
+
const [trainImagesStatus, setTrainImagesStatus] = useState<'idle' | 'loading' | 'success' | 'error'>('idle');
|
| 147 |
|
| 148 |
const [controlDatasetMode, setControlDatasetMode] = useState<DatasetMode>('upload');
|
| 149 |
const [controlModeTouched, setControlModeTouched] = useState(hasCustomControlPath);
|
|
|
|
| 161 |
const [controlUploading, setControlUploading] = useState(false);
|
| 162 |
const [controlUploadInfo, setControlUploadInfo] = useState<string | null>(null);
|
| 163 |
const [controlUploadError, setControlUploadError] = useState<string | null>(null);
|
| 164 |
+
const [controlImages, setControlImages] = useState<{ img_path: string }[]>([]);
|
| 165 |
+
const [controlImagesStatus, setControlImagesStatus] = useState<'idle' | 'loading' | 'success' | 'error'>('idle');
|
| 166 |
|
| 167 |
const modelArch = useMemo(() => {
|
| 168 |
return modelArchs.find(arch => arch.name === process.model.arch);
|
|
|
|
| 497 |
return datasetOptions;
|
| 498 |
}, [datasetOptions]);
|
| 499 |
|
| 500 |
+
const trainDatasetIdentifier = useMemo(() => {
|
| 501 |
+
return (
|
| 502 |
+
trainDatasetResolvedName || extractFolderName(trainDatasetPath) || trainDatasetName || ''
|
| 503 |
+
).trim();
|
| 504 |
+
}, [trainDatasetName, trainDatasetPath, trainDatasetResolvedName]);
|
| 505 |
+
|
| 506 |
+
const controlDatasetIdentifier = useMemo(() => {
|
| 507 |
+
if (!requiresControlDataset) {
|
| 508 |
+
return '';
|
| 509 |
+
}
|
| 510 |
+
return (
|
| 511 |
+
controlDatasetResolvedName || extractFolderName(controlDatasetPath) || controlDatasetName || ''
|
| 512 |
+
).trim();
|
| 513 |
+
}, [controlDatasetName, controlDatasetPath, controlDatasetResolvedName, requiresControlDataset]);
|
| 514 |
+
|
| 515 |
+
const canLoadTrainImages = useMemo(
|
| 516 |
+
() => Boolean(trainDatasetIdentifier) && (trainDatasetMode === 'existing' || Boolean(trainDatasetResolvedName)),
|
| 517 |
+
[trainDatasetIdentifier, trainDatasetMode, trainDatasetResolvedName],
|
| 518 |
+
);
|
| 519 |
+
|
| 520 |
+
const canLoadControlImages = useMemo(
|
| 521 |
+
() =>
|
| 522 |
+
requiresControlDataset &&
|
| 523 |
+
Boolean(controlDatasetIdentifier) &&
|
| 524 |
+
(controlDatasetMode === 'existing' || Boolean(controlDatasetResolvedName)),
|
| 525 |
+
[controlDatasetIdentifier, controlDatasetMode, controlDatasetResolvedName, requiresControlDataset],
|
| 526 |
+
);
|
| 527 |
+
|
| 528 |
+
const refreshTrainDatasetImages = useCallback(() => {
|
| 529 |
+
if (!canLoadTrainImages) {
|
| 530 |
+
setTrainImages([]);
|
| 531 |
+
setTrainImagesStatus('idle');
|
| 532 |
+
return;
|
| 533 |
+
}
|
| 534 |
+
setTrainImagesStatus('loading');
|
| 535 |
+
apiClient
|
| 536 |
+
.post('/api/datasets/listImages', { datasetName: trainDatasetIdentifier })
|
| 537 |
+
.then(res => {
|
| 538 |
+
setTrainImages(res.data?.images || []);
|
| 539 |
+
setTrainImagesStatus('success');
|
| 540 |
+
})
|
| 541 |
+
.catch(error => {
|
| 542 |
+
console.error('Failed to load training dataset images:', error);
|
| 543 |
+
setTrainImagesStatus('error');
|
| 544 |
+
});
|
| 545 |
+
}, [canLoadTrainImages, trainDatasetIdentifier]);
|
| 546 |
+
|
| 547 |
+
const refreshControlDatasetImages = useCallback(() => {
|
| 548 |
+
if (!canLoadControlImages) {
|
| 549 |
+
setControlImages([]);
|
| 550 |
+
setControlImagesStatus('idle');
|
| 551 |
+
return;
|
| 552 |
+
}
|
| 553 |
+
setControlImagesStatus('loading');
|
| 554 |
+
apiClient
|
| 555 |
+
.post('/api/datasets/listImages', { datasetName: controlDatasetIdentifier })
|
| 556 |
+
.then(res => {
|
| 557 |
+
setControlImages(res.data?.images || []);
|
| 558 |
+
setControlImagesStatus('success');
|
| 559 |
+
})
|
| 560 |
+
.catch(error => {
|
| 561 |
+
console.error('Failed to load control dataset images:', error);
|
| 562 |
+
setControlImagesStatus('error');
|
| 563 |
+
});
|
| 564 |
+
}, [canLoadControlImages, controlDatasetIdentifier]);
|
| 565 |
+
|
| 566 |
+
useEffect(() => {
|
| 567 |
+
refreshTrainDatasetImages();
|
| 568 |
+
}, [refreshTrainDatasetImages]);
|
| 569 |
+
|
| 570 |
+
useEffect(() => {
|
| 571 |
+
refreshControlDatasetImages();
|
| 572 |
+
}, [refreshControlDatasetImages]);
|
| 573 |
+
|
| 574 |
return (
|
| 575 |
<form onSubmit={handleSubmit} className="space-y-8">
|
| 576 |
<Card title="Training Setup">
|
|
|
|
| 770 |
</div>
|
| 771 |
)}
|
| 772 |
|
| 773 |
+
{canLoadTrainImages && (
|
| 774 |
+
<div className="mt-6">
|
| 775 |
+
<div className="flex items-center justify-between mb-3">
|
| 776 |
+
<h3 className="text-sm font-semibold text-gray-200">Training Dataset Files</h3>
|
| 777 |
+
<button
|
| 778 |
+
type="button"
|
| 779 |
+
className="text-xs text-blue-400 hover:text-blue-300"
|
| 780 |
+
onClick={refreshTrainDatasetImages}
|
| 781 |
+
>
|
| 782 |
+
Refresh
|
| 783 |
+
</button>
|
| 784 |
+
</div>
|
| 785 |
+
{trainImagesStatus === 'loading' && (
|
| 786 |
+
<p className="text-xs text-gray-400">Loading dataset preview…</p>
|
| 787 |
+
)}
|
| 788 |
+
{trainImagesStatus === 'error' && (
|
| 789 |
+
<p className="text-xs text-red-400">
|
| 790 |
+
Unable to load files for this dataset. Ensure the folder exists and contains images or videos.
|
| 791 |
+
</p>
|
| 792 |
+
)}
|
| 793 |
+
{trainImagesStatus === 'success' && trainImages.length === 0 && (
|
| 794 |
+
<p className="text-xs text-gray-400">No files detected yet. Upload images to see them here.</p>
|
| 795 |
+
)}
|
| 796 |
+
{trainImagesStatus === 'success' && trainImages.length > 0 && (
|
| 797 |
+
<div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
|
| 798 |
+
{trainImages.map(image => (
|
| 799 |
+
<DatasetImageCard
|
| 800 |
+
key={image.img_path}
|
| 801 |
+
imageUrl={image.img_path}
|
| 802 |
+
alt="Training dataset file"
|
| 803 |
+
onDelete={refreshTrainDatasetImages}
|
| 804 |
+
/>
|
| 805 |
+
))}
|
| 806 |
+
</div>
|
| 807 |
+
)}
|
| 808 |
+
</div>
|
| 809 |
+
)}
|
| 810 |
+
|
| 811 |
<div className="grid grid-cols-1 md:grid-cols-2 gap-6 mt-6">
|
| 812 |
<NumberInput
|
| 813 |
label="Dataset Weight"
|
|
|
|
| 894 |
/>
|
| 895 |
</div>
|
| 896 |
)}
|
| 897 |
+
|
| 898 |
+
{canLoadControlImages && (
|
| 899 |
+
<div className="mt-6">
|
| 900 |
+
<div className="flex items-center justify-between mb-3">
|
| 901 |
+
<h3 className="text-sm font-semibold text-gray-200">Control Dataset Files</h3>
|
| 902 |
+
<button
|
| 903 |
+
type="button"
|
| 904 |
+
className="text-xs text-blue-400 hover:text-blue-300"
|
| 905 |
+
onClick={refreshControlDatasetImages}
|
| 906 |
+
>
|
| 907 |
+
Refresh
|
| 908 |
+
</button>
|
| 909 |
+
</div>
|
| 910 |
+
{controlImagesStatus === 'loading' && (
|
| 911 |
+
<p className="text-xs text-gray-400">Loading control dataset preview…</p>
|
| 912 |
+
)}
|
| 913 |
+
{controlImagesStatus === 'error' && (
|
| 914 |
+
<p className="text-xs text-red-400">
|
| 915 |
+
Unable to load files for this control dataset. Ensure the folder exists and has matching filenames.
|
| 916 |
+
</p>
|
| 917 |
+
)}
|
| 918 |
+
{controlImagesStatus === 'success' && controlImages.length === 0 && (
|
| 919 |
+
<p className="text-xs text-gray-400">No files detected yet. Upload matching control images.</p>
|
| 920 |
+
)}
|
| 921 |
+
{controlImagesStatus === 'success' && controlImages.length > 0 && (
|
| 922 |
+
<div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
|
| 923 |
+
{controlImages.map(image => (
|
| 924 |
+
<DatasetImageCard
|
| 925 |
+
key={image.img_path}
|
| 926 |
+
imageUrl={image.img_path}
|
| 927 |
+
alt="Control dataset file"
|
| 928 |
+
onDelete={refreshControlDatasetImages}
|
| 929 |
+
/>
|
| 930 |
+
))}
|
| 931 |
+
</div>
|
| 932 |
+
)}
|
| 933 |
+
</div>
|
| 934 |
+
)}
|
| 935 |
</div>
|
| 936 |
)}
|
| 937 |
</Card>
|