apolinario commited on
Commit
7ab7ea8
·
1 Parent(s): d01d11a

dataset fix and tag fix

Browse files
ui/src/app/api/hf-jobs/route.ts CHANGED
@@ -598,28 +598,55 @@ def generate_model_card_readme(repo_id: str, config: dict, model_name: str, cura
598
  tags = []
599
  lower_arch = (arch or "").lower()
600
  lower_model_name = (model_config.get("name_or_path", "") or "").lower()
 
601
 
602
  instruction_arches = {'flux_kontext', 'hidream_e1', 'qwen_image_edit'}
603
  is_instruction = lower_arch in instruction_arches or 'kontext' in lower_model_name
 
 
 
 
 
 
 
 
604
 
605
  datasets_config = config.get('config', {}).get('process', [{}])[0].get('datasets', [])
606
- is_wan_video = lower_arch.startswith('wan2') or 'wan2' in lower_model_name
607
- is_video = (
608
- 'video' in lower_arch
609
- or process_config.get('type') == 'video'
610
- or any(isinstance(dataset, dict) and dataset.get('do_i2v') for dataset in datasets_config)
611
- or is_wan_video
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
612
  )
613
 
614
- if lower_arch.startswith('qwen_image_edit') or 'image-edit' in lower_model_name:
615
- tags.append("image-to-image")
616
- elif is_instruction:
617
  tags.append("image-to-image")
618
  elif is_video:
619
- if is_wan_video and ('i2v' in model_name.lower() or 'i2v' in lower_arch or 'i2v' in lower_model_name):
620
- tags.append("image-to-video")
621
- else:
622
- tags.append("text-to-video")
623
  else:
624
  tags.append("text-to-image")
625
 
 
598
  tags = []
599
  lower_arch = (arch or "").lower()
600
  lower_model_name = (model_config.get("name_or_path", "") or "").lower()
601
+ base_model_lower = (base_model or "").lower()
602
 
603
  instruction_arches = {'flux_kontext', 'hidream_e1', 'qwen_image_edit'}
604
  is_instruction = lower_arch in instruction_arches or 'kontext' in lower_model_name
605
+ is_edit_model = (
606
+ 'edit' in lower_arch
607
+ or 'image_edit' in lower_arch
608
+ or 'edit' in lower_model_name
609
+ or 'image-edit' in lower_model_name
610
+ or 'edit' in base_model_lower
611
+ or 'image-edit' in base_model_lower
612
+ )
613
 
614
  datasets_config = config.get('config', {}).get('process', [{}])[0].get('datasets', [])
615
+ sample_config = process_config.get('sample', {}) if isinstance(process_config, dict) else {}
616
+ sample_num_frames = sample_config.get('num_frames') if isinstance(sample_config, dict) else None
617
+
618
+ video_indicators = [
619
+ 'video' in lower_arch,
620
+ 'video' in lower_model_name,
621
+ 'video' in base_model_lower,
622
+ 't2v' in lower_arch,
623
+ 't2v' in lower_model_name,
624
+ 't2v' in base_model_lower,
625
+ 'i2v' in lower_arch,
626
+ 'i2v' in lower_model_name,
627
+ 'i2v' in base_model_lower,
628
+ process_config.get('type') == 'video' if isinstance(process_config, dict) else False,
629
+ any(isinstance(dataset, dict) and dataset.get('do_i2v') for dataset in datasets_config),
630
+ isinstance(sample_num_frames, (int, float)) and sample_num_frames and sample_num_frames > 1,
631
+ lower_arch.startswith('wan'),
632
+ 'wan' in lower_model_name,
633
+ 'wan' in base_model_lower,
634
+ ]
635
+ is_video = any(video_indicators)
636
+ is_i2v = any(
637
+ indicator
638
+ for indicator in [
639
+ 'i2v' in lower_arch,
640
+ 'i2v' in lower_model_name,
641
+ 'i2v' in base_model_lower,
642
+ any(isinstance(dataset, dict) and dataset.get('do_i2v') for dataset in datasets_config),
643
+ ]
644
  )
645
 
646
+ if is_instruction or is_edit_model:
 
 
647
  tags.append("image-to-image")
648
  elif is_video:
649
+ tags.append("image-to-video" if is_i2v else "text-to-video")
 
 
 
650
  else:
651
  tags.append("text-to-image")
652
 
ui/src/app/jobs/new/SimplifiedJob.tsx CHANGED
@@ -13,6 +13,7 @@ import { Button } from '@headlessui/react';
13
  import { ChevronDown, ChevronUp, Upload } from 'lucide-react';
14
  import { apiClient } from '@/utils/api';
15
  import { addUserDataset, updateUserDatasetPath } from '@/utils/storage/datasetStorage';
 
16
 
17
  type DatasetMode = 'upload' | 'existing';
18
 
@@ -53,6 +54,12 @@ const buildDatasetName = (base: string, suffix: string) => {
53
  return `${slug}${suffix}`;
54
  };
55
 
 
 
 
 
 
 
56
  type SimplifiedJobProps = {
57
  jobConfig: JobConfig;
58
  setJobConfig: (value: any, key: string) => void;
@@ -135,6 +142,8 @@ export default function SimplifiedJob({
135
  const [trainUploading, setTrainUploading] = useState(false);
136
  const [trainUploadInfo, setTrainUploadInfo] = useState<string | null>(null);
137
  const [trainUploadError, setTrainUploadError] = useState<string | null>(null);
 
 
138
 
139
  const [controlDatasetMode, setControlDatasetMode] = useState<DatasetMode>('upload');
140
  const [controlModeTouched, setControlModeTouched] = useState(hasCustomControlPath);
@@ -152,6 +161,8 @@ export default function SimplifiedJob({
152
  const [controlUploading, setControlUploading] = useState(false);
153
  const [controlUploadInfo, setControlUploadInfo] = useState<string | null>(null);
154
  const [controlUploadError, setControlUploadError] = useState<string | null>(null);
 
 
155
 
156
  const modelArch = useMemo(() => {
157
  return modelArchs.find(arch => arch.name === process.model.arch);
@@ -486,6 +497,80 @@ export default function SimplifiedJob({
486
  return datasetOptions;
487
  }, [datasetOptions]);
488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  return (
490
  <form onSubmit={handleSubmit} className="space-y-8">
491
  <Card title="Training Setup">
@@ -685,6 +770,44 @@ export default function SimplifiedJob({
685
  </div>
686
  )}
687
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  <div className="grid grid-cols-1 md:grid-cols-2 gap-6 mt-6">
689
  <NumberInput
690
  label="Dataset Weight"
@@ -771,6 +894,44 @@ export default function SimplifiedJob({
771
  />
772
  </div>
773
  )}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  </div>
775
  )}
776
  </Card>
 
13
  import { ChevronDown, ChevronUp, Upload } from 'lucide-react';
14
  import { apiClient } from '@/utils/api';
15
  import { addUserDataset, updateUserDatasetPath } from '@/utils/storage/datasetStorage';
16
+ import DatasetImageCard from '@/components/DatasetImageCard';
17
 
18
  type DatasetMode = 'upload' | 'existing';
19
 
 
54
  return `${slug}${suffix}`;
55
  };
56
 
57
+ const extractFolderName = (value?: string | null) => {
58
+ if (!value) return '';
59
+ const parts = value.split(/[\\/]+/).filter(Boolean);
60
+ return parts.length ? parts[parts.length - 1] : '';
61
+ };
62
+
63
  type SimplifiedJobProps = {
64
  jobConfig: JobConfig;
65
  setJobConfig: (value: any, key: string) => void;
 
142
  const [trainUploading, setTrainUploading] = useState(false);
143
  const [trainUploadInfo, setTrainUploadInfo] = useState<string | null>(null);
144
  const [trainUploadError, setTrainUploadError] = useState<string | null>(null);
145
+ const [trainImages, setTrainImages] = useState<{ img_path: string }[]>([]);
146
+ const [trainImagesStatus, setTrainImagesStatus] = useState<'idle' | 'loading' | 'success' | 'error'>('idle');
147
 
148
  const [controlDatasetMode, setControlDatasetMode] = useState<DatasetMode>('upload');
149
  const [controlModeTouched, setControlModeTouched] = useState(hasCustomControlPath);
 
161
  const [controlUploading, setControlUploading] = useState(false);
162
  const [controlUploadInfo, setControlUploadInfo] = useState<string | null>(null);
163
  const [controlUploadError, setControlUploadError] = useState<string | null>(null);
164
+ const [controlImages, setControlImages] = useState<{ img_path: string }[]>([]);
165
+ const [controlImagesStatus, setControlImagesStatus] = useState<'idle' | 'loading' | 'success' | 'error'>('idle');
166
 
167
  const modelArch = useMemo(() => {
168
  return modelArchs.find(arch => arch.name === process.model.arch);
 
497
  return datasetOptions;
498
  }, [datasetOptions]);
499
 
500
+ const trainDatasetIdentifier = useMemo(() => {
501
+ return (
502
+ trainDatasetResolvedName || extractFolderName(trainDatasetPath) || trainDatasetName || ''
503
+ ).trim();
504
+ }, [trainDatasetName, trainDatasetPath, trainDatasetResolvedName]);
505
+
506
+ const controlDatasetIdentifier = useMemo(() => {
507
+ if (!requiresControlDataset) {
508
+ return '';
509
+ }
510
+ return (
511
+ controlDatasetResolvedName || extractFolderName(controlDatasetPath) || controlDatasetName || ''
512
+ ).trim();
513
+ }, [controlDatasetName, controlDatasetPath, controlDatasetResolvedName, requiresControlDataset]);
514
+
515
+ const canLoadTrainImages = useMemo(
516
+ () => Boolean(trainDatasetIdentifier) && (trainDatasetMode === 'existing' || Boolean(trainDatasetResolvedName)),
517
+ [trainDatasetIdentifier, trainDatasetMode, trainDatasetResolvedName],
518
+ );
519
+
520
+ const canLoadControlImages = useMemo(
521
+ () =>
522
+ requiresControlDataset &&
523
+ Boolean(controlDatasetIdentifier) &&
524
+ (controlDatasetMode === 'existing' || Boolean(controlDatasetResolvedName)),
525
+ [controlDatasetIdentifier, controlDatasetMode, controlDatasetResolvedName, requiresControlDataset],
526
+ );
527
+
528
+ const refreshTrainDatasetImages = useCallback(() => {
529
+ if (!canLoadTrainImages) {
530
+ setTrainImages([]);
531
+ setTrainImagesStatus('idle');
532
+ return;
533
+ }
534
+ setTrainImagesStatus('loading');
535
+ apiClient
536
+ .post('/api/datasets/listImages', { datasetName: trainDatasetIdentifier })
537
+ .then(res => {
538
+ setTrainImages(res.data?.images || []);
539
+ setTrainImagesStatus('success');
540
+ })
541
+ .catch(error => {
542
+ console.error('Failed to load training dataset images:', error);
543
+ setTrainImagesStatus('error');
544
+ });
545
+ }, [canLoadTrainImages, trainDatasetIdentifier]);
546
+
547
+ const refreshControlDatasetImages = useCallback(() => {
548
+ if (!canLoadControlImages) {
549
+ setControlImages([]);
550
+ setControlImagesStatus('idle');
551
+ return;
552
+ }
553
+ setControlImagesStatus('loading');
554
+ apiClient
555
+ .post('/api/datasets/listImages', { datasetName: controlDatasetIdentifier })
556
+ .then(res => {
557
+ setControlImages(res.data?.images || []);
558
+ setControlImagesStatus('success');
559
+ })
560
+ .catch(error => {
561
+ console.error('Failed to load control dataset images:', error);
562
+ setControlImagesStatus('error');
563
+ });
564
+ }, [canLoadControlImages, controlDatasetIdentifier]);
565
+
566
+ useEffect(() => {
567
+ refreshTrainDatasetImages();
568
+ }, [refreshTrainDatasetImages]);
569
+
570
+ useEffect(() => {
571
+ refreshControlDatasetImages();
572
+ }, [refreshControlDatasetImages]);
573
+
574
  return (
575
  <form onSubmit={handleSubmit} className="space-y-8">
576
  <Card title="Training Setup">
 
770
  </div>
771
  )}
772
 
773
+ {canLoadTrainImages && (
774
+ <div className="mt-6">
775
+ <div className="flex items-center justify-between mb-3">
776
+ <h3 className="text-sm font-semibold text-gray-200">Training Dataset Files</h3>
777
+ <button
778
+ type="button"
779
+ className="text-xs text-blue-400 hover:text-blue-300"
780
+ onClick={refreshTrainDatasetImages}
781
+ >
782
+ Refresh
783
+ </button>
784
+ </div>
785
+ {trainImagesStatus === 'loading' && (
786
+ <p className="text-xs text-gray-400">Loading dataset preview…</p>
787
+ )}
788
+ {trainImagesStatus === 'error' && (
789
+ <p className="text-xs text-red-400">
790
+ Unable to load files for this dataset. Ensure the folder exists and contains images or videos.
791
+ </p>
792
+ )}
793
+ {trainImagesStatus === 'success' && trainImages.length === 0 && (
794
+ <p className="text-xs text-gray-400">No files detected yet. Upload images to see them here.</p>
795
+ )}
796
+ {trainImagesStatus === 'success' && trainImages.length > 0 && (
797
+ <div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
798
+ {trainImages.map(image => (
799
+ <DatasetImageCard
800
+ key={image.img_path}
801
+ imageUrl={image.img_path}
802
+ alt="Training dataset file"
803
+ onDelete={refreshTrainDatasetImages}
804
+ />
805
+ ))}
806
+ </div>
807
+ )}
808
+ </div>
809
+ )}
810
+
811
  <div className="grid grid-cols-1 md:grid-cols-2 gap-6 mt-6">
812
  <NumberInput
813
  label="Dataset Weight"
 
894
  />
895
  </div>
896
  )}
897
+
898
+ {canLoadControlImages && (
899
+ <div className="mt-6">
900
+ <div className="flex items-center justify-between mb-3">
901
+ <h3 className="text-sm font-semibold text-gray-200">Control Dataset Files</h3>
902
+ <button
903
+ type="button"
904
+ className="text-xs text-blue-400 hover:text-blue-300"
905
+ onClick={refreshControlDatasetImages}
906
+ >
907
+ Refresh
908
+ </button>
909
+ </div>
910
+ {controlImagesStatus === 'loading' && (
911
+ <p className="text-xs text-gray-400">Loading control dataset preview…</p>
912
+ )}
913
+ {controlImagesStatus === 'error' && (
914
+ <p className="text-xs text-red-400">
915
+ Unable to load files for this control dataset. Ensure the folder exists and has matching filenames.
916
+ </p>
917
+ )}
918
+ {controlImagesStatus === 'success' && controlImages.length === 0 && (
919
+ <p className="text-xs text-gray-400">No files detected yet. Upload matching control images.</p>
920
+ )}
921
+ {controlImagesStatus === 'success' && controlImages.length > 0 && (
922
+ <div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
923
+ {controlImages.map(image => (
924
+ <DatasetImageCard
925
+ key={image.img_path}
926
+ imageUrl={image.img_path}
927
+ alt="Control dataset file"
928
+ onDelete={refreshControlDatasetImages}
929
+ />
930
+ ))}
931
+ </div>
932
+ )}
933
+ </div>
934
+ )}
935
  </div>
936
  )}
937
  </Card>