E:\ComfyUI-aki-v1.3\models\LLM

configuration_florence2.py

@@ -77,7 +77,7 @@ class Florence2VisionConfig(PretrainedConfig):
     >>> configuration = model.config
     ```"""
 
-    model_type = "davit"
+    model_type = "florence2_vision"
     keys_to_ignore_at_inference = ["past_key_values"]
 
     def __init__(
@@ -327,7 +327,7 @@ class Florence2Config(PretrainedConfig):
         self.vocab_size = vocab_size
         self.projection_dim = projection_dim
         if vision_config is not None:
-            vision_config = Florence2VisionConfig(**vision_config)
+            vision_config = PretrainedConfig(**vision_config)
         self.vision_config = vision_config
         self.vocab_size = self.vocab_size
 

model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:03075d2d2d2bbd3e180b9ba0afae4aa8563226e2d32911656966e05b2f2ee060
-size 463221266

modeling_florence2.py

@@ -26,7 +26,7 @@ import torch.utils.checkpoint as checkpoint
 from torch.nn import CrossEntropyLoss 
 from collections import OrderedDict
 from einops import rearrange
-from timm.layers import DropPath, trunc_normal_
+from timm.models.layers import DropPath, trunc_normal_
 
 from transformers.modeling_utils import PreTrainedModel
 from transformers.generation.utils import GenerationMixin
@@ -610,10 +610,29 @@ class DaViT(nn.Module):
         self.avgpool = nn.AdaptiveAvgPool1d(1)
         self.head = nn.Linear(self.embed_dims[-1], num_classes) if num_classes > 0 else nn.Identity()
 
+        self.apply(self._init_weights)
+
     @property
     def dim_out(self):
         return self.embed_dims[-1]
 
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=0.02)
+            if m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.Conv2d):
+            nn.init.normal_(m.weight, std=0.02)
+            for name, _ in m.named_parameters():
+                if name in ['bias']:
+                    nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.weight, 1.0)
+            nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.BatchNorm2d):
+            nn.init.constant_(m.weight, 1.0)
+            nn.init.constant_(m.bias, 0)
+
     def forward_features_unpool(self, x):
         """
         forward until avg pooling 
@@ -1432,17 +1451,6 @@ class Florence2LanguagePreTrainedModel(PreTrainedModel):
             module.weight.data.normal_(mean=0.0, std=std)
             if module.padding_idx is not None:
                 module.weight.data[module.padding_idx].zero_()
-        elif isinstance(module, nn.Conv2d):
-            nn.init.normal_(module.weight, std=0.02)
-            for name, _ in module.named_parameters():
-                if name == "bias":
-                    nn.init.constant_(module.bias, 0)
-        elif isinstance(module, nn.LayerNorm):
-            nn.init.constant_(module.weight, 1.0)
-            nn.init.constant_(module.bias, 0)
-        elif isinstance(module, nn.BatchNorm2d):
-            nn.init.constant_(module.weight, 1.0)
-            nn.init.constant_(module.bias, 0)
 
     @property
     def dummy_inputs(self):
@@ -2066,20 +2074,14 @@ class Florence2LanguageForConditionalGeneration(Florence2LanguagePreTrainedModel
         # Initialize weights and apply final processing
         self.post_init()
 
-    def _tie_weights(self):
-        if self.config.tie_word_embeddings:
-            self._tie_or_clone_weights(self.model.encoder.embed_tokens, self.model.shared)
-            self._tie_or_clone_weights(self.model.decoder.embed_tokens, self.model.shared)
-            self._tie_or_clone_weights(self.lm_head, self.model.shared)
-
     def get_encoder(self):
         return self.model.get_encoder()
 
     def get_decoder(self):
         return self.model.get_decoder()
 
-    def resize_token_embeddings(self, new_num_tokens: int, pad_to_multiple_of: Optional[int] = None, **kwargs) -> nn.Embedding:
-        new_embeddings = super().resize_token_embeddings(new_num_tokens, pad_to_multiple_of, **kwargs)
+    def resize_token_embeddings(self, new_num_tokens: int, pad_to_multiple_of: Optional[int] = None) -> nn.Embedding:
+        new_embeddings = super().resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
         self._resize_final_logits_bias(new_embeddings.weight.shape[0])
         return new_embeddings
 
@@ -2529,8 +2531,6 @@ class Florence2VisionModelWithProjection(Florence2PreTrainedModel):
     FLORENCE2_START_DOCSTRING,
 )
 class Florence2ForConditionalGeneration(Florence2PreTrainedModel):
-    _tied_weights_keys = ["language_model.encoder.embed_tokens.weight", "language_model.decoder.embed_tokens.weight", "language_model.lm_head.weight"]
-
     def __init__(self, config: Florence2Config):
         super().__init__(config)
         assert config.vision_config.model_type == 'davit', 'only DaViT is supported for now'
@@ -2545,6 +2545,8 @@ class Florence2ForConditionalGeneration(Florence2PreTrainedModel):
 
         language_model = Florence2LanguageForConditionalGeneration(config=config.text_config)
 
+        if language_model._tied_weights_keys is not None:
+            self._tied_weights_keys = [f"language_model.{k}" for k in language_model._tied_weights_keys]
         self.language_model = language_model
 
         self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1
@@ -2587,8 +2589,8 @@ class Florence2ForConditionalGeneration(Florence2PreTrainedModel):
     def get_input_embeddings(self):
         return self.language_model.get_input_embeddings()
 
-    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None, **kwargs) -> nn.Embedding:
-        model_embeds = self.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of, **kwargs)
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
         # update vocab size
         self.config.text_config.vocab_size = model_embeds.num_embeddings
         self.config.vocab_size = model_embeds.num_embeddings

processing_florence2.py

@@ -20,7 +20,6 @@ import re
 import logging
 from typing import List, Optional, Union
 import numpy as np
-import math
 
 import torch
 
@@ -33,7 +32,6 @@ from transformers.tokenization_utils_base import (
     TextInput,
     TruncationStrategy,
 )
-from transformers import BartTokenizer, BartTokenizerFast
 from transformers.utils import TensorType
 
 
@@ -306,7 +304,7 @@ class Florence2Processor(ProcessorMixin):
         image_processor_input_names = self.image_processor.model_input_names
         return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
 
-    def post_process_generation(self, text=None, sequence=None, transition_beam_score=None, task=None, image_size=None):
+    def post_process_generation(self, text, task, image_size):
         """
         Post-process the output of the model to each of the task outputs.
 
@@ -319,8 +317,6 @@ class Florence2Processor(ProcessorMixin):
         task_answer_post_processing_type = self.tasks_answer_post_processing_type.get(task, 'pure_text')
         task_answer = self.post_processor(
             text=text,
-            sequence=sequence,
-            transition_beam_score=transition_beam_score,
             image_size=image_size,
             parse_tasks=task_answer_post_processing_type,
         )[task_answer_post_processing_type]
@@ -334,9 +330,6 @@ class Florence2Processor(ProcessorMixin):
             bboxes_od = [_od_instance['bbox'] for _od_instance in od_instances]
             labels_od = [str(_od_instance['cat_name']) for _od_instance in od_instances]
             final_answer = {'bboxes': bboxes_od, 'labels': labels_od}
-            if len(od_instances) and 'score' in od_instances[0]:
-                scores_od = [_od_instance['score'] for _od_instance in od_instances]
-                final_answer['scores'] = scores_od
         elif task_answer_post_processing_type in ['ocr']:
             bboxes = [_od_instance['quad_box'] for _od_instance in task_answer]
             labels = [str(_od_instance['text']) for _od_instance in task_answer]
@@ -503,7 +496,7 @@ class CoordinatesQuantizer(object):
 
 
 class Florence2PostProcesser(object):
-    r"""
+    """
     Florence-2 post process for converting text prediction to various tasks results. 
 
     Args:
@@ -598,8 +591,7 @@ class Florence2PostProcesser(object):
             'PARSE_TASKS': [
                 {
                     'TASK_NAME': 'od',
-                    'PATTERN': r'([a-zA-Z0-9 ]+)<loc_(\\d+)><loc_(\\d+)><loc_(\\d+)><loc_(\\d+)>',
-                    'SCORE_MODE': 'avg_loc_scores'
+                    'PATTERN': r'([a-zA-Z0-9 ]+)<loc_(\\d+)><loc_(\\d+)><loc_(\\d+)><loc_(\\d+)>'
                 },
                 {
                     'TASK_NAME': 'ocr',
@@ -615,7 +607,6 @@ class Florence2PostProcesser(object):
                 },
                 {
                     'TASK_NAME': 'description_with_bboxes',
-                    'SCORE_MODE': 'avg_loc_scores'
                 },
                 {
                     'TASK_NAME': 'description_with_polygons',
@@ -657,6 +648,9 @@ class Florence2PostProcesser(object):
             token_ids, skip_special_tokens=False)
         assert len(filtered_tokens) == len(token_ids)
 
+        # To avoid mixing byte-level and unicode for byte-level BPT
+        # we need to build string separately for added tokens and byte-level tokens
+        # cf. https://github.com/huggingface/transformers/issues/1133
         sub_texts = []
         for token in filtered_tokens:
             if token in self.all_special_tokens:
@@ -664,6 +658,10 @@ class Florence2PostProcesser(object):
             else:
                 if isinstance(tokenizer, (BartTokenizer, BartTokenizerFast)):
                     sub_text = tokenizer.convert_tokens_to_string([token])
+                elif isinstance(tokenizer, (T5Tokenizer, T5TokenizerFast)):
+                    # Ref: https://github.com/google/sentencepiece#whitespace-is-treated-as-a-basic-symbol
+                    # Note: Do not strip sub_text as it may have functional whitespace
+                    sub_text = token.replace('▁', ' ')
                 else:
                     raise ValueError(f'type {type(tokenizer)} not supported')
                 sub_texts.append(sub_text)
@@ -675,6 +673,13 @@ class Florence2PostProcesser(object):
             text += sub_text
             spans.append(span)
 
+        # Text format:
+        # 1. T5Tokenizer/T5TokenizerFast: 
+        #      "<loc_1><loc_2><loc_3><loc_4> transplanting dog<loc_1><loc_2><loc_3><loc_4> cat</s>"
+        #    Equivalent to t5_tokenizer.decode(input_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False, spaces_between_special_tokens=False)
+        # 2. BartTokenizer (need to double check):
+        #      "<s><loc_1><loc_2><loc_3><loc_4>transplanting dog<loc_1><loc_2><loc_3><loc_4>cat</s>"
+        #    Equivalent to bart_tokenizer.decode(input_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False, spaces_between_special_tokens=False)
         return text, spans
 
     def parse_od_from_text_and_spans(
@@ -709,7 +714,7 @@ class Florence2PostProcesser(object):
         return instances
 
     def parse_ocr_from_text_and_spans(self, 
-                                     text, 
+                                    text, 
                                      pattern, 
                                      image_size,
                                      area_threshold=-1.0,
@@ -813,26 +818,9 @@ class Florence2PostProcesser(object):
 
         return instances
 
-    def parse_description_with_bboxes_from_text_and_spans(
-            self, 
-            text, 
-            spans=None,
-            scores=None,
-            score_mode=None,
-            pattern=None, 
-            image_size=None, 
-            allow_empty_phrase=False
-        ):
-        def find_matched_token_indices(cur_span, token_spans):
-            inds = []
-            for i, token_span in enumerate(token_spans):
-                if not (token_span[1] <= cur_span[0] or token_span[0] >= cur_span[1]):
-                    inds.append(i)
-            return inds
-
-        cur_span = 0
-        if text.startswith('<s>'):   
-            cur_span += 3
+    def parse_description_with_bboxes_from_text_and_spans(self, text, pattern, image_size, allow_empty_phrase=False):
+        # temporary parse solution, split by '.'
+        # ignore <s> </s> and <pad>
 
         text = text.replace('<s>', '')
         text = text.replace('</s>', '')
@@ -854,16 +842,13 @@ class Florence2PostProcesser(object):
             phrase_text_strip = pharse_text.replace('<obj>', '', 1)
 
             if phrase_text_strip == '' and not allow_empty_phrase:
-                cur_span += len(pharse_text)
                 continue
 
             # parse phrase, get string 
             phrase = re.search(pattern, phrase_text_strip)
             if phrase is None:
-                cur_span += len(pharse_text)
                 continue
 
-            phrase_span = phrase.span()
             phrase = phrase.group()
             # remove leading and trailing spaces
             phrase = phrase.strip()
@@ -871,7 +856,6 @@ class Florence2PostProcesser(object):
             # parse bboxes by box_pattern
             bboxes_parsed = list(re.finditer(box_pattern, pharse_text))
             if len(bboxes_parsed) == 0:
-                cur_span += len(pharse_text)
                 continue
 
             # a list of list 
@@ -882,42 +866,14 @@ class Florence2PostProcesser(object):
                 size=image_size
             ).tolist()  
 
-            if score_mode == 'avg_loc_scores':
-                if spans is None or scores is None:
-                    all_scores = None
-                else:
-                    bbox_end_spans = [_bboxes_parsed.span(0) for _bboxes_parsed in bboxes_parsed]
-                    all_scores = []
-                    for _spans in bbox_end_spans:
-                        token_inds = find_matched_token_indices((_spans[0] + cur_span, _spans[1]+ cur_span), spans)
-                        loc_scores = [scores[token_i] for token_i in token_inds]
-                        score = sum(loc_scores) / len(loc_scores)
-                        all_scores.append(score)
-            elif score_mode == 'avg_cat_name_scores': 
-                if spans is None or scores is None:
-                    all_scores = None
-                else:
-                    cat_name_token_inds = find_matched_token_indices((phrase_span[0] + cur_span, phrase_span[1]+cur_span), spans)
-                    cat_name_scores = [scores[token_i] for token_i in cat_name_token_inds]
-                    score = sum(cat_name_scores) / len(cat_name_scores)
-                    all_scores = [score] * len(bboxes)
-            elif score_mode is None:
-                all_scores = None
-            else:
-                raise ValueError('Unknown score mode: {}'.format(score_mode))
-
             phrase = phrase.encode('ascii',errors='ignore').decode('ascii')
-            for _idx, _bboxes in enumerate(bboxes):
+            for _bboxes in bboxes:
                 # Prepare instance.
                 instance = {}
                 instance['bbox'] = _bboxes
                 # exclude non-ascii characters
                 instance['cat_name'] = phrase
-                if all_scores is not None:
-                    instance['score'] = math.exp(all_scores[_idx])
                 instances.append(instance)
-            
-            cur_span += len(pharse_text)
 
         return instances
 
@@ -1035,8 +991,6 @@ class Florence2PostProcesser(object):
     def __call__(
         self,
         text=None,
-        sequence=None,
-        transition_beam_score=None,
         image_size=None,
         parse_tasks=None,
     ):
@@ -1045,6 +999,7 @@ class Florence2PostProcesser(object):
             text: model outputs
             image_size: (width, height)
             parse_tasks: a list of tasks to parse, if None, parse all tasks.
+
         """
         if parse_tasks is not None:
             if isinstance(parse_tasks, str):
@@ -1053,18 +1008,7 @@ class Florence2PostProcesser(object):
                 assert _parse_task in self.parse_tasks, f'parse task {_parse_task} not supported'
         
         # sequence or text should be provided 
-        assert sequence is not None or text is not None, 'sequence or text should be provided'
-        assert sequence is None or text is None, 'only one of sequence and text should be provided'
-
-        if sequence is not None:
-            sequence = sequence.tolist()[1:]
-            text, spans = self.decode_with_spans(self.tokenizer, sequence)
-            if transition_beam_score is not None:
-                transition_beam_score = transition_beam_score.tolist()
-                assert len(sequence) == len(transition_beam_score)
-        else:
-            spans = None
-            transition_beam_score = None
+        assert text is not None, 'text should be provided'
 
         parsed_dict = {
             'text': text
@@ -1075,7 +1019,6 @@ class Florence2PostProcesser(object):
                 continue
 
             pattern = self.parse_tasks_configs[task].get('PATTERN', None)
-            score_mode = self.parse_tasks_configs[task].get('SCORE_MODE', None)
 
             if task == 'ocr':
                 instances = self.parse_ocr_from_text_and_spans(
@@ -1097,9 +1040,6 @@ class Florence2PostProcesser(object):
             elif task == 'description_with_bboxes':
                 instances = self.parse_description_with_bboxes_from_text_and_spans( 
                     text,
-                    spans=spans,
-                    scores=transition_beam_score,
-                    score_mode=score_mode,
                     pattern=pattern,
                     image_size=image_size,
                 )