Spaces:
Running
Running
File size: 1,950 Bytes
ca97aa9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
import { Processor } from "../../base/processing_utils.js";
import { AutoImageProcessor } from "../auto/image_processing_auto.js";
import { AutoTokenizer } from "../../tokenizers.js";
import { RawImage } from "../../utils/image.js";
const IMAGE_TOKEN = "<|image|>";
const IMAGE_TOKEN_PATTERN = /<\|image_\d+\|>/g;
export class Phi3VProcessor extends Processor {
static image_processor_class = AutoImageProcessor
static tokenizer_class = AutoTokenizer
/**
*
* @param {string|string[]} text
* @param {RawImage|RawImage[]} images
* @param { { padding?: boolean, truncation?: boolean, num_crops?: number } | undefined } options
* @returns {Promise<any>}
*/
async _call(text, images = null, {
padding = true,
truncation = true,
num_crops = null,
} = {}) {
if (!Array.isArray(text)) {
text = [text];
}
let text_inputs, image_inputs;
if (images) {
image_inputs = await this.image_processor(images, { num_crops });
const { num_img_tokens } = image_inputs;
// The original implementation adds a bos_token before the image tokens
// TODO: Check if this affects performance, since it looks like a bug in the original implementation
const prompt_chunks = text.map((t, i) => t.split(IMAGE_TOKEN_PATTERN).join(IMAGE_TOKEN.repeat(num_img_tokens[i])));
text_inputs = this.tokenizer(prompt_chunks, { padding, truncation });
// The model expects image tokens to be negative, so we negate the image token ids
const image_token_id = this.tokenizer.model.convert_tokens_to_ids([IMAGE_TOKEN])[0];
text_inputs.input_ids.map_(id => (id == image_token_id) ? -id : id);
} else {
text_inputs = this.tokenizer(text);
}
return {
...text_inputs,
...image_inputs,
}
}
}
|