added initial files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,70 @@
----
-title: Vision
-emoji: 🌍
-colorFrom: indigo
-colorTo: yellow
-sdk: streamlit
-sdk_version: 1.37.1
-app_file: app.py
-pinned: false
-license: gemma
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Inhabitr Design Engine 1.0
+
+#### PaliGemma Vision Model(unifying attributes model)
+Leveraging the PaliGemma vision model to unify attribute extraction, this custom trained Model identifies and outputs furniture attributes—such as primary color and secondary color, primary material, secondary material, and design style—in JSON format.
+
+# Features
+Paligemma Model: A 3-billion parameter multi-model capable of generating captions from images, performing object detection, object segmentation ,OCR and VQA.
+Custom Training: The model is custom trained on approximately 500 pairs of cropped images (sofa, accent chair) along with their captions.
+
+# Requirements
+Python 3.9+
+
+Minimum  12GB of GPU  for local inference
+
+Pip (Python package installer)
+
+# Installation
+First, clone the repository:
+
+     git clone git clone https://Apoorva_inhabitr@bitbucket.org/Inhabitr/vision_models.git
+
+# Create and Activate Python Virtual Environment
+
+For Unix/macOS
+
+     python3 -m venv env
+     source env/bin/activate
+
+For Windows
+
+     python -m venv env
+     .\env\Scripts\activate
+
+# Install the required Python libraries
+
+    pip install -r requirements.txt
+
+# Download Model Weights and Tokenizer
+
+Download the model weights and tokenizer from Google Drive 
+
+https://drive.google.com/drive/folders/1S-z374V-yd3izeBatAMQlitmZ0mbMv6s?usp=drive_link.
+
+Place the downloaded files in the models/ folder located in the root directory of the project.
+
+# Start the API
+     python main.py
+If running locally, the API will be available at http://localhost:5000.
+
+# API Endpoints for Captions
+predict attributes
+
+POST /predict
+
+Request Parameters
+
+Send  an image  file 
+
+image: The image file used as the basis for the search.
+
+Example Request with Image File 
+
+      curl -X POST http://127.0.0.1:5000/predict \
+      -H "Content-Type: multipart/form-data" \
+      -F "file=@/path/to/your/image.jpg"
+
+# Performance Considerations
+GPU: Model parameters are loaded into memory for fast inference within 1 sec after initial loading.
+
+CPU: Initial model loading may take 4-6 minutes based on your system configuration.

app.py

@@ -0,0 +1,81 @@
+import streamlit as st
+from PIL import Image
+import os
+from fastapi import FastAPI, File, UploadFile
+from pydantic import BaseModel
+import uvicorn
+import threading
+from werkzeug.utils import secure_filename
+from utils import setup_and_predict
+from io import BytesIO
+
+# Configure upload folder and allowed extensions
+UPLOAD_FOLDER = 'uploads'
+ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'}
+
+if not os.path.exists(UPLOAD_FOLDER):
+    os.makedirs(UPLOAD_FOLDER)
+
+def allowed_file(filename):
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
+def save_uploaded_file(uploaded_file):
+    filename = secure_filename(uploaded_file.name)
+    filepath = os.path.join(UPLOAD_FOLDER, filename)
+    with open(filepath, "wb") as f:
+        f.write(uploaded_file.getbuffer())
+    return filepath
+
+# FastAPI app for Postman requests
+api = FastAPI()
+
+class PredictionResponse(BaseModel):
+    filename: str
+    result: str
+
+@api.post("/predict", response_model=PredictionResponse)
+async def predict_api(file: UploadFile = File(...)):
+    if allowed_file(file.filename):
+        filename = secure_filename(file.filename)
+        filepath = os.path.join(UPLOAD_FOLDER, filename)
+
+        with open(filepath, "wb") as f:
+            f.write(await file.read())
+
+        try:
+            result = setup_and_predict(filepath)
+        except Exception as e:
+            result = f"Unable to process the request: {e}"
+
+        return PredictionResponse(filename=filename, result=result)
+
+    return {"error": "Invalid file type"}
+
+def run_api():
+    uvicorn.run(api, host="0.0.0.0", port=8000)
+
+# Streamlit app
+st.title("Upload an Image for Prediction")
+
+uploaded_file = st.file_uploader("Choose an image...", type=['png', 'jpg', 'jpeg', 'gif'])
+
+if uploaded_file is not None and allowed_file(uploaded_file.name):
+    filepath = save_uploaded_file(uploaded_file)
+
+    try:
+        result = setup_and_predict(filepath)
+        st.image(filepath, caption='Uploaded Image.', use_column_width=True)
+        st.write("Prediction Result:")
+        st.write(result)
+    except Exception as e:
+        st.error(f"Unable to process the request: {e}")
+
+    # Option to download the image
+    with open(filepath, "rb") as file:
+        st.download_button(label="Download Image", data=file, file_name=uploaded_file.name)
+
+else:
+    st.warning("Please upload a valid image file.")
+
+# Run FastAPI in a separate thread
+threading.Thread(target=run_api).start()

paligemma_tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8986bb4f423f07f8c7f70d0dbe3526fb2316056c17bae71b1ea975e77a168fc6
+size 4264023

requirements.txt

@@ -0,0 +1,13 @@
+Flask
+jax
+jaxlib
+numpy
+tensorflow
+sentencepiece
+Pillow
+ml_collections
+overrides
+einops~=0.7
+numpy>=1.26
+flax
+pillow

utils.py

@@ -0,0 +1,204 @@
+import os
+import sys
+import base64
+import functools
+import html
+import io
+import warnings
+import jax
+import jax.numpy as jnp
+import numpy as np
+import ml_collections
+import tensorflow as tf
+import sentencepiece
+from PIL import Image
+
+# TPUs with
+if "COLAB_TPU_ADDR" in os.environ:
+  raise "It seems you are using Colab with remote TPUs which is not supported."
+# Append big_vision code to python import path
+if "big_vision_repo" not in sys.path:
+  sys.path.append("big_vision_repo")
+
+# Import model definition from big_vision
+from big_vision.models.proj.paligemma import paligemma
+from big_vision.trainers.proj.paligemma import predict_fns
+# Import big vision utilities
+import big_vision.datasets.jsonl
+import big_vision.utils
+import big_vision.sharding
+# Don't let TF use the GPU or TPUs
+tf.config.set_visible_devices([], "GPU")
+tf.config.set_visible_devices([], "TPU")
+backend = jax.lib.xla_bridge.get_backend()
+
+model_path = './Sofa-attributes-paligemma-ckpt.npz'
+tokenizer_path = './paligemma_tokenizer.model'
+
+# Define model
+model_config = ml_collections.FrozenConfigDict({
+    "llm": {"vocab_size": 257_152},
+    "img": {"variant": "So400m/14", "pool_type": "none", "scan": True, "dtype_mm": "float16"}
+})
+model = paligemma.Model(**model_config)
+tokenizer = sentencepiece.SentencePieceProcessor(tokenizer_path)
+# Load params - this can take up to 1 minute in T4 colabs.
+params = paligemma.load(None, model_path, model_config)
+# Define `decode` function to sample outputs from the model.
+decode_fn = predict_fns.get_all(model)['decode']
+decode = functools.partial(decode_fn, devices=jax.devices(), eos_token=tokenizer.eos_id())
+
+# Create a pytree mask of the trainable params.
+def is_trainable_param(name, param):  # pylint: disable=unused-argument
+  if name.startswith("llm/layers/attn/"):  return True
+  if name.startswith("llm/"):              return False
+  if name.startswith("img/"):              return False
+  raise ValueError(f"Unexpected param name {name}")
+trainable_mask = big_vision.utils.tree_map_with_names(is_trainable_param, params)
+# If more than one device is available (e.g. multiple GPUs) the parameters can
+# be sharded across them to reduce HBM usage per device.
+mesh = jax.sharding.Mesh(jax.devices(), ("data"))
+
+data_sharding = jax.sharding.NamedSharding(
+    mesh, jax.sharding.PartitionSpec("data"))
+
+params_sharding = big_vision.sharding.infer_sharding(
+    params, strategy=[('.*', 'fsdp(axis="data")')], mesh=mesh)
+
+# Yes: Some donated buffers are not usable.
+warnings.filterwarnings(
+    "ignore", message="Some donated buffers were not usable")
+
+@functools.partial(jax.jit, donate_argnums=(0,), static_argnums=(1,))
+def maybe_cast_to_f32(params, trainable):
+  return jax.tree.map(lambda p, m: p.astype(jnp.float32) if m else p,
+                      params, trainable)
+
+# Loading all params in simultaneous - albeit much faster and more succinct -
+# requires more RAM than the T4 colab runtimes have by default (12GB RAM).
+# Instead we do it param by param.
+params, treedef = jax.tree.flatten(params)
+sharding_leaves = jax.tree.leaves(params_sharding)
+trainable_leaves = jax.tree.leaves(trainable_mask)
+for idx, (sharding, trainable) in enumerate(zip(sharding_leaves, trainable_leaves)):
+  params[idx] = big_vision.utils.reshard(params[idx], sharding)
+  params[idx] = maybe_cast_to_f32(params[idx], trainable)
+  params[idx].block_until_ready()
+params = jax.tree.unflatten(treedef, params)
+
+# Print params to show what the model is made of.
+def parameter_overview(params):
+  for path, arr in big_vision.utils.tree_flatten_with_names(params)[0]:
+    print(f"{path:80s} {str(arr.shape):22s} {arr.dtype}")
+
+print(" == Model params == ")
+parameter_overview(params)
+def setup_and_predict(image_path):
+    # Preprocess image and tokens
+    def preprocess_image(image, size=224):
+      # Model has been trained to handle images of different aspects ratios
+      # resized to 224x224 in the range [-1, 1]. Bilinear and antialias resize
+      # options are helpful to improve quality in some tasks.
+      image = np.asarray(image)
+      if image.ndim == 2:  # Convert image without last channel into greyscale.
+        image = np.stack((image,)*3, axis=-1)
+      image = image[..., :3]  # Remove alpha layer.
+      assert image.shape[-1] == 3
+
+      image = tf.constant(image)
+      image = tf.image.resize(image, (size, size), method='bilinear', antialias=True)
+      return image.numpy() / 127.5 - 1.0  # [0, 255]->[-1,1]
+
+    def preprocess_tokens(prefix, suffix=None, seqlen=None):
+      # Model has been trained to handle tokenized text composed of a prefix with
+      # full attention and a suffix with causal attention.
+      separator = "\n"
+      tokens = tokenizer.encode(prefix, add_bos=True) + tokenizer.encode(separator)
+      mask_ar = [0] * len(tokens)    # 0 to use full attention for prefix.
+      mask_loss = [0] * len(tokens)  # 0 to not use prefix tokens in the loss.
+
+      if suffix:
+        suffix = tokenizer.encode(suffix, add_eos=True)
+        tokens += suffix
+        mask_ar += [1] * len(suffix)    # 1 to use causal attention for suffix.
+        mask_loss += [1] * len(suffix)  # 1 to use suffix tokens in the loss.
+
+      mask_input = [1] * len(tokens)    # 1 if its a token, 0 if padding.
+      if seqlen:
+        padding = [0] * max(0, seqlen - len(tokens))
+        tokens = tokens[:seqlen] + padding
+        mask_ar = mask_ar[:seqlen] + padding
+        mask_loss = mask_loss[:seqlen] + padding
+        mask_input = mask_input[:seqlen] + padding
+
+      return jax.tree.map(np.array, (tokens, mask_ar, mask_loss, mask_input))
+
+    def postprocess_tokens(tokens):
+      tokens = tokens.tolist()  # np.array to list[int]
+      try:  # Remove tokens at and after EOS if any.
+        eos_pos = tokens.index(tokenizer.eos_id())
+        tokens = tokens[:eos_pos]
+      except ValueError:
+        pass
+      return tokenizer.decode(tokens)
+
+    # Make predictions
+    # Evaluation/inference loop.
+    SEQLEN = 128
+    def make_predictions(data_iterator, *, num_examples=None,
+                        batch_size=4, seqlen=SEQLEN, sampler="greedy"):
+      outputs = []
+      while True:
+        # Construct a list of examples in the batch.
+        examples = []
+        try:
+          for _ in range(batch_size):
+            examples.append(next(data_iterator))
+            examples[-1]["_mask"] = np.array(True)  # Indicates true example.
+        except StopIteration:
+          if len(examples) == 0:
+            return outputs
+
+        # Not enough examples to complete a batch. Pad by repeating last example.
+        while len(examples) % batch_size:
+          examples.append(dict(examples[-1]))
+          examples[-1]["_mask"] = np.array(False)  # Indicates padding example.
+
+        # Convert list of examples into a dict of np.arrays and load onto devices.
+        batch = jax.tree.map(lambda *x: np.stack(x), *examples)
+        batch = big_vision.utils.reshard(batch, data_sharding)
+
+        # Make model predictions
+        tokens = decode({"params": params}, batch=batch,
+                        max_decode_len=seqlen, sampler=sampler)
+
+        # Fetch model predictions to device and detokenize.
+        tokens, mask = jax.device_get((tokens, batch["_mask"]))
+        tokens = tokens[mask]  # remove padding examples.
+        responses = [postprocess_tokens(t) for t in tokens]
+
+        # Append to html output.
+        for example, response in zip(examples, responses):
+          outputs.append((example["image"], response))
+          if num_examples and len(outputs) >= num_examples:
+            return outputs
+
+    def test_data_iterator(file_name):
+      image = Image.open(file_name)
+      image = preprocess_image(image)
+
+      prefix = "caption en"
+      tokens, mask_ar, _, mask_input = preprocess_tokens(prefix, seqlen=SEQLEN)
+
+      yield {
+          "image": np.asarray(image),
+          "text": np.asarray(tokens),
+          "mask_ar": np.asarray(mask_ar),
+          "mask_input": np.asarray(mask_input)
+      }
+
+    # Call the prediction function and print the result
+    image, caption = make_predictions(test_data_iterator(file_name=image_path), batch_size=1)[0]
+    return caption
+
+