init

data/__init__.py

@@ -0,0 +1,3 @@
+from .hicodet import HICODet
+
+__all__ = ["HICODet"]

data/constants.py

@@ -0,0 +1,63 @@
+COCO_KEYPOINT_NAME = [
+    'nose', 'left eye', 'right eye', 'left ear', 'right ear', 'left shoulder', 'right shoulder', 'left elbow', 'right elbow', 'left wrist', 'right wrist', 'left hip', 'right hip', 'left knee', 'right knee', 'left ankle', 'right ankle'
+]
+
+COCO_KEYPOINT_NAME_TOKEN = [
+    '<nose>', '<left_eye>',  '<right_eye>',  '<left_ear>', '<right_ear>', '<left_shoulder>', '<right_shoulder>', '<left_elbow>', '<right_elbow>', '<left_wrist>', '<right_wrist>', '<left_hip>', '<right_hip>', '<left_knee>', '<right_knee>', '<left_ankle>', '<right_ankle>'
+]
+
+PART_ORDER = ["right foot", "right leg", "left leg", "left foot", "hip", "head", "right hand", "right arm", "left arm", "left hand"]
+
+KeypointLocationDescription = {
+    'nose': 'The nose is the central, protruding feature on their face, located just above the upper lip.',
+    'left eye': 'The left eye is the visual organ on the left side of their face, typically located above the left cheek and beside the nose.',
+    'right eye': 'The right eye is the visual organ on the right side of their face, typically located above the right cheek and beside the nose.',
+    'left ear': 'The left ear is the auditory organ on the left side of their head, typically located to the side of the left temple.',
+    'right ear': 'The right ear is the auditory organ on the right side of their head, typically located to the side of the right temple.',
+    'left shoulder': 'The left shoulder is the joint connecting the left arm and the torso, typically situated on the upper left side of the chest.',
+    'right shoulder': 'The right shoulder is the joint connecting the right arm and the torso, typically situated on the upper right side of the chest.',
+    'left elbow': 'The left elbow is the joint connecting the left upper arm and the left forearm, typically situated in the middle of the left arm, between left shoulder and left wrist.',
+    'right elbow': 'The right elbow is the joint connecting the right upper arm and the right forearm, typically situated in the middle of the right arm, between right shoulder and right wrist.',
+    'left wrist': 'The left wrist is the joint connecting the left forearm and the left hand, typically located at the base of the left hand.',
+    'right wrist': 'The right wrist is the joint connecting the right forearm and the right hand, typically located at the base of the right hand.',
+    'left hip': 'The left hip is the joint connecting the left thigh to the pelvis, typically located on the left side of the lower torso.',
+    'right hip': 'The right hip is the joint connecting the right thigh to the pelvis, typically located on the right side of the lower torso.',
+    'left knee': 'The left knee is the joint connecting the left thigh and the left lower leg, typically situated in the middle of the left leg, it is located between the left hip and left ankle.',
+    'right knee': 'The right knee is the joint connecting the upper leg and lower leg on the right side, it is located between the right hip and right ankle.',
+    'left ankle': 'The left ankle is the joint connecting the left lower leg and the left foot, typically located at the base of the left leg.',
+    'right ankle': 'The right ankle is the joint connecting the right lower leg and the right foot, typically located at the base of the right leg.',
+    'neck': "The neck is the part of the body connecting the head to the torso, typically situated between the shoulders.",
+    'torso': "The torso is the central part of the body, typically encompassing the chest, abdomen, and back.",
+    'pelvis': "The pelvis is the bony structure that forms the base of the spine and connects the torso to the lower body, typically located between the left hip and right hip.",
+    'left toe': "The left toe is located at the end of the left foot, typically at the front or tip of the foot.",
+    'right toe': "The right toe is the digit located at the end of the right foot, typically at the front or tip of the foot.",
+    'head_top': "The head_top keypoint is the highest point on their head, typically at the crown.",
+    'thorax': "The thorax is the central part of the torso, typically located between the neck and the abdomen."
+}
+
+KeypointLocationQuestion = {
+    'nose': ["Please provide the coordinates for the nose of the person, which is the central, protruding feature on their face, located just above the upper lip.", "I'd like to know the precise coordinates of the nose, positioned at the center of the person's face, slightly below the eyes and between them.", "Where can I find the coordinates for the nose of this individual? It's the prominent feature in the middle of their facial structure.", "Could you please output the coordinates for the nose, which is typically situated in the middle of the face, between the eyes and above the upper lip?", "I'm interested in the coordinates of the nose. It's the facial feature at the center of the face, responsible for the sense of smell.", "Where are the coordinates for the nose? It's the prominent part of the face located between the eyes and just above the upper lip.", "Please provide the coordinates for the nose, found at the center of the person's face, between the eyes and above the upper lip.", "I'd like to see the coordinates of the nose, which is positioned in the middle of the face, slightly below the eyes and between them.", "Where can I locate the coordinates for the nose of this person? It's the central feature on their face, responsible for smelling.", "Could you output the coordinates for the nose, the central facial feature located between the eyes and just above the upper lip?"], 
+    'left eye': ["Please provide the coordinates for the left eye of the person, which is the visual organ on the left side of their face, typically located above the left cheek and beside the nose.", "I'd like to know the precise coordinates of the left eye, situated on the left side of the person's face, to the side of the nose and above the left cheek.", "Where can I find the coordinates for the left eye of this individual? It's the organ responsible for vision on the left side of their face.", "Could you please output the coordinates for the left eye, which is typically found on the left side of the face, beside the nose and above the left cheek?", "I'm interested in the coordinates of the left eye. It's the visual organ on the left side of the face, adjacent to the nose and above the left cheek.", "Where are the coordinates for the left eye? It's the eye situated on the left side of the face, above the left cheek, and beside the nose.", "Please provide the coordinates for the left eye, located on the left side of the person's face, beside the nose and above the left cheek.", "I'd like to see the coordinates of the left eye, which is positioned on the left side of the face, above the left cheek, and adjacent to the nose.", "Where can I locate the coordinates for the left eye of this person? It's the visual organ responsible for left-sided vision.", "Could you output the coordinates for the left eye, the eye on the left side of the face, above the left cheek, and beside the nose?"],
+    'right eye': ["Please provide the coordinates for the right eye of the person, which is the visual organ on the right side of their face, typically located above the right cheek and beside the nose.", "I'd like to know the precise coordinates of the right eye, situated on the right side of the person's face, to the side of the nose and above the right cheek.", "Where can I find the coordinates for the right eye of this individual? It's the organ responsible for vision on the right side of their face.", "Could you please output the coordinates for the right eye, which is typically found on the right side of the face, beside the nose and above the right cheek?", "I'm interested in the coordinates of the right eye. It's the visual organ on the right side of the face, adjacent to the nose and above the right cheek.", "Where are the coordinates for the right eye? It's the eye situated on the right side of the face, above the right cheek, and beside the nose.", "Please provide the coordinates for the right eye, located on the right side of the person's face, beside the nose and above the right cheek.", "I'd like to see the coordinates of the right eye, which is positioned on the right side of the face, above the right cheek, and adjacent to the nose.", "Where can I locate the coordinates for the right eye of this person? It's the visual organ responsible for right-sided vision.", "Could you output the coordinates for the right eye, the eye on the right side of the face, above the right cheek, and beside the nose?"],
+    'left ear': ["Please provide the coordinates for the left ear of the person, which is the auditory organ on the left side of their head, typically located to the side of the left temple.", "I'd like to know the precise coordinates of the left ear, situated on the left side of the person's head, to the side of the left temple and above the left jawline.", "Where can I find the coordinates for the left ear of this individual? It's the auditory organ on the left side of their head.", "Could you please output the coordinates for the left ear, which is typically found on the left side of the head, to the side of the left temple and above the left jawline?", "I'm interested in the coordinates of the left ear. It's the auditory organ on the left side of the head, adjacent to the left temple and above the left jawline.", "Where are the coordinates for the left ear? It's the ear situated on the left side of the head, beside the left temple and above the left jawline.", "Please provide the coordinates for the left ear, located on the left side of the person's head, beside the left temple and above the left jawline.", "I'd like to see the coordinates of the left ear, which is positioned on the left side of the head, above the left jawline, and adjacent to the left temple.", "Where can I locate the coordinates for the left ear of this person? It's the auditory organ on the left side of the head.", "Could you output the coordinates for the left ear, the ear on the left side of the head, above the left jawline, and beside the left temple?"],
+    'right ear': ["Please provide the coordinates for the right ear of the person, which is the auditory organ on the right side of their head, typically located to the side of the right temple.", "I'd like to know the precise coordinates of the right ear, situated on the right side of the person's head, to the side of the right temple and above the right jawline.", "Where can I find the coordinates for the right ear of this individual? It's the auditory organ on the right side of their head.", "Could you please output the coordinates for the right ear, which is typically found on the right side of the head, to the side of the right temple and above the right jawline?", "I'm interested in the coordinates of the right ear. It's the auditory organ on the right side of the head, adjacent to the right temple and above the right jawline.", "Where are the coordinates for the right ear? It's the ear situated on the right side of the head, beside the right temple and above the right jawline.", "Please provide the coordinates for the right ear, located on the right side of the person's head, beside the right temple and above the right jawline.", "I'd like to see the coordinates of the right ear, which is positioned on the right side of the head, above the right jawline, and adjacent to the right temple.", "Where can I locate the coordinates for the right ear of this person? It's the auditory organ on the right side of the head.", "Could you output the coordinates for the right ear, the ear on the right side of the head, above the right jawline, and beside the right temple?"],
+    'left shoulder': ["Please provide the coordinates for the left shoulder of the person, which is the joint connecting the left arm and the torso, typically situated on the upper left side of the chest.", "I'd like to know the precise coordinates of the left shoulder, located on the upper part of the person's chest, on the left side.", "Where can I find the coordinates for the left shoulder of this individual? It's the joint connecting the left arm to the upper body.", "Could you please output the coordinates for the left shoulder, typically found on the upper left side of the chest, where the arm meets the torso?", "I'm interested in the coordinates of the left shoulder. It's the joint on the upper left part of the chest that allows arm movement.", "Where are the coordinates for the left shoulder? It's the shoulder joint situated on the upper left side of the chest.", "Please provide the coordinates for the left shoulder, located on the upper left side of the chest, where the arm attaches to the torso.", "I'd like to see the coordinates of the left shoulder, which is positioned on the upper part of the chest, on the left side.", "Where can I locate the coordinates for the left shoulder of this person? It's the joint that connects the left arm to the upper body.", "Could you output the coordinates for the left shoulder, the joint on the upper left part of the chest where the arm articulates with the torso?"],
+    'right shoulder': ["Please provide the coordinates for the right shoulder of the person, which is the joint connecting the right arm and the torso, typically situated on the upper right side of the chest.", "I'd like to know the precise coordinates of the right shoulder, located on the upper part of the person's chest, on the right side.", "Where can I find the coordinates for the right shoulder of this individual? It's the joint connecting the right arm to the upper body.", "Could you please output the coordinates for the right shoulder, typically found on the upper right side of the chest, where the arm meets the torso?", "I'm interested in the coordinates of the right shoulder. It's the joint on the upper right part of the chest that allows arm movement.", "Where are the coordinates for the right shoulder? It's the shoulder joint situated on the upper right side of the chest.", "Please provide the coordinates for the right shoulder, located on the upper right side of the chest, where the arm attaches to the torso.", "I'd like to see the coordinates of the right shoulder, which is positioned on the upper part of the chest, on the right side.", "Where can I locate the coordinates for the right shoulder of this person? It's the joint that connects the right arm to the upper body.", "Could you output the coordinates for the right shoulder, the joint on the upper right part of the chest where the arm articulates with the torso?"],
+    'left elbow': ["Please provide the coordinates for the left elbow of the person, which is the joint connecting the left upper arm (humerus) and the left forearm (radius and ulna), typically situated in the middle of the left arm.", "I'd like to know the precise coordinates of the left elbow, located in the middle of the left arm, where the upper arm meets the forearm.", "Where can I find the coordinates for the left elbow of this individual? It's the joint responsible for bending and straightening the left arm.", "Could you please output the coordinates for the left elbow, typically found in the center of the left arm, where the upper arm and forearm meet?", "I'm interested in the coordinates of the left elbow. It's the joint on the left arm that facilitates arm movement.", "Where are the coordinates for the left elbow? It's the elbow joint situated in the middle of the left arm.", "Please provide the coordinates for the left elbow, located in the middle of the left arm, where the upper arm connects to the forearm.", "I'd like to see the coordinates of the left elbow, which is positioned in the center of the left arm.", "Where can I locate the coordinates for the left elbow of this person? It's the joint that allows flexion and extension of the left arm.", "Could you output the coordinates for the left elbow, the joint in the middle of the left arm responsible for arm movement?"],
+    'right elbow': ["Please provide the coordinates for the right elbow of the person, which is the joint connecting the right upper arm (humerus) and the right forearm (radius and ulna), typically situated in the middle of the right arm.", "I'd like to know the precise coordinates of the right elbow, located in the middle of the right arm, where the upper arm meets the forearm.", "Where can I find the coordinates for the right elbow of this individual? It's the joint responsible for bending and straightening the right arm.", "Could you please output the coordinates for the right elbow, typically found in the center of the right arm, where the upper arm and forearm meet?", "I'm interested in the coordinates of the right elbow. It's the joint on the right arm that facilitates arm movement.", "Where are the coordinates for the right elbow? It's the elbow joint situated in the middle of the right arm.", "Please provide the coordinates for the right elbow, located in the middle of the right arm, where the upper arm connects to the forearm.", "I'd like to see the coordinates of the right elbow, which is positioned in the center of the right arm.", "Where can I locate the coordinates for the right elbow of this person? It's the joint that allows flexion and extension of the right arm.", "Could you output the coordinates for the right elbow, the joint in the middle of the right arm responsible for arm movement?"],
+    'left wrist': ["Please provide the coordinates for the left wrist of the person, which is the joint connecting the left forearm (radius and ulna) and the left hand, typically located at the base of the left hand.", "I'd like to know the precise coordinates of the left wrist, situated at the base of the left hand, where the forearm meets the hand.", "Where can I find the coordinates for the left wrist of this individual? It's the joint responsible for hand movement on the left side.", "Could you please output the coordinates for the left wrist, typically found at the base of the left hand, where the forearm connects to the hand?", "I'm interested in the coordinates of the left wrist. It's the joint on the left side that facilitates movement of the hand and wrist.", "Where are the coordinates for the left wrist? It's the wrist joint situated at the base of the left hand.", "Please provide the coordinates for the left wrist, located at the base of the left hand, where the forearm articulates with the hand.", "I'd like to see the coordinates of the left wrist, which is positioned at the base of the left hand.", "Where can I locate the coordinates for the left wrist of this person? It's the joint that allows movement of the left hand and wrist.", "Could you output the coordinates for the left wrist, the joint at the base of the left hand responsible for hand and wrist movement?"],
+    'right wrist': ["Please provide the coordinates for the right wrist of the person, which is the joint connecting the right forearm (radius and ulna) and the right hand, typically located at the base of the right hand.", "I'd like to know the precise coordinates of the right wrist, situated at the base of the right hand, where the forearm meets the hand.", "Where can I find the coordinates for the right wrist of this individual? It's the joint responsible for hand movement on the right side.", "Could you please output the coordinates for the right wrist, typically found at the base of the right hand, where the forearm connects to the hand?", "I'm interested in the coordinates of the right wrist. It's the joint on the right side that facilitates movement of the hand and wrist.", "Where are the coordinates for the right wrist? It's the wrist joint situated at the base of the right hand.", "Please provide the coordinates for the right wrist, located at the base of the right hand, where the forearm articulates with the hand.", "I'd like to see the coordinates of the right wrist, which is positioned at the base of the right hand.", "Where can I locate the coordinates for the right wrist of this person? It's the joint that allows movement of the right hand and wrist.", "Could you output the coordinates for the right wrist, the joint at the base of the right hand responsible for hand and wrist movement?"],
+    'left hip': ["Please provide the coordinates for the left hip of the person, which is the joint connecting the left thigh to the pelvis, typically located on the left side of the lower torso.", "I'd like to know the precise coordinates of the left hip, situated on the left side of the lower torso, where the thigh bone meets the pelvis.", "Where can I find the coordinates for the left hip of this individual? It's the joint responsible for leg movement on the left side.", "Could you please output the coordinates for the left hip, typically found on the left side of the lower torso, where the thigh articulates with the pelvis?", "I'm interested in the coordinates of the left hip. It's the joint on the left side that facilitates movement of the leg.", "Where are the coordinates for the left hip? It's the hip joint situated on the left side of the lower torso.", "Please provide the coordinates for the left hip, located on the left side of the lower torso, where the thigh connects to the pelvis.", "I'd like to see the coordinates of the left hip, which is positioned on the left side of the lower torso.", "Where can I locate the coordinates for the left hip of this person? It's the joint that allows movement of the left leg.", "Could you output the coordinates for the left hip, the joint on the left side responsible for leg movement?"],
+    'right hip': ["Please provide the coordinates for the right hip of the person, which is the joint connecting the right thigh to the pelvis, typically located on the right side of the lower torso.", "I'd like to know the precise coordinates of the right hip, situated on the right side of the lower torso, where the thigh bone meets the pelvis.", "Where can I find the coordinates for the right hip of this individual? It's the joint responsible for leg movement on the right side.", "Could you please output the coordinates for the right hip, typically found on the right side of the lower torso, where the thigh articulates with the pelvis?", "I'm interested in the coordinates of the right hip. It's the joint on the right side that facilitates movement of the leg.", "Where are the coordinates for the right hip? It's the hip joint situated on the right side of the lower torso.", "Please provide the coordinates for the right hip, located on the right side of the lower torso, where the thigh connects to the pelvis.", "I'd like to see the coordinates of the right hip, which is positioned on the right side of the lower torso.", "Where can I locate the coordinates for the right hip of this person? It's the joint that allows movement of the right leg.", "Could you output the coordinates for the right hip, the joint on the right side responsible for leg movement?"],
+    'left knee': ["Please provide the coordinates for the left knee of the person, which is the joint connecting the left thigh and the left lower leg, typically situated in the middle of the left leg.", "I'd like to know the precise coordinates of the left knee, located in the middle of the left leg, where the upper leg (thigh) meets the lower leg (shin).", "Where can I find the coordinates for the left knee of this individual? It's the joint responsible for bending and straightening the left leg.", "Could you please output the coordinates for the left knee, typically found in the center of the left leg, where the upper and lower leg bones meet?", "I'm interested in the coordinates of the left knee. It's the joint on the left leg that facilitates leg movement.", "Where are the coordinates for the left knee? It's the knee joint situated in the middle of the left leg.", "Please provide the coordinates for the left knee, located in the middle of the left leg, where the thigh connects to the lower leg.", "I'd like to see the coordinates of the left knee, which is positioned in the center of the left leg.", "Where can I locate the coordinates for the left knee of this person? It's the joint that allows movement of the left leg.", "Could you output the coordinates for the left knee, the joint in the middle of the left leg responsible for leg movement?"],
+    'right knee': ["Please provide the coordinates for the right knee of the person, which is the joint connecting the upper leg (thigh) and lower leg (shin) on the right side.", "I'd like to know the precise coordinates of the right knee, situated at the midpoint between the hip and the ankle on the right leg.", "Where can I find the coordinates for the right knee of this individual? It's the pivotal joint on the right leg, roughly in the middle.", "Could you please output the coordinates for the right knee, which is typically located at the front of the right leg, midway between the hip and the ankle?", "I'm interested in the coordinates of the right knee joint. It's the point where the upper and lower parts of the right leg meet.", "Where are the coordinates for the right knee? It's the joint on the right leg that allows bending and is located between the hip and the ankle.", "Please provide the coordinates for the right knee, which is found on the right leg and facilitates movement between the thigh and the shin.", "I'd like to see the coordinates of the right knee, which is positioned on the right leg, about halfway between the hip and the ankle.", "Where can I locate the coordinates for the right knee of this person? It's the joint connecting the upper and lower sections of the right leg.", "Could you output the coordinates for the right knee, the joint on the right leg that enables flexion, situated between the hip and the ankle?"],
+    'left ankle': ["Please provide the coordinates for the left ankle of the person, which is the joint connecting the left lower leg (shin) and the left foot, typically located at the base of the left leg.", "I'd like to know the precise coordinates of the left ankle, situated at the base of the left leg, where the lower leg (shin) meets the foot.", "Where can I find the coordinates for the left ankle of this individual? It's the joint responsible for foot movement on the left side.", "Could you please output the coordinates for the left ankle, typically found at the base of the left leg, where the lower leg (shin) articulates with the foot?", "I'm interested in the coordinates of the left ankle. It's the joint on the left leg that facilitates foot movement.", "Where are the coordinates for the left ankle? It's the ankle joint situated at the base of the left leg.", "Please provide the coordinates for the left ankle, located at the base of the left leg, where the shin connects to the foot.", "I'd like to see the coordinates of the left ankle, which is positioned at the base of the left leg.", "Where can I locate the coordinates for the left ankle of this person? It's the joint that allows movement of the left foot.", "Could you output the coordinates for the left ankle, the joint at the base of the left leg responsible for foot movement?"],
+    'right ankle': ["Please provide the coordinates for the right ankle of the person, which is the joint connecting the right lower leg (shin) and the right foot, typically located at the base of the right leg.", "I'd like to know the precise coordinates of the right ankle, situated at the base of the right leg, where the lower leg (shin) meets the foot.", "Where can I find the coordinates for the right ankle of this individual? It's the joint responsible for foot movement on the right side.", "Could you please output the coordinates for the right ankle, typically found at the base of the right leg, where the lower leg (shin) articulates with the foot?", "I'm interested in the coordinates of the right ankle. It's the joint on the right leg that facilitates foot movement.", "Where are the coordinates for the right ankle? It's the ankle joint situated at the base of the right leg.", "Please provide the coordinates for the right ankle, located at the base of the right leg, where the shin connects to the foot.", "I'd like to see the coordinates of the right ankle, which is positioned at the base of the right leg.", "Where can I locate the coordinates for the right ankle of this person? It's the joint that allows movement of the right foot.", "Could you output the coordinates for the right ankle, the joint at the base of the right leg responsible for foot movement?"],
+    'neck': ["Please provide the coordinates for the neck of the person, which is the part of the body connecting the head to the torso, typically situated between the shoulders.", "I'd like to know the precise coordinates of the neck, located in the area between the head and the upper body, often marked by its curvature.", "Where can I find the coordinates for the neck of this individual? It's the part of the body that supports the head and allows movement.", "Could you please output the coordinates for the neck, typically found in the region connecting the head to the shoulders?", "I'm interested in the coordinates of the neck. It's the vital part of the body that connects the head to the torso.", "Where are the coordinates for the neck? It's the area situated between the head and the upper body.", "Please provide the coordinates for the neck, located between the head and the shoulders, serving as a bridge between the two.", "I'd like to see the coordinates of the neck, which is positioned between the head and the upper body.", "Where can I locate the coordinates for the neck of this person? It's the part that allows the head to move and turn.", "Could you output the coordinates for the neck, the vital junction between the head and the torso?"],
+    'pelvis': ["Please provide the coordinates for the pelvis of the person, which is the bony structure that forms the base of the spine and connects the torso to the lower body, typically located in the lower part of the abdomen and buttocks.", "I'd like to know the precise coordinates of the pelvis, situated in the lower part of the body where the spine meets the hips and legs.", "Where can I find the coordinates for the pelvis of this individual? It's the central structure that supports the upper body and connects to the lower body.", "Could you please output the coordinates for the pelvis, typically found in the region between the abdomen and the hips?", "I'm interested in the coordinates of the pelvis. It's the foundational structure that connects the upper and lower parts of the body.", "Where are the coordinates for the pelvis? It's the bony structure located in the lower abdomen and buttocks.", "Please provide the coordinates for the pelvis, located at the base of the spine and forming the core of the body's support.", "I'd like to see the coordinates of the pelvis, which is positioned in the lower part of the body and connects the torso to the legs.", "Where can I locate the coordinates for the pelvis of this person? It's the structure that helps with balance and stability.", "Could you output the coordinates for the pelvis, the central structure connecting the upper and lower body?"],
+    'left toe': ["Please provide the coordinates for the left toe of the person, which is the digit located at the end of the left foot, typically at the front or tip of the foot.", "I'd like to know the precise coordinates of the left toe, situated at the forward end of the left foot, often marked by the toenail.", "Where can I find the coordinates for the left toe of this individual? It's the digit at the front of the left foot used for balance and movement.", "Could you please output the coordinates for the left toe, typically found at the tip of the left foot where the toes begin?", "I'm interested in the coordinates of the left toe. It's the digit on the left foot's front end that assists in walking and balance.", "Where are the coordinates for the left toe? It's the toe located at the end of the left foot, near the toenail.", "Please provide the coordinates for the left toe, situated at the front end of the left foot, often marked by the toenail.", "I'd like to see the coordinates of the left toe, which is positioned at the tip of the left foot.", "Where can I locate the coordinates for the left toe of this person? It's the digit responsible for touch and balance on the left foot.", "Could you output the coordinates for the left toe, the digit at the front end of the left foot that aids in walking and balance?"],
+    'right toe': ["Please provide the coordinates for the right toe of the person, which is the digit located at the end of the right foot, typically at the front or tip of the foot.", "I'd like to know the precise coordinates of the right toe, situated at the forward end of the right foot, often marked by the toenail.", "Where can I find the coordinates for the right toe of this individual? It's the digit at the front of the right foot used for balance and movement.", "Could you please output the coordinates for the right toe, typically found at the tip of the right foot where the toes begin?", "I'm interested in the coordinates of the right toe. It's the digit on the right foot's front end that assists in walking and balance.", "Where are the coordinates for the right toe? It's the toe located at the end of the right foot, near the toenail.", "Please provide the coordinates for the right toe, situated at the front end of the right foot, often marked by the toenail.", "I'd like to see the coordinates of the right toe, which is positioned at the tip of the right foot.", "Where can I locate the coordinates for the right toe of this person? It's the digit responsible for touch and balance on the right foot.", "Could you output the coordinates for the right toe, the digit at the front end of the right foot that aids in walking and balance?"],
+    'head_top': ["Please provide the coordinates for the head top of the person, which is the highest point on their head, typically at the crown.", "I'd like to know the precise coordinates of the head top, situated at the very top of the person's head.", "Where can I find the coordinates for the head top of this individual? It's the highest point on the head, often marked by the skull's apex.", "Could you please output the coordinates for the head top, typically located at the crown of the head?", "I'm interested in the coordinates of the head top. It's the highest point on the person's head.", "Where are the coordinates for the head top? It's the topmost point on the head, usually at the crown.", "Please provide the coordinates for the head top, located at the very top of the head, often marked by the hairline.", "I'd like to see the coordinates of the head top, which is positioned at the highest point on the head.", "Where can I locate the coordinates for the head top of this person? It's the topmost point of their skull.", "Could you output the coordinates for the head top, the highest point on the person's head?"],
+    'thorax': ["Please provide the coordinates for the thorax of the person, which is the central part of the torso, typically located between the neck and the abdomen.", "I'd like to know the precise coordinates of the thorax, situated in the middle of the upper body, often marked by the ribcage.", "Where can I find the coordinates for the thorax of this individual? It's the central region of the torso that houses the heart and lungs.", "Could you please output the coordinates for the thorax, typically found in the middle of the upper body, where the ribcage surrounds vital organs?", "I'm interested in the coordinates of the thorax. It's the central part of the torso responsible for protecting vital organs.", "Where are the coordinates for the thorax? It's the midsection of the upper body, often covered by the chest.", "Please provide the coordinates for the thorax, located between the neck and the abdomen, serving as a vital core of the body.", "I'd like to see the coordinates of the thorax, which is positioned in the middle of the upper body.", "Where can I locate the coordinates for the thorax of this person? It's the region that houses the heart and lungs.", "Could you output the coordinates for the thorax, the central part of the torso responsible for protecting vital organs?"],
+    'torso': ["Please provide the coordinates for the torso of the person, which is the central part of the body, typically encompassing the chest, abdomen, and back.", "I'd like to know the precise coordinates of the torso, situated in the middle of the body and extending from the chest to the waist.", "Where can I find the coordinates for the torso of this individual? It's the core region that houses many vital organs.", "Could you please output the coordinates for the torso, typically found in the central area of the body, covering the chest and abdomen?", "I'm interested in the coordinates of the torso. It's the main part of the body that includes the chest and abdominal regions.", "Where are the coordinates for the torso? It's the central core of the body that plays a crucial role in protecting vital organs.", "Please provide the coordinates for the torso, located in the middle of the body, between the chest and the waist.", "I'd like to see the coordinates of the torso, which is positioned at the core of the body and includes the chest and abdomen.", "Where can I locate the coordinates for the torso of this person? It's the central region responsible for housing and protecting vital organs.", "Could you output the coordinates for the torso, the central part of the body encompassing the chest, abdomen, and back?"]
+}

data/convsersation.py

@@ -0,0 +1,105 @@
+import os
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+from collections import defaultdict
+from .constants import PART_ORDER, COCO_KEYPOINT_NAME
+
+def read_hoi_file_2_dict(hoi_config):
+    hoi_dict = {}
+    with open(hoi_config, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            nums, obj, action = line.split()  # split on whitespace
+            hoi_dict[int(nums)] = [obj, action]      # use nums as int; remove int() if you want string keys
+    return hoi_dict
+
+def read_part_state_file_2_dict(part_state_config):
+    d = defaultdict(list)
+    with open(part_state_config, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            key, val = line.split(":", 1)   # split only on first ":"
+            key = key.strip()
+            val = val.strip()
+            d[key].append(val)
+    return d
+
+@dataclasses.dataclass
+class Conversation:
+    def __init__(self, system='', data_path=''):
+        super().__init__()       
+        if system == '':
+           self.system = f"""
+           You are an AI assistant. You will be given an image that contains a main human subject.
+           Task:
+           Describe the visual evidence in the image that supports the subject’s action, with an emphasis on human body parts and their interactions with objects.
+
+           Hints:
+           You may be given hints about (1) the action and (2) related objects and possible supporting body parts. You can use these hints, but you may also add other relevant evidence you observe.
+
+           Required Constraints:
+           - Start with ONE sentence that summarizes the main action in natural language.
+           - When you mention any keypoint or body part, you MUST use names ONLY from: {COCO_KEYPOINT_NAME}.
+           - Do NOT invent body-part names outside these sets (no synonyms, no paraphrases).
+           - If you are unsure which name applies, either omit the body-part mention or choose the closest valid name from the lists.
+           - Write your description in clear, concise sentences grounded in visible evidence.
+
+           Optional Constraints :
+           - Write naturally. Avoid repeating the same sentence pattern.
+           - Keep each evidence item to one line. No redundant "both left/right do the same" unless necessary.
+           """
+        else:
+            self.system = system
+         
+        self.hoi_reference = read_hoi_file_2_dict(os.path.join(data_path, 'Configs/hico_hoi_list.txt'))
+        self.part_state_reference = read_part_state_file_2_dict(os.path.join(data_path, 'Configs/Part_State_76.txt'))
+
+    def _humanpart2word(self, action_labels):
+        action_labels_in_words = []
+        part_state_keys = list(self.part_state_reference.keys())
+        for d in action_labels:
+            human_part_id = d['human_part']
+            part_state_id = d['partstate']
+
+            part_name = PART_ORDER[human_part_id]
+            for key in part_state_keys:
+                if key in part_name:
+                    states = self.part_state_reference[key]
+                    part_state = states[part_state_id]
+            action_labels_in_words.append([part_name, part_state])
+        return action_labels_in_words
+
+    def _actionid2word(self, hoi_id):
+        obj, act = self.hoi_reference[hoi_id]
+        return obj, act
+
+    def get_prompt(self, meta):
+        hoi_obj = meta['hoi_obj']
+        
+        hoi_id = hoi_obj['hoi_id']
+        obj_in_word, act_in_word = self._actionid2word(hoi_id)
+        action_labels = hoi_obj['action_labels']
+        action_labels_in_words = self._humanpart2word(action_labels)
+
+        prompt = f"""
+               Given the image, describe the visual evidence (especially body parts) that supports the action.
+               Hints: The action to support is [{act_in_word} with {obj_in_word}]. Possible visual evidence cues include: {action_labels_in_words}.
+               Use these cues as guidance. Only mention cues you can actually see in the image.
+        """
+        return prompt
+
+
+
+
+
+
+    
+
+if __name__ == "__main__":
+   pass

data/hicodet.py

@@ -0,0 +1,294 @@
+"""
+HICODet dataset under PyTorch framework
+
+Fred Zhang <frederic.zhang@anu.edu.au>
+
+The Australian National University
+Australian Centre for Robotic Vision
+"""
+
+import os
+import json
+import numpy as np
+
+from typing import Optional, List, Callable, Tuple
+from pocket.data import ImageDataset, DataSubset
+
+class HICODetSubset(DataSubset):
+    def __init__(self, *args) -> None:
+        super().__init__(*args)
+    def filename(self, idx: int) -> str:
+        """Override: return the image file name in the subset"""
+        return self._filenames[self._idx[self.pool[idx]]]
+    def image_size(self, idx: int) -> Tuple[int, int]:
+        """Override: return the size (width, height) of an image in the subset"""
+        return self._image_sizes[self._idx[self.pool[idx]]]
+    @property
+    def anno_interaction(self) -> List[int]:
+        """Override: Number of annotated box pairs for each interaction class"""
+        num_anno = [0 for _ in range(self.num_interation_cls)]
+        intra_idx = [self._idx[i] for i in self.pool]
+        for idx in intra_idx:
+            for hoi in self._anno[idx]['hoi']:
+                num_anno[hoi] += 1
+        return num_anno
+    @property
+    def anno_object(self) -> List[int]:
+        """Override: Number of annotated box pairs for each object class"""
+        num_anno = [0 for _ in range(self.num_object_cls)]
+        anno_interaction = self.anno_interaction
+        for corr in self._class_corr:
+            num_anno[corr[1]] += anno_interaction[corr[0]]
+        return num_anno
+    @property
+    def anno_action(self) -> List[int]:
+        """Override: Number of annotated box pairs for each action class"""
+        num_anno = [0 for _ in range(self.num_action_cls)]
+        anno_interaction = self.anno_interaction
+        for corr in self._class_corr:
+            num_anno[corr[2]] += anno_interaction[corr[0]]
+        return num_anno
+
+class HICODet(ImageDataset):
+    """
+    Arguments:
+        root(str): Root directory where images are downloaded to
+        anno_file(str): Path to json annotation file
+        transform(callable, optional): A function/transform that  takes in an PIL image
+            and returns a transformed version
+        target_transform(callable, optional): A function/transform that takes in the
+            target and transforms it
+        transforms (callable, optional): A function/transform that takes input sample 
+            and its target as entry and returns a transformed version.
+    """
+    def __init__(self, root: str, anno_file: str,
+            transform: Optional[Callable] = None,
+            target_transform: Optional[Callable] = None,
+            transforms: Optional[Callable] = None) -> None:
+        super(HICODet, self).__init__(root, transform, target_transform, transforms)
+        with open(anno_file, 'r') as f:
+            anno = json.load(f)
+        
+        import pdb;pdb.set_trace()
+        self.num_object_cls = 80
+        self.num_interation_cls = 600
+        self.num_action_cls = 117
+        self._anno_file = anno_file
+
+        # Load annotations
+        self._load_annotation_and_metadata(anno)
+
+    def __len__(self) -> int:
+        """Return the number of images"""
+        return len(self._idx)
+
+    def __getitem__(self, i: int) -> tuple:
+        """
+        Arguments:
+            i(int): Index to an image
+        
+        Returns:
+            tuple[image, target]: By default, the tuple consists of a PIL image and a
+                dict with the following keys:
+                    "boxes_h": list[list[4]]
+                    "boxes_o": list[list[4]]
+                    "hoi":: list[N]
+                    "verb": list[N]
+                    "object": list[N]
+        """
+        intra_idx = self._idx[i]
+        return self._transforms(
+            self.load_image(os.path.join(self._root, self._filenames[intra_idx])), 
+            self._anno[intra_idx]
+            )
+
+    def __repr__(self) -> str:
+        """Return the executable string representation"""
+        reprstr = self.__class__.__name__ + '(root=' + repr(self._root)
+        reprstr += ', anno_file='
+        reprstr += repr(self._anno_file)
+        reprstr += ')'
+        # Ignore the optional arguments
+        return reprstr
+
+    def __str__(self) -> str:
+        """Return the readable string representation"""
+        reprstr = 'Dataset: ' + self.__class__.__name__ + '\n'
+        reprstr += '\tNumber of images: {}\n'.format(self.__len__())
+        reprstr += '\tImage directory: {}\n'.format(self._root)
+        reprstr += '\tAnnotation file: {}\n'.format(self._root)
+        return reprstr
+
+    @property
+    def annotations(self) -> List[dict]:
+        return self._anno
+
+    @property
+    def class_corr(self) -> List[Tuple[int, int, int]]:
+        """
+        Class correspondence matrix in zero-based index
+        [
+            [hoi_idx, obj_idx, verb_idx],
+            ...
+        ]
+
+        Returns:
+            list[list[3]]
+        """
+        return self._class_corr.copy()
+
+    @property
+    def object_n_verb_to_interaction(self) -> List[list]:
+        """
+        The interaction classes corresponding to an object-verb pair
+
+        HICODet.object_n_verb_to_interaction[obj_idx][verb_idx] gives interaction class
+        index if the pair is valid, None otherwise
+
+        Returns:
+            list[list[117]]
+        """
+        lut = np.full([self.num_object_cls, self.num_action_cls], None)
+        for i, j, k in self._class_corr:
+            lut[j, k] = i
+        return lut.tolist()
+
+    @property
+    def object_to_interaction(self) -> List[list]:
+        """
+        The interaction classes that involve each object type
+        
+        Returns:
+            list[list]
+        """
+        obj_to_int = [[] for _ in range(self.num_object_cls)]
+        for corr in self._class_corr:
+            obj_to_int[corr[1]].append(corr[0])
+        return obj_to_int
+
+    @property
+    def object_to_verb(self) -> List[list]:
+        """
+        The valid verbs for each object type
+
+        Returns:
+            list[list]
+        """
+        obj_to_verb = [[] for _ in range(self.num_object_cls)]
+        for corr in self._class_corr:
+            obj_to_verb[corr[1]].append(corr[2])
+        return obj_to_verb
+
+    @property
+    def anno_interaction(self) -> List[int]:
+        """
+        Number of annotated box pairs for each interaction class
+
+        Returns:
+            list[600]
+        """
+        return self._num_anno.copy()
+
+    @property
+    def anno_object(self) -> List[int]:
+        """
+        Number of annotated box pairs for each object class
+
+        Returns:
+            list[80]
+        """
+        num_anno = [0 for _ in range(self.num_object_cls)]
+        for corr in self._class_corr:
+            num_anno[corr[1]] += self._num_anno[corr[0]]
+        return num_anno
+
+    @property
+    def anno_action(self) -> List[int]:
+        """
+        Number of annotated box pairs for each action class
+
+        Returns:
+            list[117]
+        """
+        num_anno = [0 for _ in range(self.num_action_cls)]
+        for corr in self._class_corr:
+            num_anno[corr[2]] += self._num_anno[corr[0]]
+        return num_anno
+
+    @property
+    def objects(self) -> List[str]:
+        """
+        Object names 
+
+        Returns:
+            list[str]
+        """
+        return self._objects.copy()
+
+    @property
+    def verbs(self) -> List[str]:
+        """
+        Verb (action) names
+
+        Returns:
+            list[str]
+        """
+        return self._verbs.copy()
+
+    @property
+    def interactions(self) -> List[str]:
+        """
+        Combination of verbs and objects
+
+        Returns:
+            list[str]
+        """
+        return [self._verbs[j] + ' ' + self.objects[i] 
+            for _, i, j in self._class_corr]
+
+    def split(self, ratio: float) -> Tuple[HICODetSubset, HICODetSubset]:
+        """
+        Split the dataset according to given ratio
+
+        Arguments:
+            ratio(float): The percentage of training set between 0 and 1
+        Returns:
+            train(Dataset)
+            val(Dataset)
+        """
+        perm = np.random.permutation(len(self._idx))
+        n = int(len(perm) * ratio)
+        return HICODetSubset(self, perm[:n]), HICODetSubset(self, perm[n:])
+
+    def filename(self, idx: int) -> str:
+        """Return the image file name given the index"""
+        return self._filenames[self._idx[idx]]
+
+    def image_size(self, idx: int) -> Tuple[int, int]:
+        """Return the size (width, height) of an image"""
+        return self._image_sizes[self._idx[idx]]
+
+    def _load_annotation_and_metadata(self, f: dict) -> None:
+        """
+        Arguments:
+            f(dict): Dictionary loaded from {anno_file}.json
+        """
+        idx = list(range(len(f['filenames'])))
+        for empty_idx in f['empty']:
+            idx.remove(empty_idx)
+
+        num_anno = [0 for _ in range(self.num_interation_cls)]
+        for anno in f['annotation']:
+            for hoi in anno['hoi']:
+                num_anno[hoi] += 1
+
+        self._idx = idx
+        self._num_anno = num_anno
+
+        self._anno = f['annotation']
+        self._filenames = f['filenames']
+        self._image_sizes = f['size']
+        self._class_corr = f['correspondence']
+        self._empty_idx = f['empty']
+        self._objects = f['objects']
+        self._verbs = f['verbs']

data/pose_hicodet.py

@@ -0,0 +1,335 @@
+import os
+import cv2
+import json
+import logging
+import random
+from typing import Dict
+
+import torch
+from torch.utils.data import Dataset
+from torchvision import transforms
+import numpy as np
+
+import transformers
+from pycocotools.coco import COCO
+
+from .constants import COCO_KEYPOINT_NAME, KeypointLocationDescription, KeypointLocationQuestion
+from .constants import COCO_KEYPOINT_NAME_TOKEN
+
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+PREFIX_IMAGE = "Image: "
+PREFIX_NO_IMAGE = "Image: N/A"
+BEGIN_DESCRIPTION = "<des>"
+END_DESCRIPTION = "</des>"
+IGNORE_INDEX = -100
+DEFAULT_EOS_TOKEN = "</s>"
+BEGIN_OPTIONS = "<opt>"
+END_OPTIONS = "</opt>"
+BEGIN_LOC = "<loc>"
+END_LOC = "</loc>"
+BEGIN_QUESTION = "<qes>"
+END_QUESTION = "</qes>"
+
+class PoseHICODetDataset(Dataset):
+    """Dataset for supervised fine-tuning."""
+    def __init__(self, data_path: str,
+                 multimodal_cfg: dict,
+                 ):
+        super(PoseHICODetDataset, self).__init__()
+        logging.warning("Loading data...")
+        self.multimodal_cfg = multimodal_cfg
+        self.mllm_image_size = multimodal_cfg['image_size']
+        self.aspect_ratio = 1.0
+        self.pixel_std = 200
+        self.num_joints = 17
+        self.num_joints_full_body = 136
+        self.list_data_dict = self._load_data(data_path)
+       
+    
+    def _iou(self, a, b):
+        x1, y1, x2, y2 = a; X1, Y1, X2, Y2 = b
+        iw = max(0, min(x2, X2) - max(x1, X1))
+        ih = max(0, min(y2, Y2) - max(y1, Y1))
+        inter = iw * ih
+        return inter / ((x2 - x1) * (y2 - y1) + (X2 - X1) * (Y2 - Y1) - inter + 1e-9)
+
+    def _match_pose_hoi_objs(self, pose_objs, hoi_objs):
+        matched_pose_objs = []
+        matched_hoi_objs = []
+        
+        for pose_obj in pose_objs:
+            for hoi_obj in hoi_objs:
+                X1, Y1, W, H = pose_obj['bbox']
+                iou = self._iou(hoi_obj['human_bbox'], [X1, Y1, X1+W, Y1+H])
+                if iou < 0.9: continue
+                if 'action_labels' not in list(hoi_obj.keys()): 
+                    continue
+                      
+                matched_pose_objs.append(pose_obj)
+                matched_hoi_objs.append(hoi_obj)
+
+        return matched_pose_objs, matched_hoi_objs
+
+    def _load_data(self, data_path):
+ 
+        # load pose annotation via coco api
+        coco_path = os.path.join(data_path, 'Annotation/hico-fullbody-pose/halpe_train_v1.json')
+        coco = COCO(coco_path)
+
+        # load instance-level hoi+part state annotation via json
+        json_path = os.path.join(data_path, "Annotation/hico-det-instance-level/hico-det-training-set-instance-level.json")
+        with open(json_path, "r", encoding="utf-8") as f:
+             hoi_data = json.load(f)   # dict (or list) depending on the JSON root
+
+        instance_id = 0
+        list_data_dict=[]
+        for index in coco.getImgIds():
+            #load pose data per image id
+            im_ann = coco.loadImgs(index)[0]
+            width = im_ann['width']
+            height = im_ann['height']
+            annIds = coco.getAnnIds(imgIds=index, iscrowd=False)
+            pose_objs = coco.loadAnns(annIds)
+            
+            #load hoi data per image id
+            file_name = im_ann['file_name']
+            hoi_objs = hoi_data[file_name]['labels']
+
+            pose_objs, hoi_objs = self._match_pose_hoi_objs(pose_objs, hoi_objs)
+        
+            for (pose_obj, hoi_obj) in zip(pose_objs, hoi_objs):
+                cls = pose_obj['category_id']
+                if cls != 1: continue
+
+                # ignore objs without keypoints annotation
+                if max(pose_obj['keypoints']) == 0:
+                    continue
+
+                assert 'action_labels' in list(hoi_obj.keys())
+
+                joints_3d = np.zeros((self.num_joints_full_body, 3), dtype=np.float32)
+                joints_3d_vis = np.zeros((self.num_joints_full_body, 3), dtype=np.float32)
+                visible = np.zeros((self.num_joints_full_body), dtype=np.float32)
+                for ipt in range(self.num_joints_full_body):
+                    joints_3d[ipt, 0] = pose_obj['keypoints'][ipt * 3 + 0]
+                    joints_3d[ipt, 1] = pose_obj['keypoints'][ipt * 3 + 1]
+                    joints_3d[ipt, 2] = 0
+                    t_vis = pose_obj['keypoints'][ipt * 3 + 2]
+                    visible[ipt] = t_vis
+                    if t_vis > 1:
+                        t_vis = 1
+                    joints_3d_vis[ipt, 0] = t_vis
+                    joints_3d_vis[ipt, 1] = t_vis
+                    joints_3d_vis[ipt, 2] = 0
+
+                center, scale = self._box2cs(pose_obj['bbox'][:4])
+                list_data_dict.append({
+                    'file_name': file_name,
+                    'image_id': index,
+                    'center': center,
+                    'scale': scale,
+                    'joints_3d': joints_3d[:self.num_joints], # the first 17 keypoints are aligned with COCO's 17 keypoints definition.
+                    'joints_3d_vis': joints_3d_vis[:self.num_joints],
+                    'instance_id': instance_id,
+                    'hoi_obj': hoi_obj,
+                })
+                instance_id += 1
+        
+        logging.warning("The number of training samples is {}".format(len(list_data_dict)))
+        logging.warning("Formatting inputs...Skip in lazy mode")
+        return list_data_dict
+
+    def __len__(self):
+        return len(self.list_data_dict)
+
+    def __getitem__(self, i):
+        sources = self.list_data_dict[i]
+        image, joints, joints_vis, c, s = self._get_image_item(sources)
+      
+        data_dict = {}
+        data_dict["image"] = image
+        data_dict["has_image"] = True
+        data_dict["meta"] = sources
+        return data_dict
+    
+    def _get_image_item(self, sources):
+        file_name = sources['file_name']
+        image_folder = self.multimodal_cfg['image_folder']
+        image_file = os.path.join(image_folder, file_name)
+        image = cv2.imread(
+            image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION
+        )
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        
+        # process image
+        joints = sources['joints_3d']
+        joints_vis = sources['joints_3d_vis']
+        c = sources['center']
+        s = sources['scale']
+        r = 0
+
+        trans = get_affine_transform(c, s, r, (int(self.mllm_image_size), int(self.mllm_image_size)))
+        image = cv2.warpAffine(
+            image,
+            trans,
+            (int(self.mllm_image_size), int(self.mllm_image_size)),
+            flags=cv2.INTER_LINEAR)
+        
+        # for i in range(self.num_joints):
+        #     if joints_vis[i, 0] > 0.0:
+        #         joints[i, 0:2] = affine_transform(joints[i, 0:2], trans)
+        
+        return image, joints, joints_vis, c, s
+    
+    def _box2cs(self, box):
+        x, y, w, h = box[:4]
+        return self._xywh2cs(x, y, w, h)
+
+    def _xywh2cs(self, x, y, w, h):
+        center = np.zeros((2), dtype=np.float32)
+        center[0] = x + w * 0.5
+        center[1] = y + h * 0.5
+
+        if w > self.aspect_ratio * h:
+            h = w * 1.0 / self.aspect_ratio
+        elif w < self.aspect_ratio * h:
+            w = h * self.aspect_ratio
+        scale = np.array(
+            [w * 1.0 / self.pixel_std, h * 1.0 / self.pixel_std],
+            dtype=np.float32)
+        if center[0] != -1:
+            # scale = scale * 1.25
+            scale = scale * 1.0
+
+        return center, scale
+    
+    def _generate_target(self, joints, joints_vis):
+        '''
+        :param joints:  [num_joints, 3]
+        :param joints_vis: [num_joints, 3]
+        :return: target, target_weight(1: visible, 0: invisible)
+        '''
+        target_weight = np.ones((self.num_joints, 1), dtype=np.float32)
+        target_weight[:, 0] = joints_vis[:, 0]
+        target = np.zeros((self.num_joints,
+                               self.heatmap_size[1],
+                               self.heatmap_size[0]),
+                              dtype=np.float32)
+
+        tmp_size = self.sigma * 3
+
+        for joint_id in range(self.num_joints):
+            feat_stride = self.vitpose_image_size / self.heatmap_size
+            mu_x = int(joints[joint_id][0] / feat_stride[0] + 0.5)
+            mu_y = int(joints[joint_id][1] / feat_stride[1] + 0.5)
+            # Check that any part of the gaussian is in-bounds
+            ul = [int(mu_x - tmp_size), int(mu_y - tmp_size)]
+            br = [int(mu_x + tmp_size + 1), int(mu_y + tmp_size + 1)]
+            if ul[0] >= self.heatmap_size[0] or ul[1] >= self.heatmap_size[1] \
+                        or br[0] < 0 or br[1] < 0:
+                    # If not, just return the image as is
+                target_weight[joint_id] = 0
+                continue
+
+            # # Generate gaussian
+            size = 2 * tmp_size + 1
+            x = np.arange(0, size, 1, np.float32)
+            y = x[:, np.newaxis]
+            x0 = y0 = size // 2
+            # The gaussian is not normalized, we want the center value to equal 1
+            g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * self.sigma ** 2))
+
+            # Usable gaussian range
+            g_x = max(0, -ul[0]), min(br[0], self.heatmap_size[0]) - ul[0]
+            g_y = max(0, -ul[1]), min(br[1], self.heatmap_size[1]) - ul[1]
+            # Image range
+            img_x = max(0, ul[0]), min(br[0], self.heatmap_size[0])
+            img_y = max(0, ul[1]), min(br[1], self.heatmap_size[1])
+
+            v = target_weight[joint_id]
+            if v > 0.5:
+                target[joint_id][img_y[0]:img_y[1], img_x[0]:img_x[1]] = \
+                        g[g_y[0]:g_y[1], g_x[0]:g_x[1]]
+
+        # if self.use_different_joints_weight:
+        #     target_weight = np.multiply(target_weight, self.joints_weight)
+
+        return target, target_weight
+
+def fliplr_joints(joints, joints_vis, width, matched_parts):
+    """
+    flip coords
+    """
+    # Flip horizontal
+    joints[:, 0] = width - joints[:, 0] - 1
+
+    # Change left-right parts
+    for pair in matched_parts:
+        joints[pair[0], :], joints[pair[1], :] = \
+            joints[pair[1], :], joints[pair[0], :].copy()
+        joints_vis[pair[0], :], joints_vis[pair[1], :] = \
+            joints_vis[pair[1], :], joints_vis[pair[0], :].copy()
+
+    return joints*joints_vis, joints_vis
+
+def transform_preds(coords, center, scale, output_size):
+    target_coords = np.zeros(coords.shape)
+    trans = get_affine_transform(center, scale, 0, output_size, inv=1)
+    for p in range(coords.shape[0]):
+        target_coords[p, 0:2] = affine_transform(coords[p, 0:2], trans)
+    return target_coords
+
+def get_affine_transform(
+        center, scale, rot, output_size,
+        shift=np.array([0, 0], dtype=np.float32), inv=0
+):
+    if not isinstance(scale, np.ndarray) and not isinstance(scale, list):
+        print(scale)
+        scale = np.array([scale, scale])
+
+    scale_tmp = scale * 200.0
+    src_w = scale_tmp[0]
+    dst_w = output_size[0]
+    dst_h = output_size[1]
+
+    rot_rad = np.pi * rot / 180
+    src_dir = get_dir([0, src_w * -0.5], rot_rad)
+    dst_dir = np.array([0, dst_w * -0.5], np.float32)
+
+    src = np.zeros((3, 2), dtype=np.float32)
+    dst = np.zeros((3, 2), dtype=np.float32)
+    src[0, :] = center + scale_tmp * shift
+    src[1, :] = center + src_dir + scale_tmp * shift
+    dst[0, :] = [dst_w * 0.5, dst_h * 0.5]
+    dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir
+
+    src[2:, :] = get_3rd_point(src[0, :], src[1, :])
+    dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :])
+
+    if inv:
+        trans = cv2.getAffineTransform(np.float32(dst), np.float32(src))
+    else:
+        trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))
+
+    return trans
+
+
+def affine_transform(pt, t):
+    new_pt = np.array([pt[0], pt[1], 1.]).T
+    new_pt = np.dot(t, new_pt)
+    return new_pt[:2]
+
+
+def get_3rd_point(a, b):
+    direct = a - b
+    return b + np.array([-direct[1], direct[0]], dtype=np.float32)
+
+
+def get_dir(src_point, rot_rad):
+    sn, cs = np.sin(rot_rad), np.cos(rot_rad)
+
+    src_result = [0, 0]
+    src_result[0] = src_point[0] * cs - src_point[1] * sn
+    src_result[1] = src_point[0] * sn + src_point[1] * cs
+
+    return src_result

scripts/annotate.sh

@@ -0,0 +1,18 @@
+IDX=0,4,
+export PYTHONPATH=$PYTHONPATH:./
+
+data_path=../datasets/HICO-Det
+model_path=./model_weights/qwen3_8b_vl_instruct
+output_dir=outputs
+
+if [ -d ${output_dir} ];then
+    echo "dir already exists"
+else
+    mkdir ${output_dir}
+fi
+
+CUDA_VISIBLE_DEVICES=$IDX OMP_NUM_THREADS=1 torchrun --nnodes=1 --nproc_per_node=2 --master_port=25005 \
+    tools/annotate.py \
+    --model-path ${model_path} \
+    --data-path ${data_path} \
+    --output-dir ${output_dir} \

tools/annotate.py

@@ -0,0 +1,186 @@
+import os
+import json
+import argparse
+import numpy as np
+from tqdm import tqdm
+
+import torch
+import torch.distributed as dist
+from torch.utils.data import DataLoader
+from torchvision import transforms as T
+
+from data.pose_hicodet import PoseHICODetDataset
+from data.convsersation import Conversation
+
+import re
+from dataclasses import dataclass
+
+from transformers import Qwen3VLForConditionalGeneration
+from transformers import AutoTokenizer, AutoConfig, AutoProcessor
+
+def disable_torch_init():
+    """
+    Disable the redundant torch default initialization to accelerate model creation.
+    """
+    setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+    setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+import os, json
+import torch
+import torch.distributed as dist
+
+def gather_labels_and_save(labels, output_path):
+    # Make sure dist is initialized (torchrun / deepspeed / accelerate usually does this)
+    world_size = dist.get_world_size()
+    rank = dist.get_rank()
+
+    gathered = [None for _ in range(world_size)]
+    dist.all_gather_object(gathered, labels)  # gathered[i] is labels from rank i
+
+    if rank == 0:
+        merged = []
+        for part in gathered:
+            merged.extend(part)
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(merged, f, ensure_ascii=False, indent=2)
+
+    dist.barrier()  # optional: ensure rank0 finished writing before others exit
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+    def __init__(self, processor, data_path):
+        self.processor = processor
+        self.conv = Conversation(
+            system='',
+            data_path=data_path
+        )
+    
+    def __call__(self, data_dicts):
+        """Collate examples for supervised fine-tuning."""
+        batch_prompts = []
+        batch_images = []
+        result_meta = []
+    
+        for i, data_dict in enumerate(data_dicts):
+            batch_images.append(data_dict['image'])
+            batch_prompts.append(self.conv.get_prompt(data_dict['meta']))
+            result_meta.append(data_dict['meta'])
+
+        messages = []
+        for prompt in zip(batch_prompts):
+            messages.append([
+                {"role": "system",
+                 "content":[
+                    {"type": "text",
+                     "text": self.conv.system},]},
+                {"role": "user",
+                 "content":[
+                    {"type": "image"},
+                    {"type": "text",
+                     "text": prompt},]},
+                ])
+
+        prompts = [self.processor.apply_chat_template(m, 
+                                                 tokenize=False,
+                                                 add_generation_prompt=True)
+                   for m in messages]
+        batch_tensors = self.processor(
+            text=prompts,
+            images=batch_images,
+            return_tensors="pt",
+            padding=True
+        )
+        return batch_tensors, result_meta
+
+@torch.no_grad()
+def worker(model, processor, dataset, args, output_dir):
+
+    rank = int(os.environ["LOCAL_RANK"])
+    world_size = int(os.environ["WORLD_SIZE"])
+    indices = list(range(rank, len(dataset), world_size))
+    print("==>" + " Worker {} Started, responsible for {} images".format(rank, len(indices)))
+
+    sub_dataset = torch.utils.data.Subset(dataset, indices)
+    batch_size = 1
+    data_loader = DataLoader(sub_dataset, batch_size=batch_size, shuffle=False, num_workers=0, collate_fn=DataCollatorForSupervisedDataset(processor, args.data_path))
+    labels = []
+    for batch_tensors, result_meta in tqdm(data_loader):
+       
+        input_ids = batch_tensors['input_ids'].cuda()
+        batch_tensors = {k: v.cuda() for k, v in batch_tensors.items() if isinstance(v, torch.Tensor)}
+        with torch.inference_mode():
+            output_dict = model.generate(do_sample=False,
+                                         output_scores=True,
+                                         return_dict_in_generate=True,
+                                         max_new_tokens=1600,
+                                         output_logits=True,
+                                         **batch_tensors,)
+           
+            output_ids = output_dict['sequences']
+        
+        for input_id, output_id, meta in zip(input_ids, output_ids, result_meta):
+            input_token_len = input_id.shape[0]
+            n_diff_input_output = (input_id != output_id[:input_token_len]).sum().item()
+            if n_diff_input_output > 0:
+                print(f'[Warning] Sample: {n_diff_input_output} output_ids are not the same as the input_ids')
+            output = processor.tokenizer.batch_decode(output_id[input_token_len:].unsqueeze(0), skip_special_tokens=True)[0]
+    
+            labels.append({
+                'file_name': meta['file_name'],
+                'image_id': meta['image_id'],
+                'instance_id': meta['instance_id'],
+                'keypoints': meta['joints_3d'].reshape(-1).tolist(),
+                'vis': meta['joints_3d_vis'].reshape(-1).tolist(),
+                'im_height': meta['hoi_obj']['height'],
+                'im_width': meta['hoi_obj']['width'],
+                'human_bbox': meta['hoi_obj']['human_bbox'],
+                'object_bbox': meta['hoi_obj']['object_bbox'],
+                'action_labels': meta['hoi_obj']['action_labels'],
+                'description': output,
+            })
+
+        break
+    
+    output_path = os.path.join(args.output_dir, 'labels.json')
+    gather_labels_and_save(labels, output_path=output_path)
+
+def eval_model(args):
+    torch.distributed.init_process_group(backend='nccl')
+    rank = int(os.environ["LOCAL_RANK"])
+    world_size = int(os.environ["WORLD_SIZE"])
+    
+    print('Init process group: world_size: {}, rank: {}'.format(world_size, rank))
+    torch.cuda.set_device(rank)
+
+    disable_torch_init()
+    model = Qwen3VLForConditionalGeneration.from_pretrained(
+        args.model_path, 
+        torch_dtype=torch.bfloat16,          
+        trust_remote_code=True               
+    )
+    model = model.cuda()
+    model.eval()
+       
+    processor = AutoProcessor.from_pretrained(
+            args.model_path,
+            trust_remote_code=True)
+    processor.tokenizer.padding_side = "left"
+    processor.tokenizer.pad_token = processor.tokenizer.eos_token
+  
+    dataset = PoseHICODetDataset(
+                data_path=args.data_path,
+                multimodal_cfg=dict(image_folder=os.path.join(args.data_path, 'Images/images/train2015'),
+                        data_augmentation=False,
+                        image_size=336,),)
+    worker(model, processor, dataset, args, args.output_dir)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
+    parser.add_argument("--data-path", type=str, default="")
+    parser.add_argument("--output-dir", type=str, default="")
+    args = parser.parse_args()
+
+    eval_model(args)
+