# SPDX-License-Identifier: MIT # Copyright (C) Intel Corporation # # Person Re-Identification: DLStreamer pipeline sample. # See README.md for prerequisites and inputs. Run: python3 dlstreamer_sample.py import gi gi.require_version("Gst", "1.0") from gi.repository import Gst Gst.init([]) import numpy as np import cv2 from gstgva import VideoFrame INPUT_VIDEO = "test_video.mp4" REFERENCE_IMAGE = "person_a.jpg" OUTPUT_VIDEO = "output_dlstreamer.mp4" DETECTION_MODEL = "intel/person-detection-retail-0013/FP16/person-detection-retail-0013.xml" REID_MODEL = "intel/person-reidentification-retail-0287/FP16/person-reidentification-retail-0287.xml" # For CPU: change "GPU" to "CPU". For NPU: change "GPU" to "NPU". DEVICE = "GPU" DET_THRESHOLD = 0.6 MATCH_THRESHOLD = 0.6 def person_embeddings(video_frame): """Yield ((x, y, w, h), normalized_embedding) for each classified person.""" for region in video_frame.regions(): rect = region.rect() emb = None for tensor in region.tensors(): if tensor.is_detection(): continue data = np.array(tensor.data(), dtype=np.float32) if data.size >= 256: emb = data[:256] if emb is None: continue emb = emb / (np.linalg.norm(emb) + 1e-9) yield (int(rect.x), int(rect.y), int(rect.w), int(rect.h)), emb def run_pipeline(source_desc, on_frame): # Convert to BGR before inference so gvaclassify's embedding tensors survive # to the appsink (a later format-changing videoconvert would strip them). pipeline = Gst.parse_launch( f"{source_desc} ! videoconvert ! video/x-raw,format=BGR ! " f"gvadetect model={DETECTION_MODEL} device={DEVICE} " f"threshold={DET_THRESHOLD} ! queue ! " f"gvaclassify model={REID_MODEL} device={DEVICE} ! queue ! " "appsink name=sink emit-signals=true sync=false max-buffers=4 drop=false" ) sink = pipeline.get_by_name("sink") sink.connect("new-sample", on_frame) pipeline.set_state(Gst.State.PLAYING) pipeline.get_bus().timed_pop_filtered( Gst.CLOCK_TIME_NONE, Gst.MessageType.EOS | Gst.MessageType.ERROR) pipeline.set_state(Gst.State.NULL) # 1. Compute the reference embedding from the enrolled person image. # person_a.jpg is already a person crop, so run the re-identification model on # the whole frame (inference-region=full-frame) instead of detecting first. ref = {"emb": None} def on_reference(sink): sample = sink.emit("pull-sample") if sample is None: return Gst.FlowReturn.OK vf = VideoFrame(sample.get_buffer(), caps=sample.get_caps()) for tensor in vf.tensors(): data = np.array(tensor.data(), dtype=np.float32) if data.size >= 256: emb = data[:256] ref["emb"] = emb / (np.linalg.norm(emb) + 1e-9) return Gst.FlowReturn.OK reference_pipeline = Gst.parse_launch( f"filesrc location={REFERENCE_IMAGE} ! jpegdec ! videoconvert ! " f"video/x-raw,format=BGR ! " f"gvainference model={REID_MODEL} device={DEVICE} " f"inference-region=full-frame ! queue ! " "appsink name=sink emit-signals=true sync=false max-buffers=4 drop=false" ) ref_sink = reference_pipeline.get_by_name("sink") ref_sink.connect("new-sample", on_reference) reference_pipeline.set_state(Gst.State.PLAYING) reference_pipeline.get_bus().timed_pop_filtered( Gst.CLOCK_TIME_NONE, Gst.MessageType.EOS | Gst.MessageType.ERROR) reference_pipeline.set_state(Gst.State.NULL) if ref["emb"] is None: raise SystemExit("Could not compute the reference embedding") ref_emb = ref["emb"] # 2. Process the video, boxing only persons that match the reference identity. writer = {"w": None} match_frames = 0 def on_video(sink): global match_frames sample = sink.emit("pull-sample") if sample is None: return Gst.FlowReturn.OK vf = VideoFrame(sample.get_buffer(), caps=sample.get_caps()) matches = [] for (x, y, w, h), emb in person_embeddings(vf): similarity = float(np.dot(emb, ref_emb)) if similarity >= MATCH_THRESHOLD: matches.append((x, y, w, h, similarity)) with vf.data() as mat: frame = mat.copy() if writer["w"] is None: frame_h, frame_w = frame.shape[:2] structure = sample.get_caps().get_structure(0) ok_fr, fps_n, fps_d = structure.get_fraction("framerate") fps = fps_n / fps_d if ok_fr and fps_d else 12 writer["w"] = cv2.VideoWriter( OUTPUT_VIDEO, cv2.VideoWriter_fourcc(*"mp4v"), fps, (frame_w, frame_h)) for x, y, w, h, similarity in matches: cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) cv2.putText(frame, f"RE-ID {similarity:.2f}", (x, max(15, y - 8)), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) if matches: match_frames += 1 writer["w"].write(frame) return Gst.FlowReturn.OK run_pipeline(f"filesrc location={INPUT_VIDEO} ! decodebin3", on_video) if writer["w"] is not None: writer["w"].release() print(f"Frames with a re-identified person: {match_frames}", flush=True) print(f"Saved: {OUTPUT_VIDEO}", flush=True)