Update app.py

app.py

@@ -21,6 +21,7 @@ import time
 import matplotlib.pyplot as plt
 from pathlib import Path
 import plotly.express as px
+from google.cloud import videointelligence
 
 # Set page config
 st.set_page_config(
@@ -515,6 +516,33 @@ def process_video_file(video_file, analysis_types):
         "labels": {}
     }
     
+    # Add Video Intelligence API integration for semantic analysis
+    if "Semantic" in analysis_types:
+        # Initialize video intelligence client
+        video_client = videointelligence.VideoIntelligenceServiceClient(credentials=credentials)
+        
+        # Set up features for semantic analysis
+        features = [
+            videointelligence.Feature.LABEL_DETECTION,
+            videointelligence.Feature.SHOT_CHANGE_DETECTION,
+            videointelligence.Feature.ACTION_RECOGNITION
+        ]
+        
+        # Process video for semantic understanding
+        operation = video_client.annotate_video(
+            request={"features": features, "input_content": video_file.read()}
+        )
+        
+        # Get semantic results
+        semantic_results = operation.result()
+        
+        # Store semantic insights for visualization
+        semantic_insights = {
+            "actions": extract_actions(semantic_results),
+            "segments": extract_segments(semantic_results),
+            "scene_labels": extract_labels(semantic_results)
+        }
+    
     try:
         frame_count = 0
         while frame_count < max_frames:  # Limit to 10 seconds
@@ -697,7 +725,10 @@ def process_video_file(video_file, analysis_types):
         os.unlink(output_path)
         
         # Return both the video and the detection statistics
-        return processed_video_bytes, detection_stats
+        return processed_video_bytes, {
+            "detection_stats": detection_stats,
+            "semantic_insights": semantic_insights if "Semantic" in analysis_types else None
+        }
         
     except Exception as e:
         # Clean up on error
@@ -959,6 +990,8 @@ def main():
             analysis_types.append("Face Detection")
         if st.sidebar.checkbox("Text Recognition"):
             analysis_types.append("Text")
+        if st.sidebar.checkbox("Semantic Analysis", value=True):
+            analysis_types.append("Semantic")
             
         st.sidebar.markdown("---")
         st.sidebar.warning("⚠️ Video analysis may use a significant amount of API calls. Use responsibly.")
@@ -996,7 +1029,7 @@ def main():
                     with st.spinner("Processing video (max 10 seconds)..."):
                         try:
                             # Process the video with enhanced detail
-                            processed_video, detection_stats = process_video_file(uploaded_file, analysis_types)
+                            processed_video, results = process_video_file(uploaded_file, analysis_types)
                             
                             if processed_video:
                                 # Offer download of processed video
@@ -1015,11 +1048,11 @@ def main():
                                 st.markdown("### Detailed Analysis Results")
                                 
                                 # Display object detection summary
-                                if "Objects" in analysis_types and detection_stats["objects"]:
+                                if "Objects" in analysis_types and results["detection_stats"]["objects"]:
                                     st.markdown("#### 📦 Objects Detected")
                                     
                                     # Sort objects by frequency
-                                    sorted_objects = dict(sorted(detection_stats["objects"].items(), 
+                                    sorted_objects = dict(sorted(results["detection_stats"]["objects"].items(), 
                                                         key=lambda x: x[1], reverse=True))
                                     
                                     # Create bar chart for objects
@@ -1040,21 +1073,21 @@ def main():
                                                 st.markdown(f"- {obj}: {count} occurrences")
                                 
                                 # Display face detection summary
-                                if "Face Detection" in analysis_types and detection_stats["faces"] > 0:
+                                if "Face Detection" in analysis_types and results["detection_stats"]["faces"] > 0:
                                     st.markdown("#### 👤 Face Analysis")
-                                    st.markdown(f"Total faces detected: {detection_stats['faces']}")
+                                    st.markdown(f"Total faces detected: {results['detection_stats']['faces']}")
                                 
                                 # Display text detection summary
-                                if "Text" in analysis_types and detection_stats["text_blocks"] > 0:
+                                if "Text" in analysis_types and results["detection_stats"]["text_blocks"] > 0:
                                     st.markdown("#### 📝 Text Analysis")
-                                    st.markdown(f"Total text blocks detected: {detection_stats['text_blocks']}")
+                                    st.markdown(f"Total text blocks detected: {results['detection_stats']['text_blocks']}")
                                 
                                 # Display label detection summary
-                                if "Labels" in analysis_types and detection_stats["labels"]:
+                                if "Labels" in analysis_types and results["detection_stats"]["labels"]:
                                     st.markdown("#### 🏷️ Scene Labels")
                                     
                                     # Sort labels by frequency
-                                    sorted_labels = dict(sorted(detection_stats["labels"].items(), 
+                                    sorted_labels = dict(sorted(results["detection_stats"]["labels"].items(), 
                                                        key=lambda x: x[1], reverse=True))
                                     
                                     # Create pie chart for top labels
@@ -1068,6 +1101,25 @@ def main():
                                         ax.pie(top_labels.values(), labels=top_labels.keys(), autopct='%1.1f%%')
                                         ax.set_title('Distribution of Scene Labels')
                                         st.pyplot(fig)
+                                
+                                # Display semantic analysis results
+                                if "Semantic" in analysis_types and results["semantic_insights"]:
+                                    st.markdown("### 🧠 Semantic Understanding")
+                                    
+                                    # Display scene context
+                                    st.markdown("#### Video Context")
+                                    for label in results["semantic_insights"]["scene_labels"][:10]:
+                                        st.markdown(f"- {label['description']}: {label['confidence']:.1%}")
+                                    
+                                    # Display activities detected
+                                    st.markdown("#### Activities")
+                                    for action in results["semantic_insights"]["actions"]:
+                                        st.markdown(f"- {action['description']} at {action['time_segment']}")
+                                    
+                                    # Visualize relationships and context
+                                    st.markdown("#### Scene Segments")
+                                    fig = create_segment_visualization(results["semantic_insights"]["segments"])
+                                    st.plotly_chart(fig)
                         
                         except Exception as e:
                             st.error(f"Error processing video: {str(e)}")