Object tracking support (#31)

Author: ferjm

Cargo.lock

@@ -32,7 +32,7 @@ regex = "1.10.3"
 serde = "1.0"
 serde_json = "1.0"
 tokio = { version = "1.0", features = ["rt-multi-thread"] }
-edge-impulse-runner = { git = "https://github.com/edgeimpulse/edge-impulse-runner-rs.git", rev = "0ddef6a", default-features = false }
+edge-impulse-runner = { git = "https://github.com/edgeimpulse/edge-impulse-runner-rs.git", rev = "c357286", default-features = false }
 tempfile = "3.10"
 
 [dev-dependencies]

Cargo.toml

@@ -47,14 +47,41 @@ The plugin exposes results and ingestion status through standardized mechanisms:
           "x": 24,
           "y": 145,
           "width": 352,
-          "height": 239
+          "height": 239,
+          "object_id": 1
         }
       ]
     }
   }
   ```
 - **Video Metadata:**
   - Each detected object is attached as a `VideoRegionOfInterestMeta` with bounding box coordinates, label, and confidence.
+  - When object tracking is enabled, the `object_id` field is also included to track objects across frames.
+
+#### 1.1. Object Tracking
+- **Bus Message Example:**
+  ```json
+  {
+    "timestamp": 1234567890,
+    "type": "object-tracking",
+    "result": {
+      "object_tracking": [
+        {
+          "label": "person",
+          "value": 0.95,
+          "x": 24,
+          "y": 145,
+          "width": 352,
+          "height": 239,
+          "object_id": 1
+        }
+      ]
+    }
+  }
+  ```
+- **Video Metadata:**
+  - Object tracking results are attached as `VideoRegionOfInterestMeta` with bounding box coordinates, label, confidence, and object_id.
+  - The `object_id` field allows tracking the same object across multiple frames.
 
 #### 2. Classification
 - **Bus Message Example:**
@@ -530,7 +557,7 @@ Video overlay element that visualizes inference results by drawing bounding boxe
 Element Details:
 - Long name: Edge Impulse Overlay
 - Class: Filter/Effect/Video
-- Description: Draws bounding boxes on video frames based on ROI metadata
+- Description: Draws bounding boxes on video frames based on ROI metadata, including object tracking IDs
 
 Pad Templates:
 - Sink/Source pads (Always available):
@@ -543,7 +570,7 @@ Pad Templates:
 
 Key features:
 - Draws bounding boxes for object detection and visual anomaly detection results (from VideoRegionOfInterestMeta)
-- Displays class labels with confidence scores
+- Displays class labels with confidence scores and object tracking IDs when available
 - Supports wide range of video formats
 - Automatic text sizing: Calculates optimal font size based on frame dimensions and bounding box sizes
 - Text scale control: Use `text-scale-ratio` property to fine-tune text size (0.1x to 5.0x scaling)

README.md

@@ -373,6 +373,12 @@ fn create_pipeline(args: &VideoClassifyParams) -> Result<gst::Pipeline, Box<dyn
         .build()
         .expect("Could not create edgeimpulseoverlay element.");
 
+    println!("🎨 Overlay settings:");
+    println!("   📏 Stroke width: {}", args.stroke_width);
+    println!("   🎨 Text color: {}", args.text_color);
+    println!("   🎨 Background color: {}", args.background_color);
+    println!("   📏 Text scale ratio: {}", args.text_scale_ratio);
+
     let videoconvert2 = gst::ElementFactory::make("videoconvert")
         .property("n-threads", 4u32)
         .build()
@@ -480,12 +486,19 @@ fn example_main() -> Result<(), Box<dyn Error>> {
                     println!("   📋 Project Name: {}", metadata.project_name);
                     println!("   👤 Project Owner: {}", metadata.project_owner);
                     println!("   🏷️  Deploy Version: {}", metadata.deploy_version);
+                    println!(
+                        "   🎯 Has Object Tracking: {}",
+                        metadata.has_object_tracking
+                    );
 
                     if let Ok(result) = structure.get::<String>("result") {
                         if let Ok(json) = serde_json::from_str::<serde_json::Value>(&result) {
                             if let Some(boxes) = json["bounding_boxes"].as_array() {
                                 println!("   🎯 Object detection model");
                                 println!("   📦 Bounding boxes: {} objects", boxes.len());
+                                if !metadata.has_object_tracking {
+                                    println!("   ⚠️  Object tracking not enabled in this model");
+                                }
                             } else if json.get("anomaly").is_some() {
                                 println!("   🔍 Anomaly detection model");
                             } else if let Some(classification) = json["classification"].as_object()
@@ -548,9 +561,40 @@ fn example_main() -> Result<(), Box<dyn Error>> {
                                     if let (Some(label), Some(value)) =
                                         (bbox["label"].as_str(), bbox["value"].as_f64())
                                     {
+                                        let object_id_info =
+                                            if let Some(object_id) = bbox["object_id"].as_u64() {
+                                                format!(" (ID: {})", object_id)
+                                            } else {
+                                                String::new()
+                                            };
+
+                                        println!(
+                                            "  {}{}: {:.1}% (x: {}, y: {}, w: {}, h: {})",
+                                            label,
+                                            object_id_info,
+                                            value * 100.0,
+                                            bbox["x"].as_f64().unwrap_or(0.0),
+                                            bbox["y"].as_f64().unwrap_or(0.0),
+                                            bbox["width"].as_f64().unwrap_or(0.0),
+                                            bbox["height"].as_f64().unwrap_or(0.0)
+                                        );
+                                    }
+                                }
+                            }
+                            // Handle object tracking (separate field)
+                            else if let Some(tracking_boxes) = json["object_tracking"].as_array()
+                            {
+                                println!("Object tracking results:");
+                                for bbox in tracking_boxes {
+                                    if let (Some(label), Some(value), Some(object_id)) = (
+                                        bbox["label"].as_str(),
+                                        bbox["value"].as_f64(),
+                                        bbox["object_id"].as_u64(),
+                                    ) {
                                         println!(
-                                            "  {}: {:.1}% (x: {}, y: {}, w: {}, h: {})",
+                                            "  {} (ID: {}): {:.1}% (x: {}, y: {}, w: {}, h: {})",
                                             label,
+                                            object_id,
                                             value * 100.0,
                                             bbox["x"].as_f64().unwrap_or(0.0),
                                             bbox["y"].as_f64().unwrap_or(0.0),

examples/video_inference.rs

@@ -486,18 +486,21 @@ impl VideoFilterImpl for EdgeImpulseOverlay {
                 let width = meta.w as i32;
                 let height = meta.h as i32;
 
-                // Get label and confidence from parameters
+                // Get label, confidence, and object_id from parameters
                 let params_result = roi.params().find_map(|param| {
                     if param.name() != "detection" {
                         return None;
                     }
 
                     let label = param.get::<String>("label").ok()?;
                     let confidence = param.get::<f64>("confidence").ok()?;
-                    Some((label, confidence))
+                    let object_id = param.get::<u64>("object_id").ok();
+                    Some((label, confidence, object_id))
                 });
 
-                params_result.map(|(label, confidence)| (x, y, width, height, label, confidence))
+                params_result.map(|(label, confidence, object_id)| {
+                    (x, y, width, height, label, confidence, object_id)
+                })
             })
             .collect();
 
@@ -640,26 +643,52 @@ impl VideoFilterImpl for EdgeImpulseOverlay {
             );
 
             // Process each ROI
-            for (x, y, width, height, label, confidence) in rois {
+            for (x, y, width, height, label, confidence, object_id) in rois {
+                let display_label = if let Some(id) = object_id {
+                    format!("{} (ID: {})", label, id)
+                } else {
+                    label.clone()
+                };
+
                 gst::debug!(
                     CAT,
                     obj = self.obj(),
                     "Rendering ROI: {} ({:.1}%) at ({}, {}) {}x{}",
-                    label,
+                    display_label,
                     confidence * 100.0,
                     x,
                     y,
                     width,
                     height
                 );
 
-                // Get or assign color for this label
-                let color = if let Some(color) = label_colors.get(&label) {
+                // Get or assign color for this object (prefer object_id over label)
+                let color_key = if let Some(id) = object_id {
+                    format!("object_{}", id)
+                } else {
+                    label.clone()
+                };
+
+                let color = if let Some(color) = label_colors.get(&color_key) {
                     *color
                 } else {
                     let next_color = COLORS.get(label_colors.len() % COLORS.len()).unwrap();
                     let mut label_colors = self.label_colors.lock().unwrap();
-                    label_colors.insert(label.clone(), *next_color);
+                    label_colors.insert(color_key.clone(), *next_color);
+
+                    gst::debug!(
+                        CAT,
+                        obj = self.obj(),
+                        "Assigned color {:?} to {} (key: {})",
+                        next_color,
+                        if object_id.is_some() {
+                            format!("object ID {}", object_id.unwrap())
+                        } else {
+                            format!("label '{}'", label)
+                        },
+                        color_key
+                    );
+
                     *next_color
                 };
 
@@ -670,7 +699,7 @@ impl VideoFilterImpl for EdgeImpulseOverlay {
                         y,
                         width,
                         height,
-                        roi_type: label.clone(),
+                        roi_type: display_label.clone(),
                         color,
                     };
 
@@ -682,7 +711,7 @@ impl VideoFilterImpl for EdgeImpulseOverlay {
 
                 // Draw label if enabled
                 if settings.show_labels {
-                    let text = format!("{} {:.1}%", label, confidence * 100.0);
+                    let text = format!("{} {:.1}%", display_label, confidence * 100.0);
                     let text_x = x + 2;
                     // Position the text just slightly below the top of the bounding box
                     let text_y = y + 2;