OpenShot Library | libopenshot  0.3.2
Tracker.cpp
Go to the documentation of this file.
1 
10 // Copyright (c) 2008-2019 OpenShot Studios, LLC
11 //
12 // SPDX-License-Identifier: LGPL-3.0-or-later
13 
14 #include <string>
15 #include <memory>
16 #include <fstream>
17 #include <iostream>
18 
19 #include "effects/Tracker.h"
20 #include "Exceptions.h"
21 #include "Timeline.h"
22 #include "trackerdata.pb.h"
23 
24 #include <google/protobuf/util/time_util.h>
25 
26 #include <QImage>
27 #include <QPainter>
28 #include <QRectF>
29 
30 using namespace std;
31 using namespace openshot;
32 using google::protobuf::util::TimeUtil;
33 
35 Tracker::Tracker(std::string clipTrackerDataPath)
36 {
37  // Init effect properties
38  init_effect_details();
39  // Instantiate a TrackedObjectBBox object and point to it
40  TrackedObjectBBox trackedDataObject;
41  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
42  // Tries to load the tracked object's data from protobuf file
43  trackedData->LoadBoxData(clipTrackerDataPath);
44  ClipBase* parentClip = this->ParentClip();
45  trackedData->ParentClip(parentClip);
46  trackedData->Id(std::to_string(0));
47  // Insert TrackedObject with index 0 to the trackedObjects map
48  trackedObjects.insert({0, trackedData});
49 }
50 
51 // Default constructor
52 Tracker::Tracker()
53 {
54  // Init effect properties
55  init_effect_details();
56  // Instantiate a TrackedObjectBBox object and point to it
57  TrackedObjectBBox trackedDataObject;
58  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59  ClipBase* parentClip = this->ParentClip();
60  trackedData->ParentClip(parentClip);
61  trackedData->Id(std::to_string(0));
62  // Insert TrackedObject with index 0 to the trackedObjects map
63  trackedObjects.insert({0, trackedData});
64 }
65 
66 
67 // Init effect settings
68 void Tracker::init_effect_details()
69 {
71  InitEffectInfo();
72 
74  info.class_name = "Tracker";
75  info.name = "Tracker";
76  info.description = "Track the selected bounding box through the video.";
77  info.has_audio = false;
78  info.has_video = true;
79  info.has_tracked_object = true;
80 
81  this->TimeScale = 1.0;
82 }
83 
84 // This method is required for all derived classes of EffectBase, and returns a
85 // modified openshot::Frame object
86 std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
87 {
88  // Get the frame's image
89  cv::Mat frame_image = frame->GetImageCV();
90 
91  // Initialize the Qt rectangle that will hold the positions of the bounding-box
92  QRectF boxRect;
93  // Initialize the image of the TrackedObject child clip
94  std::shared_ptr<QImage> childClipImage = nullptr;
95 
96  // Check if frame isn't NULL
97  if(!frame_image.empty() &&
98  trackedData->Contains(frame_number) &&
99  trackedData->visible.GetValue(frame_number) == 1)
100  {
101  // Get the width and height of the image
102  float fw = frame_image.size().width;
103  float fh = frame_image.size().height;
104 
105  // Get the bounding-box of given frame
106  BBox fd = trackedData->GetBox(frame_number);
107 
108  // Check if track data exists for the requested frame
109  if (trackedData->draw_box.GetValue(frame_number) == 1)
110  {
111  std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
112  int stroke_width = trackedData->stroke_width.GetValue(frame_number);
113  float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
114  std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
115  float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
116 
117  // Create a rotated rectangle object that holds the bounding box
118  cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
119  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
120  (int) (fd.angle) );
121 
122  DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
123  DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
124  }
125 
126  // Get the image of the Tracked Object' child clip
127  if (trackedData->ChildClipId() != ""){
128  // Cast the parent timeline of this effect
129  Timeline* parentTimeline = static_cast<Timeline *>(ParentTimeline());
130  if (parentTimeline){
131  // Get the Tracked Object's child clip
132  Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
133  if (childClip){
134  // Get the image of the child clip for this frame
135  std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(frame_number);
136  childClipImage = childClipFrame->GetImage();
137 
138  // Set the Qt rectangle with the bounding-box properties
139  boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
140  (int)((fd.cy - fd.height/2)*fh),
141  (int)(fd.width*fw),
142  (int)(fd.height*fh) );
143  }
144  }
145  }
146 
147  }
148 
149  // Set image with drawn box to frame
150  // If the input image is NULL or doesn't have tracking data, it's returned as it came
151  frame->SetImageCV(frame_image);
152 
153  // Set the bounding-box image with the Tracked Object's child clip image
154  if (childClipImage){
155  // Get the frame image
156  QImage frameImage = *(frame->GetImage());
157 
158  // Set a Qt painter to the frame image
159  QPainter painter(&frameImage);
160 
161  // Draw the child clip image inside the bounding-box
162  painter.drawImage(boxRect, *childClipImage);
163 
164  // Set the frame image as the composed image
165  frame->AddImage(std::make_shared<QImage>(frameImage));
166  }
167 
168  return frame;
169 }
170 
171 void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
172  // Get the bouding box vertices
173  cv::Point2f vertices2f[4];
174  box.points(vertices2f);
175 
176  // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
177  // select min enclosing rectangle to draw on a small portion of the image
178  // cv::Rect rect = box.boundingRect();
179  // cv::Mat image = frame_image(rect)
180 
181  if(is_background){
182  cv::Mat overlayFrame;
183  frame_image.copyTo(overlayFrame);
184 
185  // draw bounding box background
186  cv::Point vertices[4];
187  for(int i = 0; i < 4; ++i){
188  vertices[i] = vertices2f[i];}
189 
190  cv::Rect rect = box.boundingRect();
191  cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
192  // add opacity
193  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
194  }
195  else{
196  cv::Mat overlayFrame;
197  frame_image.copyTo(overlayFrame);
198 
199  // Draw bounding box
200  for (int i = 0; i < 4; i++)
201  {
202  cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
203  thickness, cv::LINE_AA);
204  }
205 
206  // add opacity
207  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
208  }
209 }
210 
211 // Get the indexes and IDs of all visible objects in the given frame
212 std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
213 
214  // Initialize the JSON objects
215  Json::Value root;
216  root["visible_objects_index"] = Json::Value(Json::arrayValue);
217  root["visible_objects_id"] = Json::Value(Json::arrayValue);
218 
219  // Iterate through the tracked objects
220  for (const auto& trackedObject : trackedObjects){
221  // Get the tracked object JSON properties for this frame
222  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
223  if (trackedObjectJSON["visible"]["value"].asBool()){
224  // Save the object's index and ID if it's visible in this frame
225  root["visible_objects_index"].append(trackedObject.first);
226  root["visible_objects_id"].append(trackedObject.second->Id());
227  }
228  }
229 
230  return root.toStyledString();
231 }
232 
233 // Generate JSON string of this object
234 std::string Tracker::Json() const {
235 
236  // Return formatted string
237  return JsonValue().toStyledString();
238 }
239 
240 // Generate Json::Value for this object
241 Json::Value Tracker::JsonValue() const {
242 
243  // Create root json object
244  Json::Value root = EffectBase::JsonValue(); // get parent properties
245 
246  // Save the effect's properties on root
247  root["type"] = info.class_name;
248  root["protobuf_data_path"] = protobuf_data_path;
249  root["BaseFPS"]["num"] = BaseFPS.num;
250  root["BaseFPS"]["den"] = BaseFPS.den;
251  root["TimeScale"] = this->TimeScale;
252 
253  // Add trackedObjects IDs to JSON
254  Json::Value objects;
255  for (auto const& trackedObject : trackedObjects){
256  Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
257  // add object json
258  objects[trackedObject.second->Id()] = trackedObjectJSON;
259  }
260  root["objects"] = objects;
261 
262  // return JsonValue
263  return root;
264 }
265 
266 // Load JSON string into this object
267 void Tracker::SetJson(const std::string value) {
268 
269  // Parse JSON string into JSON objects
270  try
271  {
272  const Json::Value root = openshot::stringToJson(value);
273  // Set all values that match
274  SetJsonValue(root);
275  }
276  catch (const std::exception& e)
277  {
278  // Error parsing JSON (or missing keys)
279  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
280  }
281  return;
282 }
283 
284 // Load Json::Value into this object
285 void Tracker::SetJsonValue(const Json::Value root) {
286 
287  // Set parent data
288  EffectBase::SetJsonValue(root);
289 
290  if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
291  {
292  if (!root["BaseFPS"]["num"].isNull())
293  {
294  BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
295  }
296  if (!root["BaseFPS"]["den"].isNull())
297  {
298  BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
299  }
300  }
301 
302  if (!root["TimeScale"].isNull())
303  TimeScale = (double) root["TimeScale"].asDouble();
304 
305  // Set data from Json (if key is found)
306  if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
307  {
308  protobuf_data_path = root["protobuf_data_path"].asString();
309  if(!trackedData->LoadBoxData(protobuf_data_path))
310  {
311  std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
312  protobuf_data_path = "";
313  }
314  }
315 
316  if (!root["objects"].isNull()){
317  for (auto const& trackedObject : trackedObjects){
318  std::string obj_id = std::to_string(trackedObject.first);
319  if(!root["objects"][obj_id].isNull()){
320  trackedObject.second->SetJsonValue(root["objects"][obj_id]);
321  }
322  }
323  }
324 
325  // Set the tracked object's ids
326  if (!root["objects_id"].isNull()){
327  for (auto const& trackedObject : trackedObjects){
328  Json::Value trackedObjectJSON;
329  trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
330  trackedObject.second->SetJsonValue(trackedObjectJSON);
331  }
332  }
333 
334  return;
335 }
336 
337 // Get all properties for a specific frame
338 std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
339 
340  // Generate JSON properties list
341  Json::Value root;
342 
343  // Add trackedObject properties to JSON
344  Json::Value objects;
345  for (auto const& trackedObject : trackedObjects){
346  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
347  // add object json
348  objects[trackedObject.second->Id()] = trackedObjectJSON;
349  }
350  root["objects"] = objects;
351 
352  // Append effect's properties
353  root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
354  root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
355  root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
356  root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
357  root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
358  root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
359 
360  // Return formatted string
361  return root.toStyledString();
362 }
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:33
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:389
Exception for invalid JSON.
Definition: Exceptions.h:218
This class represents a timeline.
Definition: Timeline.h:150
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:408
This class contains the properties of a tracked object and functions to manipulate it.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]