Open3D (C++ API)  0.16.1
VideoScaler.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// The MIT License (MIT)
5//
6// Copyright (c) 2018-2021 www.open3d.org
7//
8// Permission is hereby granted, free of charge, to any person obtaining a copy
9// of this software and associated documentation files (the "Software"), to deal
10// in the Software without restriction, including without limitation the rights
11// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12// copies of the Software, and to permit persons to whom the Software is
13// furnished to do so, subject to the following conditions:
14//
15// The above copyright notice and this permission notice shall be included in
16// all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24// IN THE SOFTWARE.
25// ----------------------------------------------------------------------------
26// ----------------------------------------------------------------------------
27// Contains source code from
28// https://github.com/mpromonet/webrtc-streamer
29//
30// This software is in the public domain, furnished "as is", without technical
31// support, and with no warranty, express or implied, as to its usefulness for
32// any purpose.
33// ----------------------------------------------------------------------------
34//
35// This is a private header. It shall be hidden from Open3D's public API. Do not
36// put this in Open3D.h.in.
37
38#pragma once
39
40#include <api/media_stream_interface.h>
41#include <media/base/video_broadcaster.h>
42
44
45namespace open3d {
46namespace visualization {
47namespace webrtc_server {
48
49class VideoScaler : public rtc::VideoSinkInterface<webrtc::VideoFrame>,
50 public rtc::VideoSourceInterface<webrtc::VideoFrame> {
51public:
52 VideoScaler(rtc::scoped_refptr<BitmapTrackSourceInterface> video_source,
53 const std::map<std::string, std::string> &opts)
54 : video_source_(video_source),
55 width_(0),
56 height_(0),
57 rotation_(webrtc::kVideoRotation_0),
58 roi_x_(0),
59 roi_y_(0),
60 roi_width_(0),
61 roi_height_(0) {
62 if (opts.find("width") != opts.end()) {
63 width_ = std::stoi(opts.at("width"));
64 }
65 if (opts.find("height") != opts.end()) {
66 height_ = std::stoi(opts.at("height"));
67 }
68 if (opts.find("rotation") != opts.end()) {
69 int rotation = std::stoi(opts.at("rotation"));
70 switch (rotation) {
71 case 90:
72 rotation_ = webrtc::kVideoRotation_90;
73 break;
74 case 180:
75 rotation_ = webrtc::kVideoRotation_180;
76 break;
77 case 270:
78 rotation_ = webrtc::kVideoRotation_270;
79 break;
80 }
81 }
82 if (opts.find("roi_x") != opts.end()) {
83 roi_x_ = std::stoi(opts.at("roi_x"));
84 if (roi_x_ < 0) {
85 utility::LogWarning("Ignore roi_x={}, it must be >=0", roi_x_);
86 roi_x_ = 0;
87 }
88 }
89 if (opts.find("roi_y") != opts.end()) {
90 roi_y_ = std::stoi(opts.at("roi_y"));
91 if (roi_y_ < 0) {
92 utility::LogWarning("Ignore roi_y_={}, it must be >=0", roi_y_);
93 roi_y_ = 0;
94 }
95 }
96 if (opts.find("roi_width") != opts.end()) {
97 roi_width_ = std::stoi(opts.at("roi_width"));
98 if (roi_width_ <= 0) {
99 utility::LogWarning("Ignore roi_width={}, it must be >0",
100 roi_width_);
101 roi_width_ = 0;
102 }
103 }
104 if (opts.find("roi_height") != opts.end()) {
105 roi_height_ = std::stoi(opts.at("roi_height"));
106 if (roi_height_ <= 0) {
107 utility::LogWarning("Ignore roi_height={}, it must be >0",
108 roi_height_);
109 roi_height_ = 0;
110 }
111 }
112 }
113
114 virtual ~VideoScaler() {}
115
116 void OnFrame(const webrtc::VideoFrame &frame) override {
117 if (roi_x_ >= frame.width()) {
119 "The ROI position protrudes beyond the right edge of the "
120 "image. Ignore roi_x.");
121 roi_x_ = 0;
122 }
123 if (roi_y_ >= frame.height()) {
125 "The ROI position protrudes beyond the right edge of the "
126 "image. Ignore roi_y_.");
127 roi_y_ = 0;
128 }
129 if (roi_width_ != 0 && (roi_width_ + roi_x_) > frame.width()) {
131 "The ROI position protrudes beyond the right edge of the "
132 "image. Ignore roi_width_.");
133 roi_width_ = 0;
134 }
135 if (roi_height_ != 0 && (roi_height_ + roi_y_) > frame.height()) {
137 "The ROI position protrudes beyond the right edge of the "
138 "image. Ignore roi_height_.");
139 roi_height_ = 0;
140 }
141
142 if (roi_width_ == 0) {
143 roi_width_ = frame.width() - roi_x_;
144 }
145 if (roi_height_ == 0) {
146 roi_height_ = frame.height() - roi_y_;
147 }
148
149 // source image is croped but destination image size is not set
150 if ((roi_width_ != frame.width() || roi_height_ != frame.height()) &&
151 (height_ == 0 && width_ == 0)) {
152 height_ = roi_height_;
153 width_ = roi_width_;
154 }
155
156 if ((height_ == 0) && (width_ == 0) &&
157 (rotation_ == webrtc::kVideoRotation_0)) {
158 broadcaster_.OnFrame(frame);
159 } else {
160 int height = height_;
161 int width = width_;
162 if ((height == 0) && (width == 0)) {
163 height = frame.height();
164 width = frame.width();
165 } else if (height == 0) {
166 height = (roi_height_ * width) / roi_width_;
167 } else if (width == 0) {
168 width = (roi_width_ * height) / roi_height_;
169 }
170 rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer =
171 webrtc::I420Buffer::Create(width, height);
172 if (roi_width_ != frame.width() || roi_height_ != frame.height()) {
173 scaled_buffer->CropAndScaleFrom(
174 *frame.video_frame_buffer()->ToI420(), roi_x_, roi_y_,
175 roi_width_, roi_height_);
176 } else {
177 scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
178 }
179 webrtc::VideoFrame scaledFrame =
180 webrtc::VideoFrame(scaled_buffer, frame.timestamp(),
181 frame.render_time_ms(), rotation_);
182
183 broadcaster_.OnFrame(scaledFrame);
184 }
185 }
186
187 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame> *sink,
188 const rtc::VideoSinkWants &wants) override {
189 video_source_->AddOrUpdateSink(this, wants);
190
191 broadcaster_.AddOrUpdateSink(sink, wants);
192 }
193
195 rtc::VideoSinkInterface<webrtc::VideoFrame> *sink) override {
196 video_source_->RemoveSink(this);
197
198 broadcaster_.RemoveSink(sink);
199 }
200
201 int width() { return roi_width_; }
202 int height() { return roi_height_; }
203
204private:
205 rtc::scoped_refptr<BitmapTrackSourceInterface> video_source_;
206 rtc::VideoBroadcaster broadcaster_;
207
208 int width_;
209 int height_;
210 webrtc::VideoRotation rotation_;
211 int roi_x_;
212 int roi_y_;
213 int roi_width_;
214 int roi_height_;
215};
216
217} // namespace webrtc_server
218} // namespace visualization
219} // namespace open3d
Rect frame
Definition: BitmapWindowSystem.cpp:49
#define LogWarning(...)
Definition: Logging.h:79
void RemoveSink(rtc::VideoSinkInterface< webrtc::VideoFrame > *sink) override
Definition: VideoScaler.h:194
void OnFrame(const webrtc::VideoFrame &frame) override
Definition: VideoScaler.h:116
virtual ~VideoScaler()
Definition: VideoScaler.h:114
int width()
Definition: VideoScaler.h:201
void AddOrUpdateSink(rtc::VideoSinkInterface< webrtc::VideoFrame > *sink, const rtc::VideoSinkWants &wants) override
Definition: VideoScaler.h:187
VideoScaler(rtc::scoped_refptr< BitmapTrackSourceInterface > video_source, const std::map< std::string, std::string > &opts)
Definition: VideoScaler.h:52
int height()
Definition: VideoScaler.h:202
Definition: PinholeCameraIntrinsic.cpp:35