webrtc_video_stream.cc 11 KB
Newer Older
sergeyu's avatar
sergeyu committed
1 2 3 4 5 6
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "remoting/protocol/webrtc_video_stream.h"

7 8
#include <utility>

9
#include "base/bind.h"
sergeyu's avatar
sergeyu committed
10
#include "base/logging.h"
11
#include "base/memory/ptr_util.h"
12
#include "build/build_config.h"
13
#include "remoting/base/constants.h"
14
#include "remoting/codec/webrtc_video_encoder_proxy.h"
15
#include "remoting/codec/webrtc_video_encoder_vpx.h"
16 17
#include "remoting/protocol/frame_stats.h"
#include "remoting/protocol/host_video_stats_dispatcher.h"
18
#include "remoting/protocol/webrtc_dummy_video_capturer.h"
19
#include "remoting/protocol/webrtc_frame_scheduler_simple.h"
20
#include "remoting/protocol/webrtc_transport.h"
21 22 23
#include "third_party/webrtc/api/mediastreaminterface.h"
#include "third_party/webrtc/api/peerconnectioninterface.h"
#include "third_party/webrtc/api/test/fakeconstraints.h"
24
#include "third_party/webrtc/media/base/videocapturer.h"
sergeyu's avatar
sergeyu committed
25

26 27 28 29
#if defined(USE_H264_ENCODER)
#include "remoting/codec/webrtc_video_encoder_gpu.h"
#endif

sergeyu's avatar
sergeyu committed
30 31 32
namespace remoting {
namespace protocol {

33 34
namespace {

35 36 37
const char kStreamLabel[] = "screen_stream";
const char kVideoLabel[] = "screen_video";

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
std::string EncodeResultToString(WebrtcVideoEncoder::EncodeResult result) {
  using EncodeResult = WebrtcVideoEncoder::EncodeResult;

  switch (result) {
    case EncodeResult::SUCCEEDED:
      return "Succeeded";
    case EncodeResult::FRAME_SIZE_EXCEEDS_CAPABILITY:
      return "Frame size exceeds capability";
    case EncodeResult::UNKNOWN_ERROR:
      return "Unknown error";
  }
  NOTREACHED();
  return "";
}

}  // namespace

55
struct WebrtcVideoStream::FrameStats {
56 57 58
  // The following fields is not null only for one frame after each incoming
  // input event.
  InputEventTimestamps input_event_timestamps;
59 60 61 62 63 64

  base::TimeTicks capture_started_time;
  base::TimeTicks capture_ended_time;
  base::TimeDelta capture_delay;
  base::TimeTicks encode_started_time;
  base::TimeTicks encode_ended_time;
65 66

  uint32_t capturer_id = 0;
67 68
};

69
WebrtcVideoStream::WebrtcVideoStream()
70 71 72 73 74 75 76 77 78 79 80 81 82
    : video_stats_dispatcher_(kStreamLabel), weak_factory_(this) {
  encoder_selector_.RegisterEncoder(
      base::Bind(&WebrtcVideoEncoderVpx::IsSupportedByVP8),
      base::Bind(&WebrtcVideoEncoderVpx::CreateForVP8));
  encoder_selector_.RegisterEncoder(
      base::Bind(&WebrtcVideoEncoderVpx::IsSupportedByVP9),
      base::Bind(&WebrtcVideoEncoderVpx::CreateForVP9));
#if defined(USE_H264_ENCODER)
  encoder_selector_.RegisterEncoder(
      base::Bind(&WebrtcVideoEncoderGpu::IsSupportedByH264),
      base::Bind(&WebrtcVideoEncoderGpu::CreateForH264));
#endif
}
sergeyu's avatar
sergeyu committed
83 84

WebrtcVideoStream::~WebrtcVideoStream() {
85
  DCHECK(thread_checker_.CalledOnValidThread());
86 87 88 89
  if (stream_) {
    for (const auto& track : stream_->GetVideoTracks()) {
      stream_->RemoveTrack(track.get());
    }
90
    peer_connection_->RemoveStream(stream_.get());
sergeyu's avatar
sergeyu committed
91
  }
92 93
}

94
void WebrtcVideoStream::Start(
95
    std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer,
96
    WebrtcTransport* webrtc_transport,
97
    scoped_refptr<base::SequencedTaskRunner> encode_task_runner) {
98
  DCHECK(thread_checker_.CalledOnValidThread());
99 100
  DCHECK(webrtc_transport);
  DCHECK(desktop_capturer);
101
  DCHECK(encode_task_runner);
102 103 104

  scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_connection_factory(
      webrtc_transport->peer_connection_factory());
105
  peer_connection_ = webrtc_transport->peer_connection();
106
  DCHECK(peer_connection_factory);
107 108
  DCHECK(peer_connection_);

109
  encode_task_runner_ = std::move(encode_task_runner);
110 111
  capturer_ = std::move(desktop_capturer);
  webrtc_transport_ = webrtc_transport;
112 113 114 115 116

  webrtc_transport_->video_encoder_factory()->RegisterEncoderSelectedCallback(
      base::Bind(&WebrtcVideoStream::OnEncoderCreated,
                 weak_factory_.GetWeakPtr()));

117
  capturer_->Start(this);
118 119 120 121 122 123

  // Set video stream constraints.
  webrtc::FakeConstraints video_constraints;
  video_constraints.AddMandatory(
      webrtc::MediaConstraintsInterface::kMinFrameRate, 5);

124
  rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> src =
125 126
      peer_connection_factory->CreateVideoSource(new WebrtcDummyVideoCapturer(),
                                                 &video_constraints);
127
  rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track =
128
      peer_connection_factory->CreateVideoTrack(kVideoLabel, src);
129 130 131

  stream_ = peer_connection_factory->CreateLocalMediaStream(kStreamLabel);

132 133 134 135 136 137 138 139 140
  // AddTrack() may fail only if there is another track with the same name,
  // which is impossible because it's a brand new stream.
  bool result = stream_->AddTrack(video_track.get());
  DCHECK(result);

  // AddStream() may fail if there is another stream with the same name or when
  // the PeerConnection is closed, neither is expected.
  result = peer_connection_->AddStream(stream_.get());
  DCHECK(result);
141

142 143 144 145
  scheduler_.reset(new WebrtcFrameSchedulerSimple());
  scheduler_->Start(
      webrtc_transport_->video_encoder_factory(),
      base::Bind(&WebrtcVideoStream::CaptureNextFrame, base::Unretained(this)));
146

147 148 149
  video_stats_dispatcher_.Init(webrtc_transport_->CreateOutgoingChannel(
                                   video_stats_dispatcher_.channel_name()),
                               this);
sergeyu's avatar
sergeyu committed
150 151
}

152 153 154
void WebrtcVideoStream::SetEventTimestampsSource(
    scoped_refptr<InputEventTimestampsSource> event_timestamps_source) {
  event_timestamps_source_ = event_timestamps_source;
sergeyu's avatar
sergeyu committed
155 156
}

157
void WebrtcVideoStream::Pause(bool pause) {
158
  DCHECK(thread_checker_.CalledOnValidThread());
159
  scheduler_->Pause(pause);
sergeyu's avatar
sergeyu committed
160 161 162 163 164 165 166 167 168 169
}

void WebrtcVideoStream::SetLosslessEncode(bool want_lossless) {
  NOTIMPLEMENTED();
}

void WebrtcVideoStream::SetLosslessColor(bool want_lossless) {
  NOTIMPLEMENTED();
}

170
void WebrtcVideoStream::SetObserver(Observer* observer) {
171
  DCHECK(thread_checker_.CalledOnValidThread());
172
  observer_ = observer;
173 174 175 176 177 178 179
}

void WebrtcVideoStream::OnCaptureResult(
    webrtc::DesktopCapturer::Result result,
    std::unique_ptr<webrtc::DesktopFrame> frame) {
  DCHECK(thread_checker_.CalledOnValidThread());

180 181
  current_frame_stats_->capture_ended_time = base::TimeTicks::Now();
  current_frame_stats_->capture_delay =
182
      base::TimeDelta::FromMilliseconds(frame ? frame->capture_time_ms() : 0);
183

184
  WebrtcVideoEncoder::FrameParams frame_params;
185
  if (!scheduler_->OnFrameCaptured(frame.get(), &frame_params)) {
186
    return;
187 188 189 190 191 192 193 194 195 196 197 198 199 200
  }

  // TODO(sergeyu): Handle ERROR_PERMANENT result here.
  if (frame) {
    webrtc::DesktopVector dpi =
        frame->dpi().is_zero() ? webrtc::DesktopVector(kDefaultDpi, kDefaultDpi)
                               : frame->dpi();

    if (!frame_size_.equals(frame->size()) || !frame_dpi_.equals(dpi)) {
      frame_size_ = frame->size();
      frame_dpi_ = dpi;
      if (observer_)
        observer_->OnVideoSizeChanged(this, frame_size_, frame_dpi_);
    }
201 202

    current_frame_stats_->capturer_id = frame->capturer_id();
203 204 205 206 207 208 209 210

    if (!encoder_) {
      encoder_selector_.SetDesktopFrame(*frame);
      encoder_ = encoder_selector_.CreateEncoder();

      // TODO(zijiehe): Permanently stop the video stream if we cannot create an
      // encoder for the |frame|.
    }
211
  }
212

213 214 215 216 217 218
  if (encoder_) {
    current_frame_stats_->encode_started_time = base::TimeTicks::Now();
    encoder_->Encode(
        std::move(frame), frame_params,
        base::Bind(&WebrtcVideoStream::OnFrameEncoded, base::Unretained(this)));
  }
219 220
}

221 222 223 224 225 226 227 228 229 230
void WebrtcVideoStream::OnChannelInitialized(
    ChannelDispatcherBase* channel_dispatcher) {
  DCHECK(&video_stats_dispatcher_ == channel_dispatcher);
}
void WebrtcVideoStream::OnChannelClosed(
    ChannelDispatcherBase* channel_dispatcher) {
  DCHECK(&video_stats_dispatcher_ == channel_dispatcher);
  LOG(WARNING) << "video_stats channel was closed.";
}

231 232 233
void WebrtcVideoStream::CaptureNextFrame() {
  DCHECK(thread_checker_.CalledOnValidThread());

234 235 236
  current_frame_stats_.reset(new FrameStats());
  current_frame_stats_->capture_started_time = base::TimeTicks::Now();
  current_frame_stats_->input_event_timestamps =
237
      event_timestamps_source_->TakeLastEventTimestamps();
238

239
  capturer_->CaptureFrame();
240 241
}

242
void WebrtcVideoStream::OnFrameEncoded(
243
    WebrtcVideoEncoder::EncodeResult encode_result,
244
    std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame) {
245 246
  DCHECK(thread_checker_.CalledOnValidThread());

247 248
  current_frame_stats_->encode_ended_time = base::TimeTicks::Now();

249
  HostFrameStats stats;
250
  scheduler_->OnFrameEncoded(frame.get(), &stats);
251

252 253 254
  if (encode_result != WebrtcVideoEncoder::EncodeResult::SUCCEEDED) {
    LOG(ERROR) << "Video encoder returns error "
               << EncodeResultToString(encode_result);
255 256
    // TODO(zijiehe): Restart the video stream.
    encoder_.reset();
257 258 259
    return;
  }

260
  if (!frame) {
261 262 263
    return;
  }

264 265
  webrtc::EncodedImageCallback::Result result =
      webrtc_transport_->video_encoder_factory()->SendEncodedFrame(
266
          *frame, current_frame_stats_->capture_started_time);
267 268 269 270 271 272 273 274
  if (result.error != webrtc::EncodedImageCallback::Result::OK) {
    // TODO(sergeyu): Stop the stream.
    LOG(ERROR) << "Failed to send video frame.";
    return;
  }

  // Send FrameStats message.
  if (video_stats_dispatcher_.is_connected()) {
275
    stats.frame_size = frame ? frame->data.size() : 0;
276

277
    if (!current_frame_stats_->input_event_timestamps.is_null()) {
278
      stats.capture_pending_delay =
279 280
          current_frame_stats_->capture_started_time -
          current_frame_stats_->input_event_timestamps.host_timestamp;
281
      stats.latest_event_timestamp =
282
          current_frame_stats_->input_event_timestamps.client_timestamp;
283
    }
284

285
    stats.capture_delay = current_frame_stats_->capture_delay;
286

287
    // Total overhead time for IPC and threading when capturing frames.
288
    stats.capture_overhead_delay =
289 290
        (current_frame_stats_->capture_ended_time -
         current_frame_stats_->capture_started_time) -
291 292
        stats.capture_delay;

293 294
    stats.encode_pending_delay = current_frame_stats_->encode_started_time -
                                 current_frame_stats_->capture_ended_time;
295

296 297
    stats.encode_delay = current_frame_stats_->encode_ended_time -
                         current_frame_stats_->encode_started_time;
298

299
    stats.capturer_id = current_frame_stats_->capturer_id;
300 301 302

    video_stats_dispatcher_.OnVideoFrameStats(result.frame_id, stats);
  }
sergeyu's avatar
sergeyu committed
303 304
}

305 306
void WebrtcVideoStream::OnEncoderCreated(webrtc::VideoCodecType codec_type) {
  DCHECK(thread_checker_.CalledOnValidThread());
307 308
  // The preferred codec id depends on the order of
  // |encoder_selector_|.RegisterEncoder().
309
  if (codec_type == webrtc::kVideoCodecVP8) {
310 311
    LOG(WARNING) << "VP8 video codec is preferred.";
    encoder_selector_.SetPreferredCodec(0);
312
  } else if (codec_type == webrtc::kVideoCodecVP9) {
313 314
    LOG(WARNING) << "VP9 video codec is preferred.";
    encoder_selector_.SetPreferredCodec(1);
315
  } else if (codec_type == webrtc::kVideoCodecH264) {
316
#if defined(USE_H264_ENCODER)
317 318
    LOG(WARNING) << "H264 video codec is preferred.";
    encoder_selector_.SetPreferredCodec(2);
319
#else
320
    NOTIMPLEMENTED();
321
#endif
322 323 324 325 326
  } else {
    LOG(FATAL) << "Unknown codec type: " << codec_type;
  }
}

sergeyu's avatar
sergeyu committed
327 328
}  // namespace protocol
}  // namespace remoting