video_frame_pump.h 6.39 KB
Newer Older
1
// Copyright 2015 The Chromium Authors. All rights reserved.
2 3 4
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6
#ifndef REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_
#define REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_
7

8 9 10
#include <stddef.h>
#include <stdint.h>

11
#include <memory>
avi's avatar
avi committed
12
#include <vector>
13

14
#include "base/macros.h"
sergeyu's avatar
sergeyu committed
15
#include "base/threading/thread_checker.h"
16 17
#include "base/time/time.h"
#include "base/timer/timer.h"
18
#include "remoting/codec/video_encoder.h"
19
#include "remoting/proto/video.pb.h"
20
#include "remoting/protocol/capture_scheduler.h"
sergeyu's avatar
sergeyu committed
21
#include "remoting/protocol/video_stream.h"
22
#include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
23

24
namespace base {
25
class SingleThreadTaskRunner;
26 27
}  // namespace base

28
namespace remoting {
29
namespace protocol {
30

31
class VideoFeedbackStub;
32
class VideoStub;
33

sergeyu's avatar
sergeyu committed
34 35
// Class responsible for scheduling frame captures from a screen capturer.,
// delivering them to a VideoEncoder to encode, and
36 37
// finally passing the encoded video packets to the specified VideoStub to send
// on the network.
38 39 40
//
// THREADING
//
41 42 43
// This class is supplied TaskRunners to use for capture, encode and network
// operations.  Capture, encode and network transmission tasks are interleaved
// as illustrated below:
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
//
// |       CAPTURE       ENCODE     NETWORK
// |    .............
// |    .  Capture  .
// |    .............
// |                  ............
// |                  .          .
// |    ............. .          .
// |    .  Capture  . .  Encode  .
// |    ............. .          .
// |                  .          .
// |                  ............
// |    ............. ............ ..........
// |    .  Capture  . .          . .  Send  .
// |    ............. .          . ..........
// |                  .  Encode  .
// |                  .          .
// |                  .          .
// |                  ............
// | Time
// v
//
66
// VideoFramePump would ideally schedule captures so as to saturate the slowest
67 68 69
// of the capture, encode and network processes.  However, it also needs to
// rate-limit captures to avoid overloading the host system, either by consuming
// too much CPU, or hogging the host's graphics subsystem.
sergeyu's avatar
sergeyu committed
70 71
class VideoFramePump : public VideoStream,
                       public webrtc::DesktopCapturer::Callback {
72
 public:
73
  // Creates a VideoFramePump running capture, encode and network tasks on the
sergeyu's avatar
sergeyu committed
74 75
  // supplied TaskRunners. Video will be pumped to |video_stub|, which must
  // outlive the pump..
76 77 78 79
  VideoFramePump(scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
                 std::unique_ptr<webrtc::DesktopCapturer> capturer,
                 std::unique_ptr<VideoEncoder> encoder,
                 protocol::VideoStub* video_stub);
sergeyu's avatar
sergeyu committed
80
  ~VideoFramePump() override;
81

sergeyu's avatar
sergeyu committed
82
  // VideoStream interface.
83 84
  void SetEventTimestampsSource(scoped_refptr<InputEventTimestampsSource>
                                    event_timestamps_source) override;
sergeyu's avatar
sergeyu committed
85 86 87
  void Pause(bool pause) override;
  void SetLosslessEncode(bool want_lossless) override;
  void SetLosslessColor(bool want_lossless) override;
88
  void SetObserver(Observer* observer) override;
89

90 91 92 93
  protocol::VideoFeedbackStub* video_feedback_stub() {
    return &capture_scheduler_;
  }

94
 private:
95 96 97 98
  struct FrameTimestamps {
    FrameTimestamps();
    ~FrameTimestamps();

99 100 101
    // The following field is not-null for a single frame after each incoming
    // input event.
    InputEventTimestamps input_event_timestamps;
102 103 104 105 106 107 108 109 110

    base::TimeTicks capture_started_time;
    base::TimeTicks capture_ended_time;
    base::TimeTicks encode_started_time;
    base::TimeTicks encode_ended_time;
    base::TimeTicks can_send_time;
  };

  struct PacketWithTimestamps {
111 112
    PacketWithTimestamps(std::unique_ptr<VideoPacket> packet,
                         std::unique_ptr<FrameTimestamps> timestamps);
113 114
    ~PacketWithTimestamps();

115 116
    std::unique_ptr<VideoPacket> packet;
    std::unique_ptr<FrameTimestamps> timestamps;
117 118
  };

sergeyu's avatar
sergeyu committed
119
  // webrtc::DesktopCapturer::Callback interface.
120 121
  void OnCaptureResult(webrtc::DesktopCapturer::Result result,
                       std::unique_ptr<webrtc::DesktopFrame> frame) override;
122

sergeyu's avatar
sergeyu committed
123
  // Callback for CaptureScheduler.
124
  void CaptureNextFrame();
125

126
  // Task running on the encoder thread to encode the |frame|.
127
  static std::unique_ptr<PacketWithTimestamps> EncodeFrame(
128
      VideoEncoder* encoder,
129 130
      std::unique_ptr<webrtc::DesktopFrame> frame,
      std::unique_ptr<FrameTimestamps> timestamps);
131 132

  // Task called when a frame has finished encoding.
133
  void OnFrameEncoded(std::unique_ptr<PacketWithTimestamps> packet);
134 135

  // Sends |packet| to the client.
136
  void SendPacket(std::unique_ptr<PacketWithTimestamps> packet);
137 138 139 140

  // Helper called from SendPacket() to calculate timing fields in the |packet|
  // before sending it.
  void UpdateFrameTimers(VideoPacket* packet, FrameTimestamps* timestamps);
141

142
  // Callback passed to |video_stub_|.
143 144 145 146 147 148 149
  void OnVideoPacketSent();

  // Called by |keep_alive_timer_|.
  void SendKeepAlivePacket();

  // Callback for |video_stub_| called after a keep-alive packet is sent.
  void OnKeepAlivePacketSent();
150

sergeyu's avatar
sergeyu committed
151
  // Task runner used to run |encoder_|.
152
  scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_;
153

sergeyu's avatar
sergeyu committed
154
  // Capturer used to capture the screen.
155
  std::unique_ptr<webrtc::DesktopCapturer> capturer_;
156

157
  // Used to encode captured frames. Always accessed on the encode thread.
158
  std::unique_ptr<VideoEncoder> encoder_;
159

160 161
  scoped_refptr<InputEventTimestampsSource> event_timestamps_source_;

sergeyu's avatar
sergeyu committed
162
  // Interface through which video frames are passed to the client.
163
  protocol::VideoStub* video_stub_;
164

165
  Observer* observer_ = nullptr;
sergeyu's avatar
sergeyu committed
166
  webrtc::DesktopSize frame_size_;
sergeyu's avatar
sergeyu committed
167
  webrtc::DesktopVector frame_dpi_;
sergeyu's avatar
sergeyu committed
168

169 170
  // Timer used to ensure that we send empty keep-alive frames to the client
  // even when the video stream is paused or encoder is busy.
sergeyu's avatar
sergeyu committed
171 172 173 174 175
  base::Timer keep_alive_timer_;

  // CaptureScheduler calls CaptureNextFrame() whenever a new frame needs to be
  // captured.
  CaptureScheduler capture_scheduler_;
176

177
  // Timestamps for the frame that's being captured.
178
  std::unique_ptr<FrameTimestamps> captured_frame_timestamps_;
179 180 181

  bool send_pending_ = false;

avi's avatar
avi committed
182
  std::vector<std::unique_ptr<PacketWithTimestamps>> pending_packets_;
183

184 185
  base::ThreadChecker thread_checker_;

sergeyu's avatar
sergeyu committed
186
  base::WeakPtrFactory<VideoFramePump> weak_factory_;
187

188
  DISALLOW_COPY_AND_ASSIGN(VideoFramePump);
189 190
};

191
}  // namespace protocol
192 193
}  // namespace remoting

194
#endif  // REMOTING_PROTOCOL_VIDEO_FRAME_PUMP_H_