qtdemux.c 452 KB
Newer Older
Artyom Baginski's avatar
Artyom Baginski committed
1
2
/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
3
 * Copyright (C) <2003> David A. Schleef <ds@schleef.org>
4
 * Copyright (C) <2006> Wim Taymans <wim@fluendo.com>
5
 * Copyright (C) <2007> Julien Moutte <julien@fluendo.com>
6
 * Copyright (C) <2009> Tim-Philipp Müller <tim centricular net>
7
 * Copyright (C) <2009> STEricsson <benjamin.gaignard@stericsson.com>
8
 * Copyright (C) <2013> Sreerenj Balachandran <sreerenj.balachandran@intel.com>
9
10
 * Copyright (C) <2013> Intel Corporation
 * Copyright (C) <2014> Centricular Ltd
11
 * Copyright (C) <2015> YouView TV Ltd.
12
 * Copyright (C) <2016> British Broadcasting Corporation
13
 *
Artyom Baginski's avatar
Artyom Baginski committed
14
15
16
17
18
19
20
21
22
23
24
25
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
26
27
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
Artyom Baginski's avatar
Artyom Baginski committed
28
29
 */

30
31
32
33
/**
 * SECTION:element-qtdemux
 *
 * Demuxes a .mov file into raw or compressed audio and/or video streams.
34
 *
35
36
 * This element supports both push and pull-based scheduling, depending on the
 * capabilities of the upstream elements.
37
38
 *
 * <refsect2>
39
 * <title>Example launch line</title>
40
 * |[
41
 * gst-launch-1.0 filesrc location=test.mov ! qtdemux name=demux  demux.audio_0 ! queue ! decodebin ! audioconvert ! audioresample ! autoaudiosink   demux.video_0 ! queue ! decodebin ! videoconvert ! videoscale ! autovideosink
42
 * ]| Play (parse and decode) a .mov file and try to output it to
43
44
45
46
47
48
 * an automatically detected soundcard and videosink. If the MOV file contains
 * compressed audio or video data, this will only work if you have the
 * right decoder elements/plugins installed.
 * </refsect2>
 */

49
50
51
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
52
53
54

#include "gst/gst-i18n-plugin.h"

55
#include <glib/gprintf.h>
56
#include <gst/tag/tag.h>
Wim Taymans's avatar
Wim Taymans committed
57
#include <gst/audio/audio.h>
58
#include <gst/video/video.h>
59

60
#include "qtatomparser.h"
61
62
#include "qtdemux_types.h"
#include "qtdemux_dump.h"
63
#include "fourcc.h"
64
#include "descriptors.h"
65
#include "qtdemux_lang.h"
66
#include "qtdemux.h"
67
#include "qtpalette.h"
68

69
70
71
#include "gst/riff/riff-media.h"
#include "gst/riff/riff-read.h"

72
73
#include <gst/pbutils/pbutils.h>

74
#include <stdio.h>
75
#include <stdlib.h>
Artyom Baginski's avatar
Artyom Baginski committed
76
#include <string.h>
77

78
79
80
#include <math.h>
#include <gst/math-compat.h>

81
82
83
#ifdef HAVE_ZLIB
# include <zlib.h>
#endif
84

85
86
87
/* max. size considered 'sane' for non-mdat atoms */
#define QTDEMUX_MAX_ATOM_SIZE (25*1024*1024)

88
89
90
/* if the sample index is larger than this, something is likely wrong */
#define QTDEMUX_MAX_SAMPLE_INDEX_SIZE (50*1024*1024)

91
92
93
94
95
96
/* For converting qt creation times to unix epoch times */
#define QTDEMUX_SECONDS_PER_DAY (60 * 60 * 24)
#define QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970 17
#define QTDEMUX_SECONDS_FROM_1904_TO_1970 (((1970 - 1904) * (guint64) 365 + \
    QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970) * QTDEMUX_SECONDS_PER_DAY)

97
#define STREAM_IS_EOS(s) (s->time_position == GST_CLOCK_TIME_NONE)
98

99
100
#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )

101
GST_DEBUG_CATEGORY (qtdemux_debug);
102

103
typedef struct _QtDemuxSegment QtDemuxSegment;
104
typedef struct _QtDemuxSample QtDemuxSample;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
105

106
107
typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
108
109
struct _QtDemuxSample
{
110
  guint32 size;
111
  gint32 pts_offset;            /* Add this value to timestamp to get the pts */
112
  guint64 offset;
113
114
  guint64 timestamp;            /* DTS In mov time */
  guint32 duration;             /* In mov time */
115
  gboolean keyframe;            /* TRUE when this packet is a keyframe */
116
117
};

118
119
120
121
122
123
124
/* Macros for converting to/from timescale */
#define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale))
#define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND))

#define QTTIME_TO_GSTTIME(qtdemux, value) (gst_util_uint64_scale((value), GST_SECOND, (qtdemux)->timescale))
#define GSTTIME_TO_QTTIME(qtdemux, value) (gst_util_uint64_scale((value), (qtdemux)->timescale, GST_SECOND))

125
/* timestamp is the DTS */
126
#define QTSAMPLE_DTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp))
127
/* timestamp + offset + cslg_shift is the outgoing PTS */
Nicolas Dufresne's avatar
Nicolas Dufresne committed
128
#define QTSAMPLE_PTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (stream)->cslg_shift + (sample)->pts_offset))
129
130
/* timestamp + offset is the PTS used for internal seek calcuations */
#define QTSAMPLE_PTS_NO_CSLG(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (sample)->pts_offset))
131
/* timestamp + duration - dts is the duration */
132
#define QTSAMPLE_DUR_DTS(stream, sample, dts) (QTSTREAMTIME_TO_GSTTIME ((stream), (sample)->timestamp + (sample)->duration) - (dts))
133
134
135

#define QTSAMPLE_KEYFRAME(stream,sample) ((stream)->all_keyframe || (sample)->keyframe)

136
137
138
139
140
141
142
143
144
145
146
/*
 * Quicktime has tracks and segments. A track is a continuous piece of
 * multimedia content. The track is not always played from start to finish but
 * instead, pieces of the track are 'cut out' and played in sequence. This is
 * what the segments do.
 *
 * Inside the track we have keyframes (K) and delta frames. The track has its
 * own timing, which starts from 0 and extends to end. The position in the track
 * is called the media_time.
 *
 * The segments now describe the pieces that should be played from this track
147
 * and are basically tuples of media_time/duration/rate entries. We can have
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 * multiple segments and they are all played after one another. An example:
 *
 * segment 1: media_time: 1 second, duration: 1 second, rate 1
 * segment 2: media_time: 3 second, duration: 2 second, rate 2
 *
 * To correctly play back this track, one must play: 1 second of media starting
 * from media_time 1 followed by 2 seconds of media starting from media_time 3
 * at a rate of 2.
 *
 * Each of the segments will be played at a specific time, the first segment at
 * time 0, the second one after the duration of the first one, etc.. Note that
 * the time in resulting playback is not identical to the media_time of the
 * track anymore.
 *
 * Visually, assuming the track has 4 second of media_time:
 *
 *                (a)                   (b)          (c)              (d)
 *         .-----------------------------------------------------------.
 * track:  | K.....K.........K........K.......K.......K...........K... |
 *         '-----------------------------------------------------------'
Stefan Kost's avatar
Stefan Kost committed
168
 *         0              1              2              3              4
169
170
171
172
173
174
 *           .------------^              ^   .----------^              ^
 *          /              .-------------'  /       .------------------'
 *         /              /          .-----'       /
 *         .--------------.         .--------------.
 *         | segment 1    |         | segment 2    |
 *         '--------------'         '--------------'
Stefan Kost's avatar
Stefan Kost committed
175
 *
176
 * The challenge here is to cut out the right pieces of the track for each of
177
178
 * the playback segments. This fortunately can easily be done with the SEGMENT
 * events of GStreamer.
179
180
181
182
183
184
185
186
 *
 * For playback of segment 1, we need to provide the decoder with the keyframe
 * (a), in the above figure, but we must instruct it only to output the decoded
 * data between second 1 and 2. We do this with a SEGMENT event for 1 to 2, time
 * position set to the time of the segment: 0.
 *
 * We then proceed to push data from keyframe (a) to frame (b). The decoder
 * decodes but clips all before media_time 1.
Stefan Kost's avatar
Stefan Kost committed
187
 *
188
189
190
191
192
193
 * After finishing a segment, we push out a new SEGMENT event with the clipping
 * boundaries of the new data.
 *
 * This is a good usecase for the GStreamer accumulated SEGMENT events.
 */

194
195
196
struct _QtDemuxSegment
{
  /* global time and duration, all gst time */
197
198
199
  GstClockTime time;
  GstClockTime stop_time;
  GstClockTime duration;
200
  /* media time of trak, all gst time */
201
202
  GstClockTime media_start;
  GstClockTime media_stop;
203
  gdouble rate;
204
205
  /* Media start time in trak timescale units */
  guint32 trak_media_start;
206
207
};

208
209
#define QTSEGMENT_IS_EMPTY(s) ((s)->media_start == GST_CLOCK_TIME_NONE)

210
211
212
213
214
215
216
/* Used with fragmented MP4 files (mfra atom) */
typedef struct
{
  GstClockTime ts;
  guint64 moof_offset;
} QtDemuxRandomAccessEntry;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
217
218
struct _QtDemuxStream
{
219
220
221
  GstPad *pad;

  /* stream type */
222
223
  guint32 subtype;
  GstCaps *caps;
224
  guint32 fourcc;
225
  gboolean sparse;
226

227
228
229
230
  gboolean new_caps;            /* If TRUE, caps need to be generated (by
                                 * calling _configure_stream()) This happens
                                 * for MSS and fragmented streams */

231
  gboolean new_stream;          /* signals that a stream_start is required */
232
233
234
235
  gboolean on_keyframe;         /* if this stream last pushed buffer was a
                                 * keyframe. This is important to identify
                                 * where to stop pushing buffers after a
                                 * segment stop time */
236

237
238
239
  /* if the stream has a redirect URI in its headers, we store it here */
  gchar *redirect_uri;

240
241
242
  /* track id */
  guint track_id;

243
  /* duration/scale */
244
  guint64 duration;             /* in timescale units */
245
  guint32 timescale;
246

247
  /* language */
248
  gchar lang_id[4];             /* ISO 639-2T language code */
249

250
251
252
253
  /* our samples */
  guint32 n_samples;
  QtDemuxSample *samples;
  gboolean all_keyframe;        /* TRUE when all samples are keyframes (no stss) */
254
  guint32 first_duration;       /* duration in timescale of first sample, used for figuring out
255
                                   the framerate */
256
257
258
  guint32 n_samples_moof;       /* sample count in a moof */
  guint64 duration_moof;        /* duration in timescale of a moof, used for figure out
                                 * the framerate of fragmented format stream */
259
  guint64 duration_last_moof;
260
261
262
263
264
265
266

  guint32 offset_in_sample;     /* Offset in the current sample, used for
                                 * streams which have got exceedingly big
                                 * sample size (such as 24s of raw audio).
                                 * Only used when max_buffer_size is non-NULL */
  guint32 max_buffer_size;      /* Maximum allowed size for output buffers.
                                 * Currently only set for raw audio streams*/
267

268
269
  /* if we use chunks or samples */
  gboolean sampled;
270
  guint padding;
271
272

  /* video info */
273
274
  gint width;
  gint height;
275
276
277
278
279
  /* aspect ratio */
  gint display_width;
  gint display_height;
  gint par_w;
  gint par_h;
280
281
282
  /* Numerator/denominator framerate */
  gint fps_n;
  gint fps_d;
283
  GstVideoColorimetry colorimetry;
284
285
  guint16 bits_per_sample;
  guint16 color_table_id;
286
  GstMemory *rgb8_palette;
287
288
  guint interlace_mode;
  guint field_order;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
289

290
  /* audio info */
291
292
  gdouble rate;
  gint n_channels;
293
294
295
296
  guint samples_per_packet;
  guint samples_per_frame;
  guint bytes_per_packet;
  guint bytes_per_sample;
297
  guint bytes_per_frame;
298
  guint compression;
299

300
301
302
303
304
  /* allocation */
  gboolean use_allocator;
  GstAllocator *allocator;
  GstAllocationParams params;

305
306
  gsize alignment;

307
308
  /* when a discontinuity is pending */
  gboolean discont;
309

310
311
312
  /* list of buffers to push first */
  GSList *buffers;

313
314
315
316
  /* if we need to clip this buffer. This is only needed for uncompressed
   * data */
  gboolean need_clip;

317
318
319
  /* buffer needs some custom processing, e.g. subtitles */
  gboolean need_process;

320
321
322
  /* current position */
  guint32 segment_index;
  guint32 sample_index;
323
  GstClockTime time_position;   /* in gst time */
324
  guint64 accumulated_base;
325

326
327
328
  /* the Gst segment we are processing out, used for clipping */
  GstSegment segment;

329
330
331
  /* quicktime segments */
  guint32 n_segments;
  QtDemuxSegment *segments;
332
  gboolean dummy_segment;
333
334
  guint32 from_sample;
  guint32 to_sample;
335
336

  gboolean sent_eos;
337
  GstTagList *stream_tags;
338
  gboolean send_global_tags;
339
340

  GstEvent *pending_event;
341
342
343
344
345
346
347
348
349

  GstByteReader stco;
  GstByteReader stsz;
  GstByteReader stsc;
  GstByteReader stts;
  GstByteReader stss;
  GstByteReader stps;
  GstByteReader ctts;

350
  gboolean chunks_are_samples;  /* TRUE means treat chunks as samples */
351
  gint64 stbl_index;
352
353
  /* stco */
  guint co_size;
354
355
356
357
358
359
  GstByteReader co_chunk;
  guint32 first_chunk;
  guint32 current_chunk;
  guint32 last_chunk;
  guint32 samples_per_chunk;
  guint32 stco_sample_index;
360
361
362
  /* stsz */
  guint32 sample_size;          /* 0 means variable sizes are stored in stsz */
  /* stsc */
363
  guint32 stsc_index;
364
  guint32 n_samples_per_chunk;
365
366
367
  guint32 stsc_chunk_index;
  guint32 stsc_sample_index;
  guint64 chunk_offset;
368
  /* stts */
369
370
  guint32 stts_index;
  guint32 stts_samples;
371
  guint32 n_sample_times;
372
  guint32 stts_sample_index;
373
  guint64 stts_time;
374
  guint32 stts_duration;
375
  /* stss */
376
  gboolean stss_present;
377
  guint32 n_sample_syncs;
378
  guint32 stss_index;
379
  /* stps */
380
  gboolean stps_present;
381
  guint32 n_sample_partial_syncs;
382
  guint32 stps_index;
383
384
385
  QtDemuxRandomAccessEntry *ra_entries;
  guint n_ra_entries;

386
387
  const QtDemuxRandomAccessEntry *pending_seek;

388
389
390
  /* ctts */
  gboolean ctts_present;
  guint32 n_composition_times;
391
392
393
394
  guint32 ctts_index;
  guint32 ctts_sample_index;
  guint32 ctts_count;
  gint32 ctts_soffset;
395

Nicolas Dufresne's avatar
Nicolas Dufresne committed
396
397
398
  /* cslg */
  guint32 cslg_shift;

399
400
401
402
403
  /* fragmented */
  gboolean parsed_trex;
  guint32 def_sample_duration;
  guint32 def_sample_size;
  guint32 def_sample_flags;
404
405

  gboolean disabled;
406
407
408
409

  /* stereoscopic video streams */
  GstVideoMultiviewMode multiview_mode;
  GstVideoMultiviewFlags multiview_flags;
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426

  /* protected streams */
  gboolean protected;
  guint32 protection_scheme_type;
  guint32 protection_scheme_version;
  gpointer protection_scheme_info;      /* specific to the protection scheme */
  GQueue protection_scheme_event_queue;
};

/* Contains properties and cryptographic info for a set of samples from a
 * track protected using Common Encryption (cenc) */
struct _QtDemuxCencSampleSetInfo
{
  GstStructure *default_properties;

  /* @crypto_info holds one GstStructure per sample */
  GPtrArray *crypto_info;
427
428
};

429
430
static const gchar *
qt_demux_state_string (enum QtDemuxState state)
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
431
{
432
433
434
435
436
437
438
439
440
441
442
443
444
  switch (state) {
    case QTDEMUX_STATE_INITIAL:
      return "<INITIAL>";
    case QTDEMUX_STATE_HEADER:
      return "<HEADER>";
    case QTDEMUX_STATE_MOVIE:
      return "<MOVIE>";
    case QTDEMUX_STATE_BUFFER_MDAT:
      return "<BUFFER_MDAT>";
    default:
      return "<UNKNOWN>";
  }
}
445

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
446
static GNode *qtdemux_tree_get_child_by_type (GNode * node, guint32 fourcc);
447
static GNode *qtdemux_tree_get_child_by_type_full (GNode * node,
448
    guint32 fourcc, GstByteReader * parser);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
449
static GNode *qtdemux_tree_get_sibling_by_type (GNode * node, guint32 fourcc);
450
451
static GNode *qtdemux_tree_get_sibling_by_type_full (GNode * node,
    guint32 fourcc, GstByteReader * parser);
452

453
454
static GstFlowReturn qtdemux_add_fragmented_samples (GstQTDemux * qtdemux);

David Schleef's avatar
David Schleef committed
455
static GstStaticPadTemplate gst_qtdemux_sink_template =
456
    GST_STATIC_PAD_TEMPLATE ("sink",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
457
    GST_PAD_SINK,
458
    GST_PAD_ALWAYS,
459
460
    GST_STATIC_CAPS ("video/quicktime; video/mj2; audio/x-m4a; "
        "application/x-3gp")
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
461
    );
David Schleef's avatar
David Schleef committed
462
463

static GstStaticPadTemplate gst_qtdemux_videosrc_template =
Wim Taymans's avatar
Wim Taymans committed
464
GST_STATIC_PAD_TEMPLATE ("video_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
465
466
467
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
David Schleef's avatar
David Schleef committed
468
469

static GstStaticPadTemplate gst_qtdemux_audiosrc_template =
Wim Taymans's avatar
Wim Taymans committed
470
GST_STATIC_PAD_TEMPLATE ("audio_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
471
472
473
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
Artyom Baginski's avatar
Artyom Baginski committed
474

475
static GstStaticPadTemplate gst_qtdemux_subsrc_template =
Wim Taymans's avatar
Wim Taymans committed
476
GST_STATIC_PAD_TEMPLATE ("subtitle_%u",
477
478
479
480
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
481
482
#define gst_qtdemux_parent_class parent_class
G_DEFINE_TYPE (GstQTDemux, gst_qtdemux, GST_TYPE_ELEMENT);
Artyom Baginski's avatar
Artyom Baginski committed
483

484
static void gst_qtdemux_dispose (GObject * object);
485

486
487
static guint32
gst_qtdemux_find_index_linear (GstQTDemux * qtdemux, QtDemuxStream * str,
488
    GstClockTime media_time);
489
490
491
492
static guint32
gst_qtdemux_find_index_for_given_media_offset_linear (GstQTDemux * qtdemux,
    QtDemuxStream * str, gint64 media_offset);

493
#if 0
494
495
static void gst_qtdemux_set_index (GstElement * element, GstIndex * index);
static GstIndex *gst_qtdemux_get_index (GstElement * element);
496
#endif
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
497
498
static GstStateChangeReturn gst_qtdemux_change_state (GstElement * element,
    GstStateChange transition);
Wim Taymans's avatar
Wim Taymans committed
499
static gboolean qtdemux_sink_activate (GstPad * sinkpad, GstObject * parent);
Wim Taymans's avatar
Wim Taymans committed
500
501
static gboolean qtdemux_sink_activate_mode (GstPad * sinkpad,
    GstObject * parent, GstPadMode mode, gboolean active);
502
503

static void gst_qtdemux_loop (GstPad * pad);
Wim Taymans's avatar
Wim Taymans committed
504
505
506
507
static GstFlowReturn gst_qtdemux_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * inbuf);
static gboolean gst_qtdemux_handle_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
Thiago Santos's avatar
Thiago Santos committed
508
static gboolean gst_qtdemux_setcaps (GstQTDemux * qtdemux, GstCaps * caps);
509
510
static gboolean gst_qtdemux_configure_stream (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
511
512
static GstFlowReturn gst_qtdemux_process_adapter (GstQTDemux * demux,
    gboolean force);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
513

514
515
static gboolean qtdemux_parse_moov (GstQTDemux * qtdemux,
    const guint8 * buffer, guint length);
516
static gboolean qtdemux_parse_node (GstQTDemux * qtdemux, GNode * node,
517
    const guint8 * buffer, guint length);
518
static gboolean qtdemux_parse_tree (GstQTDemux * qtdemux);
Thiago Santos's avatar
Thiago Santos committed
519
520
static void qtdemux_parse_udta (GstQTDemux * qtdemux, GstTagList * taglist,
    GNode * udta);
521

522
static void gst_qtdemux_handle_esds (GstQTDemux * qtdemux,
523
    QtDemuxStream * stream, GNode * esds, GstTagList * list);
524
525
static GstCaps *qtdemux_video_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
526
    gchar ** codec_name);
527
528
static GstCaps *qtdemux_audio_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data, int len,
529
    gchar ** codec_name);
530
static GstCaps *qtdemux_sub_caps (GstQTDemux * qtdemux,
531
532
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data,
    gchar ** codec_name);
533
534
535
536
static GstCaps *qtdemux_generic_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
    gchar ** codec_name);

537
538
static gboolean qtdemux_parse_samples (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 n);
539
static GstFlowReturn qtdemux_expose_streams (GstQTDemux * qtdemux);
540
541
static void gst_qtdemux_stream_free (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thiago Santos's avatar
Thiago Santos committed
542
543
static void gst_qtdemux_stream_clear (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Arnaud Vrac's avatar
Arnaud Vrac committed
544
static void gst_qtdemux_remove_stream (GstQTDemux * qtdemux, int index);
545
static GstFlowReturn qtdemux_prepare_streams (GstQTDemux * qtdemux);
546
547
static void qtdemux_do_allocation (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
548
549
550
551
552
static gboolean gst_qtdemux_activate_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 seg_idx, GstClockTime offset);
static gboolean gst_qtdemux_stream_update_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, gint seg_idx, GstClockTime offset,
    GstClockTime * _start, GstClockTime * _stop);
553
554
static void gst_qtdemux_send_gap_for_segment (GstQTDemux * demux,
    QtDemuxStream * stream, gint segment_index, GstClockTime pos);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
555

556
static gboolean qtdemux_pull_mfro_mfra (GstQTDemux * qtdemux);
557
static void check_update_duration (GstQTDemux * qtdemux, GstClockTime duration);
558

559
560
561
562
static gchar *qtdemux_uuid_bytes_to_string (gconstpointer uuid_bytes);

static GstStructure *qtdemux_get_cenc_sample_properties (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint sample_index);
563
564
static void gst_qtdemux_append_protection_system_id (GstQTDemux * qtdemux,
    const gchar * id);
565
static void qtdemux_gst_structure_free (GstStructure * gststructure);
566

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
567
568
static void
gst_qtdemux_class_init (GstQTDemuxClass * klass)
Artyom Baginski's avatar
Artyom Baginski committed
569
570
571
572
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
573
574
  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
Artyom Baginski's avatar
Artyom Baginski committed
575

576
  parent_class = g_type_class_peek_parent (klass);
577

578
579
  gobject_class->dispose = gst_qtdemux_dispose;

580
  gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state);
581
#if 0
582
583
  gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index);
  gstelement_class->get_index = GST_DEBUG_FUNCPTR (gst_qtdemux_get_index);
584
#endif
585
586

  gst_tag_register_musicbrainz_tags ();
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
587

588
589
590
591
592
593
594
595
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_sink_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_videosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_audiosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_subsrc_template);
596
  gst_element_class_set_static_metadata (gstelement_class, "QuickTime demuxer",
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
597
598
599
600
601
602
      "Codec/Demuxer",
      "Demultiplex a QuickTime file into audio and video streams",
      "David Schleef <ds@schleef.org>, Wim Taymans <wim@fluendo.com>");

  GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin");

Artyom Baginski's avatar
Artyom Baginski committed
603
604
}

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
605
static void
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
606
gst_qtdemux_init (GstQTDemux * qtdemux)
Artyom Baginski's avatar
Artyom Baginski committed
607
{
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
608
  qtdemux->sinkpad =
609
      gst_pad_new_from_static_template (&gst_qtdemux_sink_template, "sink");
610
  gst_pad_set_activate_function (qtdemux->sinkpad, qtdemux_sink_activate);
Wim Taymans's avatar
Wim Taymans committed
611
612
  gst_pad_set_activatemode_function (qtdemux->sinkpad,
      qtdemux_sink_activate_mode);
613
614
  gst_pad_set_chain_function (qtdemux->sinkpad, gst_qtdemux_chain);
  gst_pad_set_event_function (qtdemux->sinkpad, gst_qtdemux_handle_sink_event);
615
  gst_element_add_pad (GST_ELEMENT_CAST (qtdemux), qtdemux->sinkpad);
616

617
618
  qtdemux->state = QTDEMUX_STATE_INITIAL;
  qtdemux->pullbased = FALSE;
619
  qtdemux->posted_redirect = FALSE;
620
621
622
  qtdemux->neededbytes = 16;
  qtdemux->todrop = 0;
  qtdemux->adapter = gst_adapter_new ();
623
  qtdemux->offset = 0;
624
  qtdemux->first_mdat = -1;
625
  qtdemux->got_moov = FALSE;
626
  qtdemux->mdatoffset = -1;
627
  qtdemux->mdatbuffer = NULL;
628
  qtdemux->restoredata_buffer = NULL;
629
  qtdemux->restoredata_offset = -1;
630
  qtdemux->fragment_start = -1;
631
  qtdemux->fragment_start_offset = -1;
632
633
634
  qtdemux->media_caps = NULL;
  qtdemux->exposed = FALSE;
  qtdemux->mss_mode = FALSE;
635
  qtdemux->pending_newsegment = NULL;
636
  qtdemux->upstream_format_is_time = FALSE;
637
638
  qtdemux->have_group_id = FALSE;
  qtdemux->group_id = G_MAXUINT;
639
640
641
  qtdemux->cenc_aux_info_offset = 0;
  qtdemux->cenc_aux_info_sizes = NULL;
  qtdemux->cenc_aux_sample_count = 0;
642
643
  qtdemux->protection_system_ids = NULL;
  g_queue_init (&qtdemux->protection_event_queue);
644
  gst_segment_init (&qtdemux->segment, GST_FORMAT_TIME);
645
646
  qtdemux->tag_list = gst_tag_list_new_empty ();
  gst_tag_list_set_scope (qtdemux->tag_list, GST_TAG_SCOPE_GLOBAL);
Thiago Santos's avatar
Thiago Santos committed
647
  qtdemux->flowcombiner = gst_flow_combiner_new ();
Wim Taymans's avatar
Wim Taymans committed
648
649

  GST_OBJECT_FLAG_SET (qtdemux, GST_ELEMENT_FLAG_INDEXABLE);
Artyom Baginski's avatar
Artyom Baginski committed
650
651
}

652
653
654
655
656
657
658
659
660
static void
gst_qtdemux_dispose (GObject * object)
{
  GstQTDemux *qtdemux = GST_QTDEMUX (object);

  if (qtdemux->adapter) {
    g_object_unref (G_OBJECT (qtdemux->adapter));
    qtdemux->adapter = NULL;
  }
661
  gst_tag_list_unref (qtdemux->tag_list);
Thiago Santos's avatar
Thiago Santos committed
662
  gst_flow_combiner_free (qtdemux->flowcombiner);
663
664
665
  g_queue_foreach (&qtdemux->protection_event_queue, (GFunc) gst_event_unref,
      NULL);
  g_queue_clear (&qtdemux->protection_event_queue);
666

667
668
669
  g_free (qtdemux->cenc_aux_info_sizes);
  qtdemux->cenc_aux_info_sizes = NULL;

670
  G_OBJECT_CLASS (parent_class)->dispose (object);
671
672
}

673
674
675
676
static void
gst_qtdemux_post_no_playable_stream_error (GstQTDemux * qtdemux)
{
  if (qtdemux->posted_redirect) {
677
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
678
679
680
        (_("This file contains no playable streams.")),
        ("no known streams found, a redirect message has been posted"));
  } else {
681
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
682
683
684
685
686
        (_("This file contains no playable streams.")),
        ("no known streams found"));
  }
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
687
688
689
static GstBuffer *
_gst_buffer_new_wrapped (gpointer mem, gsize size, GFreeFunc free_func)
{
Wim Taymans's avatar
Wim Taymans committed
690
691
  return gst_buffer_new_wrapped_full (free_func ? 0 : GST_MEMORY_FLAG_READONLY,
      mem, size, 0, size, mem, free_func);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
692
693
}

694
695
696
697
698
static GstFlowReturn
gst_qtdemux_pull_atom (GstQTDemux * qtdemux, guint64 offset, guint64 size,
    GstBuffer ** buf)
{
  GstFlowReturn flow;
Wim Taymans's avatar
Wim Taymans committed
699
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
700
  gsize bsize;
701

702
  if (G_UNLIKELY (size == 0)) {
703
704
705
706
707
708
709
    GstFlowReturn ret;
    GstBuffer *tmp = NULL;

    ret = gst_qtdemux_pull_atom (qtdemux, offset, sizeof (guint32), &tmp);
    if (ret != GST_FLOW_OK)
      return ret;

Wim Taymans's avatar
Wim Taymans committed
710
711
    gst_buffer_map (tmp, &map, GST_MAP_READ);
    size = QT_UINT32 (map.data);
712
    GST_DEBUG_OBJECT (qtdemux, "size 0x%08" G_GINT64_MODIFIER "x", size);
713

Wim Taymans's avatar
Wim Taymans committed
714
    gst_buffer_unmap (tmp, &map);
715
716
717
    gst_buffer_unref (tmp);
  }

718
719
  /* Sanity check: catch bogus sizes (fuzzed/broken files) */
  if (G_UNLIKELY (size > QTDEMUX_MAX_ATOM_SIZE)) {
720
721
722
723
724
    if (qtdemux->state != QTDEMUX_STATE_MOVIE && qtdemux->got_moov) {
      /* we're pulling header but already got most interesting bits,
       * so never mind the rest (e.g. tags) (that much) */
      GST_WARNING_OBJECT (qtdemux, "atom has bogus size %" G_GUINT64_FORMAT,
          size);
725
      return GST_FLOW_EOS;
726
727
728
729
730
731
    } else {
      GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
          (_("This file is invalid and cannot be played.")),
          ("atom has bogus size %" G_GUINT64_FORMAT, size));
      return GST_FLOW_ERROR;
    }
732
733
734
735
736
737
738
  }

  flow = gst_pad_pull_range (qtdemux->sinkpad, offset, size, buf);

  if (G_UNLIKELY (flow != GST_FLOW_OK))
    return flow;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
739
  bsize = gst_buffer_get_size (*buf);
740
  /* Catch short reads - we don't want any partial atoms */
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
741
  if (G_UNLIKELY (bsize < size)) {
742
743
    GST_WARNING_OBJECT (qtdemux,
        "short read: %" G_GSIZE_FORMAT " < %" G_GUINT64_FORMAT, bsize, size);
744
745
    gst_buffer_unref (*buf);
    *buf = NULL;
746
    return GST_FLOW_EOS;
747
748
749
750
751
  }

  return flow;
}

752
#if 1
753
static gboolean
754
755
756
gst_qtdemux_src_convert (GstQTDemux * qtdemux, GstPad * pad,
    GstFormat src_format, gint64 src_value, GstFormat dest_format,
    gint64 * dest_value)
757
758
{
  gboolean res = TRUE;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
759
  QtDemuxStream *stream = gst_pad_get_element_private (pad);
760
  gint32 index;
761

Andy Wingo's avatar
Andy Wingo committed
762
763
764
765
  if (stream->subtype != FOURCC_vide) {
    res = FALSE;
    goto done;
  }
766
767
768

  switch (src_format) {
    case GST_FORMAT_TIME:
769
770
771
      switch (dest_format) {
        case GST_FORMAT_BYTES:{
          index = gst_qtdemux_find_index_linear (qtdemux, stream, src_value);
772
773
774
775
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
776
777
778
779
780
781

          *dest_value = stream->samples[index].offset;

          GST_DEBUG_OBJECT (qtdemux, "Format Conversion Time->Offset :%"
              GST_TIME_FORMAT "->%" G_GUINT64_FORMAT,
              GST_TIME_ARGS (src_value), *dest_value);
782
          break;
783
        }
784
785
786
        default:
          res = FALSE;
          break;
787
788
789
      }
      break;
    case GST_FORMAT_BYTES:
790
791
792
793
794
795
      switch (dest_format) {
        case GST_FORMAT_TIME:{
          index =
              gst_qtdemux_find_index_for_given_media_offset_linear (qtdemux,
              stream, src_value);

796
797
798
799
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
800
801

          *dest_value =
802
803
804
805
806
              QTSTREAMTIME_TO_GSTTIME (stream,
              stream->samples[index].timestamp);
          GST_DEBUG_OBJECT (qtdemux,
              "Format Conversion Offset->Time :%" G_GUINT64_FORMAT "->%"
              GST_TIME_FORMAT, src_value, GST_TIME_ARGS (*dest_value));
807
          break;
808
        }
809
810
811
        default:
          res = FALSE;
          break;
812
813
814
815
      }
      break;
    default:
      res = FALSE;
816
      break;
817
818
  }

Andy Wingo's avatar
Andy Wingo committed
819
done:
820
821
  return res;
}
822
#endif
823

824
static gboolean
825
gst_qtdemux_get_duration (GstQTDemux * qtdemux, GstClockTime * duration)
826
{
827
  gboolean res = FALSE;
828
829
830

  *duration = GST_CLOCK_TIME_NONE;

831
832
833
834
835
836
  if (qtdemux->duration != 0 &&
      qtdemux->duration != G_MAXINT64 && qtdemux->timescale != 0) {
    *duration = QTTIME_TO_GSTTIME (qtdemux, qtdemux->duration);
    res = TRUE;
  } else {
    *duration = GST_CLOCK_TIME_NONE;
837
  }
838

839
840
841
  return res;
}

842
static gboolean
Wim Taymans's avatar
Wim Taymans committed
843
844
gst_qtdemux_handle_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
845
{
846
  gboolean res = FALSE;
Wim Taymans's avatar
Wim Taymans committed
847
  GstQTDemux *qtdemux = GST_QTDEMUX (parent);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
848

849
850
  GST_LOG_OBJECT (pad, "%s query", GST_QUERY_TYPE_NAME (query));

851
  switch (GST_QUERY_TYPE (query)) {
852
853
854
855
856
857
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME
          && GST_CLOCK_TIME_IS_VALID (qtdemux->segment.position)) {
858
        gst_query_set_position (query, GST_FORMAT_TIME,
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
859
            qtdemux->segment.position);
Wim Taymans's avatar
Wim Taymans committed
860
861
        res = TRUE;
      }
862
    }
Wim Taymans's avatar
Wim Taymans committed
863
      break;
864
865
    case GST_QUERY_DURATION:{
      GstFormat fmt;
866

867
868
      gst_query_parse_duration (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME) {
869
870
871
        /* First try to query upstream */
        res = gst_pad_query_default (pad, parent, query);
        if (!res) {
872
          GstClockTime duration;
873
          if (gst_qtdemux_get_duration (qtdemux, &duration) && duration > 0) {
874
875
876
            gst_query_set_duration (query, GST_FORMAT_TIME, duration);
            res = TRUE;
          }
877
878
879
        }
      }
      break;
880
    }
881
882
    case GST_QUERY_CONVERT:{
      GstFormat src_fmt, dest_fmt;
883
      gint64 src_value, dest_value = 0;
884
885
886

      gst_query_parse_convert (query, &src_fmt, &src_value, &dest_fmt, NULL);

887
      res = gst_qtdemux_src_convert (qtdemux, pad,
888
889
890
891
892
893
894
895
896
897
898
          src_fmt, src_value, dest_fmt, &dest_value);
      if (res) {
        gst_query_set_convert (query, src_fmt, src_value, dest_fmt, dest_value);
        res = TRUE;
      }
      break;
    }
    case GST_QUERY_FORMATS:
      gst_query_set_formats (query, 2, GST_FORMAT_TIME, GST_FORMAT_BYTES);
      res = TRUE;
      break;
899
900
    case GST_QUERY_SEEKING:{
      GstFormat fmt;
901
      gboolean seekable;
902

Thiago Santos's avatar
Thiago Santos committed
903
904
905
906
907
908
      /* try upstream first */
      res = gst_pad_query_default (pad, parent, query);

      if (!res) {
        gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
        if (fmt == GST_FORMAT_TIME) {
909
          GstClockTime duration;
Thiago Santos's avatar
Thiago Santos committed
910
911
912
913
914
915
916
917
918
919
920
921
922
923

          gst_qtdemux_get_duration (qtdemux, &duration);
          seekable = TRUE;
          if (!qtdemux->pullbased) {
            GstQuery *q;

            /* we might be able with help from upstream */
            seekable = FALSE;
            q = gst_query_new_seeking (GST_FORMAT_BYTES);
            if (gst_pad_peer_query (qtdemux->sinkpad, q)) {
              gst_query_parse_seeking (q, &fmt, &seekable, NULL, NULL);
              GST_LOG_OBJECT (qtdemux, "upstream BYTE seekable %d", seekable);
            }
            gst_query_unref (q);
924
          }
Thiago Santos's avatar
Thiago Santos committed
925
926
          gst_query_set_seeking (query, GST_FORMAT_TIME, seekable, 0, duration);
          res = TRUE;
927
        }
928
      }
929
      break;
930
    }
Wim Taymans's avatar
Wim Taymans committed
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
    case GST_QUERY_SEGMENT:
    {
      GstFormat format;
      gint64 start, stop;

      format = qtdemux->segment.format;

      start =
          gst_segment_to_stream_time (&qtdemux->segment, format,
          qtdemux->segment.start);
      if ((stop = qtdemux->segment.stop) == -1)
        stop = qtdemux->segment.duration;
      else
        stop = gst_segment_to_stream_time (&qtdemux->segment, format, stop);

      gst_query_set_segment (query, qtdemux->segment.rate, format, start, stop);
      res = TRUE;
      break;
    }
950
    default:
Wim Taymans's avatar
Wim Taymans committed
951
      res = gst_pad_query_default (pad, parent, query);
952
953
954
955
956
957
      break;
  }

  return res;
}

958
959
960
961
962
963
964
static void
gst_qtdemux_push_tags (GstQTDemux * qtdemux, QtDemuxStream * stream)
{
  if (G_LIKELY (stream->pad)) {
    GST_DEBUG_OBJECT (qtdemux, "Checking pad %s:%s for tags",
        GST_DEBUG_PAD_NAME (stream->pad));

965
    if (!gst_tag_list_is_empty (stream->stream_tags)) {
966
      GST_DEBUG_OBJECT (qtdemux, "Sending tags %" GST_PTR_FORMAT,
967
          stream->stream_tags);
968
      gst_pad_push_event (stream->pad,
969
          gst_event_new_tag (gst_tag_list_ref (stream->stream_tags)));
970
971
    }

972
    if (G_UNLIKELY (stream->send_global_tags)) {
973
974
975
      GST_DEBUG_OBJECT (qtdemux, "Sending global tags %" GST_PTR_FORMAT,
          qtdemux->tag_list);
      gst_pad_push_event (stream->pad,
976
          gst_event_new_tag (gst_tag_list_ref (qtdemux->tag_list)));
977
978
979
980
981
      stream->send_global_tags = FALSE;
    }
  }
}

982
/* push event on all source pads; takes ownership of the event */
983
static void
984
gst_qtdemux_push_event (GstQTDemux * qtdemux, GstEvent * event)
985
986
{
  guint n;
987
  gboolean has_valid_stream = FALSE;
988
  GstEventType etype = GST_EVENT_TYPE (event);
989
990
991
992
993

  GST_DEBUG_OBJECT (qtdemux, "pushing %s event on all source pads",
      GST_EVENT_TYPE_NAME (event));

  for (n = 0; n <