qtdemux.c 463 KB
Newer Older
Artyom Baginski's avatar
Artyom Baginski committed
1
2
/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
3
 * Copyright (C) <2003> David A. Schleef <ds@schleef.org>
4
 * Copyright (C) <2006> Wim Taymans <wim@fluendo.com>
5
 * Copyright (C) <2007> Julien Moutte <julien@fluendo.com>
6
 * Copyright (C) <2009> Tim-Philipp Müller <tim centricular net>
7
 * Copyright (C) <2009> STEricsson <benjamin.gaignard@stericsson.com>
8
 * Copyright (C) <2013> Sreerenj Balachandran <sreerenj.balachandran@intel.com>
9
10
 * Copyright (C) <2013> Intel Corporation
 * Copyright (C) <2014> Centricular Ltd
11
 * Copyright (C) <2015> YouView TV Ltd.
12
 * Copyright (C) <2016> British Broadcasting Corporation
13
 *
Artyom Baginski's avatar
Artyom Baginski committed
14
15
16
17
18
19
20
21
22
23
24
25
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
26
27
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
Artyom Baginski's avatar
Artyom Baginski committed
28
29
 */

30
31
32
33
/**
 * SECTION:element-qtdemux
 *
 * Demuxes a .mov file into raw or compressed audio and/or video streams.
34
 *
35
36
 * This element supports both push and pull-based scheduling, depending on the
 * capabilities of the upstream elements.
37
38
 *
 * <refsect2>
39
 * <title>Example launch line</title>
40
 * |[
41
 * gst-launch-1.0 filesrc location=test.mov ! qtdemux name=demux  demux.audio_0 ! queue ! decodebin ! audioconvert ! audioresample ! autoaudiosink   demux.video_0 ! queue ! decodebin ! videoconvert ! videoscale ! autovideosink
42
 * ]| Play (parse and decode) a .mov file and try to output it to
43
44
45
46
47
48
 * an automatically detected soundcard and videosink. If the MOV file contains
 * compressed audio or video data, this will only work if you have the
 * right decoder elements/plugins installed.
 * </refsect2>
 */

49
50
51
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
52
53
54

#include "gst/gst-i18n-plugin.h"

55
#include <glib/gprintf.h>
56
#include <gst/tag/tag.h>
Wim Taymans's avatar
Wim Taymans committed
57
#include <gst/audio/audio.h>
58
#include <gst/video/video.h>
59

60
#include "qtatomparser.h"
61
62
#include "qtdemux_types.h"
#include "qtdemux_dump.h"
63
#include "fourcc.h"
64
#include "descriptors.h"
65
#include "qtdemux_lang.h"
66
#include "qtdemux.h"
67
#include "qtpalette.h"
68

69
70
71
#include "gst/riff/riff-media.h"
#include "gst/riff/riff-read.h"

72
73
#include <gst/pbutils/pbutils.h>

74
#include <stdio.h>
75
#include <stdlib.h>
Artyom Baginski's avatar
Artyom Baginski committed
76
#include <string.h>
77

78
79
80
#include <math.h>
#include <gst/math-compat.h>

81
82
83
#ifdef HAVE_ZLIB
# include <zlib.h>
#endif
84

85
86
87
/* max. size considered 'sane' for non-mdat atoms */
#define QTDEMUX_MAX_ATOM_SIZE (25*1024*1024)

88
89
90
/* if the sample index is larger than this, something is likely wrong */
#define QTDEMUX_MAX_SAMPLE_INDEX_SIZE (50*1024*1024)

91
92
93
94
95
96
/* For converting qt creation times to unix epoch times */
#define QTDEMUX_SECONDS_PER_DAY (60 * 60 * 24)
#define QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970 17
#define QTDEMUX_SECONDS_FROM_1904_TO_1970 (((1970 - 1904) * (guint64) 365 + \
    QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970) * QTDEMUX_SECONDS_PER_DAY)

97
98
#define QTDEMUX_TREE_NODE_FOURCC(n) (QT_FOURCC(((guint8 *) (n)->data) + 4))

99
#define STREAM_IS_EOS(s) (s->time_position == GST_CLOCK_TIME_NONE)
100

101
102
#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )

103
GST_DEBUG_CATEGORY (qtdemux_debug);
104

105
typedef struct _QtDemuxSegment QtDemuxSegment;
106
typedef struct _QtDemuxSample QtDemuxSample;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
107

108
109
typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
110
111
struct _QtDemuxSample
{
112
  guint32 size;
113
  gint32 pts_offset;            /* Add this value to timestamp to get the pts */
114
  guint64 offset;
115
116
  guint64 timestamp;            /* DTS In mov time */
  guint32 duration;             /* In mov time */
117
  gboolean keyframe;            /* TRUE when this packet is a keyframe */
118
119
};

120
121
122
123
124
125
126
/* Macros for converting to/from timescale */
#define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale))
#define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND))

#define QTTIME_TO_GSTTIME(qtdemux, value) (gst_util_uint64_scale((value), GST_SECOND, (qtdemux)->timescale))
#define GSTTIME_TO_QTTIME(qtdemux, value) (gst_util_uint64_scale((value), (qtdemux)->timescale, GST_SECOND))

127
/* timestamp is the DTS */
128
#define QTSAMPLE_DTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp))
129
/* timestamp + offset + cslg_shift is the outgoing PTS */
Nicolas Dufresne's avatar
Nicolas Dufresne committed
130
#define QTSAMPLE_PTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (stream)->cslg_shift + (sample)->pts_offset))
131
132
/* timestamp + offset is the PTS used for internal seek calcuations */
#define QTSAMPLE_PTS_NO_CSLG(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (sample)->pts_offset))
133
/* timestamp + duration - dts is the duration */
134
#define QTSAMPLE_DUR_DTS(stream, sample, dts) (QTSTREAMTIME_TO_GSTTIME ((stream), (sample)->timestamp + (sample)->duration) - (dts))
135
136
137

#define QTSAMPLE_KEYFRAME(stream,sample) ((stream)->all_keyframe || (sample)->keyframe)

138
139
140
141
142
143
144
145
146
147
148
/*
 * Quicktime has tracks and segments. A track is a continuous piece of
 * multimedia content. The track is not always played from start to finish but
 * instead, pieces of the track are 'cut out' and played in sequence. This is
 * what the segments do.
 *
 * Inside the track we have keyframes (K) and delta frames. The track has its
 * own timing, which starts from 0 and extends to end. The position in the track
 * is called the media_time.
 *
 * The segments now describe the pieces that should be played from this track
149
 * and are basically tuples of media_time/duration/rate entries. We can have
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 * multiple segments and they are all played after one another. An example:
 *
 * segment 1: media_time: 1 second, duration: 1 second, rate 1
 * segment 2: media_time: 3 second, duration: 2 second, rate 2
 *
 * To correctly play back this track, one must play: 1 second of media starting
 * from media_time 1 followed by 2 seconds of media starting from media_time 3
 * at a rate of 2.
 *
 * Each of the segments will be played at a specific time, the first segment at
 * time 0, the second one after the duration of the first one, etc.. Note that
 * the time in resulting playback is not identical to the media_time of the
 * track anymore.
 *
 * Visually, assuming the track has 4 second of media_time:
 *
 *                (a)                   (b)          (c)              (d)
 *         .-----------------------------------------------------------.
 * track:  | K.....K.........K........K.......K.......K...........K... |
 *         '-----------------------------------------------------------'
Stefan Kost's avatar
Stefan Kost committed
170
 *         0              1              2              3              4
171
172
173
174
175
176
 *           .------------^              ^   .----------^              ^
 *          /              .-------------'  /       .------------------'
 *         /              /          .-----'       /
 *         .--------------.         .--------------.
 *         | segment 1    |         | segment 2    |
 *         '--------------'         '--------------'
Stefan Kost's avatar
Stefan Kost committed
177
 *
178
 * The challenge here is to cut out the right pieces of the track for each of
179
180
 * the playback segments. This fortunately can easily be done with the SEGMENT
 * events of GStreamer.
181
182
183
184
185
186
187
188
 *
 * For playback of segment 1, we need to provide the decoder with the keyframe
 * (a), in the above figure, but we must instruct it only to output the decoded
 * data between second 1 and 2. We do this with a SEGMENT event for 1 to 2, time
 * position set to the time of the segment: 0.
 *
 * We then proceed to push data from keyframe (a) to frame (b). The decoder
 * decodes but clips all before media_time 1.
Stefan Kost's avatar
Stefan Kost committed
189
 *
190
191
192
193
194
195
 * After finishing a segment, we push out a new SEGMENT event with the clipping
 * boundaries of the new data.
 *
 * This is a good usecase for the GStreamer accumulated SEGMENT events.
 */

196
197
198
struct _QtDemuxSegment
{
  /* global time and duration, all gst time */
199
200
201
  GstClockTime time;
  GstClockTime stop_time;
  GstClockTime duration;
202
  /* media time of trak, all gst time */
203
204
  GstClockTime media_start;
  GstClockTime media_stop;
205
  gdouble rate;
206
207
  /* Media start time in trak timescale units */
  guint32 trak_media_start;
208
209
};

210
211
#define QTSEGMENT_IS_EMPTY(s) ((s)->media_start == GST_CLOCK_TIME_NONE)

212
213
214
215
216
217
218
/* Used with fragmented MP4 files (mfra atom) */
typedef struct
{
  GstClockTime ts;
  guint64 moof_offset;
} QtDemuxRandomAccessEntry;

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
typedef struct _QtDemuxStreamStsdEntry
{
  GstCaps *caps;
  guint32 fourcc;
  gboolean sparse;

  /* video info */
  gint width;
  gint height;
  gint par_w;
  gint par_h;
  /* Numerator/denominator framerate */
  gint fps_n;
  gint fps_d;
  GstVideoColorimetry colorimetry;
  guint16 bits_per_sample;
  guint16 color_table_id;
  GstMemory *rgb8_palette;
  guint interlace_mode;
  guint field_order;

  /* audio info */
  gdouble rate;
  gint n_channels;
  guint samples_per_packet;
  guint samples_per_frame;
  guint bytes_per_packet;
  guint bytes_per_sample;
  guint bytes_per_frame;
  guint compression;

  /* if we use chunks or samples */
  gboolean sampled;
  guint padding;

} QtDemuxStreamStsdEntry;

#define CUR_STREAM(s) (&((s)->stsd_entries[(s)->cur_stsd_entry_index]))

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
258
259
struct _QtDemuxStream
{
260
261
  GstPad *pad;

262
263
264
265
  QtDemuxStreamStsdEntry *stsd_entries;
  guint stsd_entries_length;
  guint cur_stsd_entry_index;

266
  /* stream type */
267
268
  guint32 subtype;

269
270
271
272
  gboolean new_caps;            /* If TRUE, caps need to be generated (by
                                 * calling _configure_stream()) This happens
                                 * for MSS and fragmented streams */

273
  gboolean new_stream;          /* signals that a stream_start is required */
274
275
276
277
  gboolean on_keyframe;         /* if this stream last pushed buffer was a
                                 * keyframe. This is important to identify
                                 * where to stop pushing buffers after a
                                 * segment stop time */
278

279
280
281
  /* if the stream has a redirect URI in its headers, we store it here */
  gchar *redirect_uri;

282
283
284
  /* track id */
  guint track_id;

285
  /* duration/scale */
286
  guint64 duration;             /* in timescale units */
287
  guint32 timescale;
288

289
  /* language */
290
  gchar lang_id[4];             /* ISO 639-2T language code */
291

292
293
294
295
  /* our samples */
  guint32 n_samples;
  QtDemuxSample *samples;
  gboolean all_keyframe;        /* TRUE when all samples are keyframes (no stss) */
296
  guint32 first_duration;       /* duration in timescale of first sample, used for figuring out
297
                                   the framerate */
298
299
300
  guint32 n_samples_moof;       /* sample count in a moof */
  guint64 duration_moof;        /* duration in timescale of a moof, used for figure out
                                 * the framerate of fragmented format stream */
301
  guint64 duration_last_moof;
302
303
304
305
306
307
308

  guint32 offset_in_sample;     /* Offset in the current sample, used for
                                 * streams which have got exceedingly big
                                 * sample size (such as 24s of raw audio).
                                 * Only used when max_buffer_size is non-NULL */
  guint32 max_buffer_size;      /* Maximum allowed size for output buffers.
                                 * Currently only set for raw audio streams*/
309

310
  /* video info */
311
312
313
  /* aspect ratio */
  gint display_width;
  gint display_height;
314

315
316
317
318
319
  /* allocation */
  gboolean use_allocator;
  GstAllocator *allocator;
  GstAllocationParams params;

320
321
  gsize alignment;

322
323
  /* when a discontinuity is pending */
  gboolean discont;
324

325
326
327
  /* list of buffers to push first */
  GSList *buffers;

328
329
330
331
  /* if we need to clip this buffer. This is only needed for uncompressed
   * data */
  gboolean need_clip;

332
333
334
  /* buffer needs some custom processing, e.g. subtitles */
  gboolean need_process;

335
336
337
  /* current position */
  guint32 segment_index;
  guint32 sample_index;
338
  GstClockTime time_position;   /* in gst time */
339
  guint64 accumulated_base;
340

341
342
343
  /* the Gst segment we are processing out, used for clipping */
  GstSegment segment;

344
345
346
  /* quicktime segments */
  guint32 n_segments;
  QtDemuxSegment *segments;
347
  gboolean dummy_segment;
348
349
  guint32 from_sample;
  guint32 to_sample;
350
351

  gboolean sent_eos;
352
  GstTagList *stream_tags;
353
  gboolean send_global_tags;
354
355

  GstEvent *pending_event;
356
357
358
359
360
361
362
363
364

  GstByteReader stco;
  GstByteReader stsz;
  GstByteReader stsc;
  GstByteReader stts;
  GstByteReader stss;
  GstByteReader stps;
  GstByteReader ctts;

365
  gboolean chunks_are_samples;  /* TRUE means treat chunks as samples */
366
  gint64 stbl_index;
367
368
  /* stco */
  guint co_size;
369
370
371
372
373
  GstByteReader co_chunk;
  guint32 first_chunk;
  guint32 current_chunk;
  guint32 last_chunk;
  guint32 samples_per_chunk;
374
  guint32 stsd_sample_description_id;
375
  guint32 stco_sample_index;
376
377
378
  /* stsz */
  guint32 sample_size;          /* 0 means variable sizes are stored in stsz */
  /* stsc */
379
  guint32 stsc_index;
380
  guint32 n_samples_per_chunk;
381
382
383
  guint32 stsc_chunk_index;
  guint32 stsc_sample_index;
  guint64 chunk_offset;
384
  /* stts */
385
386
  guint32 stts_index;
  guint32 stts_samples;
387
  guint32 n_sample_times;
388
  guint32 stts_sample_index;
389
  guint64 stts_time;
390
  guint32 stts_duration;
391
  /* stss */
392
  gboolean stss_present;
393
  guint32 n_sample_syncs;
394
  guint32 stss_index;
395
  /* stps */
396
  gboolean stps_present;
397
  guint32 n_sample_partial_syncs;
398
  guint32 stps_index;
399
400
401
  QtDemuxRandomAccessEntry *ra_entries;
  guint n_ra_entries;

402
403
  const QtDemuxRandomAccessEntry *pending_seek;

404
405
406
  /* ctts */
  gboolean ctts_present;
  guint32 n_composition_times;
407
408
409
410
  guint32 ctts_index;
  guint32 ctts_sample_index;
  guint32 ctts_count;
  gint32 ctts_soffset;
411

Nicolas Dufresne's avatar
Nicolas Dufresne committed
412
413
414
  /* cslg */
  guint32 cslg_shift;

415
416
  /* fragmented */
  gboolean parsed_trex;
417
  guint32 def_sample_description_index; /* index is 1-based */
418
419
420
  guint32 def_sample_duration;
  guint32 def_sample_size;
  guint32 def_sample_flags;
421
422

  gboolean disabled;
423
424
425
426

  /* stereoscopic video streams */
  GstVideoMultiviewMode multiview_mode;
  GstVideoMultiviewFlags multiview_flags;
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443

  /* protected streams */
  gboolean protected;
  guint32 protection_scheme_type;
  guint32 protection_scheme_version;
  gpointer protection_scheme_info;      /* specific to the protection scheme */
  GQueue protection_scheme_event_queue;
};

/* Contains properties and cryptographic info for a set of samples from a
 * track protected using Common Encryption (cenc) */
struct _QtDemuxCencSampleSetInfo
{
  GstStructure *default_properties;

  /* @crypto_info holds one GstStructure per sample */
  GPtrArray *crypto_info;
444
445
};

446
447
static const gchar *
qt_demux_state_string (enum QtDemuxState state)
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
448
{
449
450
451
452
453
454
455
456
457
458
459
460
461
  switch (state) {
    case QTDEMUX_STATE_INITIAL:
      return "<INITIAL>";
    case QTDEMUX_STATE_HEADER:
      return "<HEADER>";
    case QTDEMUX_STATE_MOVIE:
      return "<MOVIE>";
    case QTDEMUX_STATE_BUFFER_MDAT:
      return "<BUFFER_MDAT>";
    default:
      return "<UNKNOWN>";
  }
}
462

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
463
static GNode *qtdemux_tree_get_child_by_type (GNode * node, guint32 fourcc);
464
static GNode *qtdemux_tree_get_child_by_type_full (GNode * node,
465
    guint32 fourcc, GstByteReader * parser);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
466
static GNode *qtdemux_tree_get_sibling_by_type (GNode * node, guint32 fourcc);
467
468
static GNode *qtdemux_tree_get_sibling_by_type_full (GNode * node,
    guint32 fourcc, GstByteReader * parser);
469

470
471
static GstFlowReturn qtdemux_add_fragmented_samples (GstQTDemux * qtdemux);

David Schleef's avatar
David Schleef committed
472
static GstStaticPadTemplate gst_qtdemux_sink_template =
473
    GST_STATIC_PAD_TEMPLATE ("sink",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
474
    GST_PAD_SINK,
475
    GST_PAD_ALWAYS,
476
477
    GST_STATIC_CAPS ("video/quicktime; video/mj2; audio/x-m4a; "
        "application/x-3gp")
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
478
    );
David Schleef's avatar
David Schleef committed
479
480

static GstStaticPadTemplate gst_qtdemux_videosrc_template =
Wim Taymans's avatar
Wim Taymans committed
481
GST_STATIC_PAD_TEMPLATE ("video_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
482
483
484
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
David Schleef's avatar
David Schleef committed
485
486

static GstStaticPadTemplate gst_qtdemux_audiosrc_template =
Wim Taymans's avatar
Wim Taymans committed
487
GST_STATIC_PAD_TEMPLATE ("audio_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
488
489
490
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
Artyom Baginski's avatar
Artyom Baginski committed
491

492
static GstStaticPadTemplate gst_qtdemux_subsrc_template =
Wim Taymans's avatar
Wim Taymans committed
493
GST_STATIC_PAD_TEMPLATE ("subtitle_%u",
494
495
496
497
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
498
499
#define gst_qtdemux_parent_class parent_class
G_DEFINE_TYPE (GstQTDemux, gst_qtdemux, GST_TYPE_ELEMENT);
Artyom Baginski's avatar
Artyom Baginski committed
500

501
static void gst_qtdemux_dispose (GObject * object);
502

503
504
static guint32
gst_qtdemux_find_index_linear (GstQTDemux * qtdemux, QtDemuxStream * str,
505
    GstClockTime media_time);
506
507
508
509
static guint32
gst_qtdemux_find_index_for_given_media_offset_linear (GstQTDemux * qtdemux,
    QtDemuxStream * str, gint64 media_offset);

510
#if 0
511
512
static void gst_qtdemux_set_index (GstElement * element, GstIndex * index);
static GstIndex *gst_qtdemux_get_index (GstElement * element);
513
#endif
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
514
515
static GstStateChangeReturn gst_qtdemux_change_state (GstElement * element,
    GstStateChange transition);
Wim Taymans's avatar
Wim Taymans committed
516
static gboolean qtdemux_sink_activate (GstPad * sinkpad, GstObject * parent);
Wim Taymans's avatar
Wim Taymans committed
517
518
static gboolean qtdemux_sink_activate_mode (GstPad * sinkpad,
    GstObject * parent, GstPadMode mode, gboolean active);
519
520

static void gst_qtdemux_loop (GstPad * pad);
Wim Taymans's avatar
Wim Taymans committed
521
522
523
524
static GstFlowReturn gst_qtdemux_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * inbuf);
static gboolean gst_qtdemux_handle_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
Thiago Santos's avatar
Thiago Santos committed
525
static gboolean gst_qtdemux_setcaps (GstQTDemux * qtdemux, GstCaps * caps);
526
527
static gboolean gst_qtdemux_configure_stream (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
528
529
static void gst_qtdemux_stream_check_and_change_stsd_index (GstQTDemux * demux,
    QtDemuxStream * stream);
530
531
static GstFlowReturn gst_qtdemux_process_adapter (GstQTDemux * demux,
    gboolean force);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
532

533
534
static gboolean qtdemux_parse_moov (GstQTDemux * qtdemux,
    const guint8 * buffer, guint length);
535
static gboolean qtdemux_parse_node (GstQTDemux * qtdemux, GNode * node,
536
    const guint8 * buffer, guint length);
537
static gboolean qtdemux_parse_tree (GstQTDemux * qtdemux);
Thiago Santos's avatar
Thiago Santos committed
538
539
static void qtdemux_parse_udta (GstQTDemux * qtdemux, GstTagList * taglist,
    GNode * udta);
540

541
static void gst_qtdemux_handle_esds (GstQTDemux * qtdemux,
542
543
    QtDemuxStream * stream, QtDemuxStreamStsdEntry * entry, GNode * esds,
    GstTagList * list);
544
static GstCaps *qtdemux_video_caps (GstQTDemux * qtdemux,
545
546
    QtDemuxStream * stream, QtDemuxStreamStsdEntry * entry, guint32 fourcc,
    const guint8 * stsd_entry_data, gchar ** codec_name);
547
static GstCaps *qtdemux_audio_caps (GstQTDemux * qtdemux,
548
549
550
551
    QtDemuxStream * stream, QtDemuxStreamStsdEntry * entry, guint32 fourcc,
    const guint8 * data, int len, gchar ** codec_name);
static GstCaps *qtdemux_sub_caps (GstQTDemux * qtdemux, QtDemuxStream * stream,
    QtDemuxStreamStsdEntry * entry, guint32 fourcc, const guint8 * data,
552
    gchar ** codec_name);
553
static GstCaps *qtdemux_generic_caps (GstQTDemux * qtdemux,
554
555
    QtDemuxStream * stream, QtDemuxStreamStsdEntry * entry, guint32 fourcc,
    const guint8 * stsd_entry_data, gchar ** codec_name);
556

557
558
static gboolean qtdemux_parse_samples (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 n);
559
static GstFlowReturn qtdemux_expose_streams (GstQTDemux * qtdemux);
560
561
static void gst_qtdemux_stream_free (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thiago Santos's avatar
Thiago Santos committed
562
563
static void gst_qtdemux_stream_clear (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Arnaud Vrac's avatar
Arnaud Vrac committed
564
static void gst_qtdemux_remove_stream (GstQTDemux * qtdemux, int index);
565
static GstFlowReturn qtdemux_prepare_streams (GstQTDemux * qtdemux);
566
567
static void qtdemux_do_allocation (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
568
569
570
571
572
static gboolean gst_qtdemux_activate_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 seg_idx, GstClockTime offset);
static gboolean gst_qtdemux_stream_update_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, gint seg_idx, GstClockTime offset,
    GstClockTime * _start, GstClockTime * _stop);
573
574
static void gst_qtdemux_send_gap_for_segment (GstQTDemux * demux,
    QtDemuxStream * stream, gint segment_index, GstClockTime pos);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
575

576
static gboolean qtdemux_pull_mfro_mfra (GstQTDemux * qtdemux);
577
static void check_update_duration (GstQTDemux * qtdemux, GstClockTime duration);
578

579
580
581
582
static gchar *qtdemux_uuid_bytes_to_string (gconstpointer uuid_bytes);

static GstStructure *qtdemux_get_cenc_sample_properties (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint sample_index);
583
584
static void gst_qtdemux_append_protection_system_id (GstQTDemux * qtdemux,
    const gchar * id);
585
static void qtdemux_gst_structure_free (GstStructure * gststructure);
586

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
587
588
static void
gst_qtdemux_class_init (GstQTDemuxClass * klass)
Artyom Baginski's avatar
Artyom Baginski committed
589
590
591
592
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
593
594
  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
Artyom Baginski's avatar
Artyom Baginski committed
595

596
  parent_class = g_type_class_peek_parent (klass);
597

598
599
  gobject_class->dispose = gst_qtdemux_dispose;

600
  gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state);
601
#if 0
602
603
  gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index);
  gstelement_class->get_index = GST_DEBUG_FUNCPTR (gst_qtdemux_get_index);
604
#endif
605
606

  gst_tag_register_musicbrainz_tags ();
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
607

608
609
610
611
612
613
614
615
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_sink_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_videosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_audiosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_subsrc_template);
616
  gst_element_class_set_static_metadata (gstelement_class, "QuickTime demuxer",
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
617
618
619
620
621
622
      "Codec/Demuxer",
      "Demultiplex a QuickTime file into audio and video streams",
      "David Schleef <ds@schleef.org>, Wim Taymans <wim@fluendo.com>");

  GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin");

Artyom Baginski's avatar
Artyom Baginski committed
623
624
}

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
625
static void
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
626
gst_qtdemux_init (GstQTDemux * qtdemux)
Artyom Baginski's avatar
Artyom Baginski committed
627
{
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
628
  qtdemux->sinkpad =
629
      gst_pad_new_from_static_template (&gst_qtdemux_sink_template, "sink");
630
  gst_pad_set_activate_function (qtdemux->sinkpad, qtdemux_sink_activate);
Wim Taymans's avatar
Wim Taymans committed
631
632
  gst_pad_set_activatemode_function (qtdemux->sinkpad,
      qtdemux_sink_activate_mode);
633
634
  gst_pad_set_chain_function (qtdemux->sinkpad, gst_qtdemux_chain);
  gst_pad_set_event_function (qtdemux->sinkpad, gst_qtdemux_handle_sink_event);
635
  gst_element_add_pad (GST_ELEMENT_CAST (qtdemux), qtdemux->sinkpad);
636

637
638
  qtdemux->state = QTDEMUX_STATE_INITIAL;
  qtdemux->pullbased = FALSE;
639
  qtdemux->posted_redirect = FALSE;
640
641
642
  qtdemux->neededbytes = 16;
  qtdemux->todrop = 0;
  qtdemux->adapter = gst_adapter_new ();
643
  qtdemux->offset = 0;
644
  qtdemux->first_mdat = -1;
645
  qtdemux->got_moov = FALSE;
646
  qtdemux->mdatoffset = -1;
647
  qtdemux->mdatbuffer = NULL;
648
  qtdemux->restoredata_buffer = NULL;
649
  qtdemux->restoredata_offset = -1;
650
  qtdemux->fragment_start = -1;
651
  qtdemux->fragment_start_offset = -1;
652
653
654
  qtdemux->media_caps = NULL;
  qtdemux->exposed = FALSE;
  qtdemux->mss_mode = FALSE;
655
  qtdemux->pending_newsegment = NULL;
656
  qtdemux->upstream_format_is_time = FALSE;
657
658
  qtdemux->have_group_id = FALSE;
  qtdemux->group_id = G_MAXUINT;
659
660
661
  qtdemux->cenc_aux_info_offset = 0;
  qtdemux->cenc_aux_info_sizes = NULL;
  qtdemux->cenc_aux_sample_count = 0;
662
663
  qtdemux->protection_system_ids = NULL;
  g_queue_init (&qtdemux->protection_event_queue);
664
  gst_segment_init (&qtdemux->segment, GST_FORMAT_TIME);
665
666
  qtdemux->tag_list = gst_tag_list_new_empty ();
  gst_tag_list_set_scope (qtdemux->tag_list, GST_TAG_SCOPE_GLOBAL);
Thiago Santos's avatar
Thiago Santos committed
667
  qtdemux->flowcombiner = gst_flow_combiner_new ();
Wim Taymans's avatar
Wim Taymans committed
668
669

  GST_OBJECT_FLAG_SET (qtdemux, GST_ELEMENT_FLAG_INDEXABLE);
Artyom Baginski's avatar
Artyom Baginski committed
670
671
}

672
673
674
675
676
677
678
679
680
static void
gst_qtdemux_dispose (GObject * object)
{
  GstQTDemux *qtdemux = GST_QTDEMUX (object);

  if (qtdemux->adapter) {
    g_object_unref (G_OBJECT (qtdemux->adapter));
    qtdemux->adapter = NULL;
  }
681
  gst_tag_list_unref (qtdemux->tag_list);
Thiago Santos's avatar
Thiago Santos committed
682
  gst_flow_combiner_free (qtdemux->flowcombiner);
683
684
685
  g_queue_foreach (&qtdemux->protection_event_queue, (GFunc) gst_event_unref,
      NULL);
  g_queue_clear (&qtdemux->protection_event_queue);
686

687
688
689
  g_free (qtdemux->cenc_aux_info_sizes);
  qtdemux->cenc_aux_info_sizes = NULL;

690
  G_OBJECT_CLASS (parent_class)->dispose (object);
691
692
}

693
694
695
696
static void
gst_qtdemux_post_no_playable_stream_error (GstQTDemux * qtdemux)
{
  if (qtdemux->posted_redirect) {
697
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
698
699
700
        (_("This file contains no playable streams.")),
        ("no known streams found, a redirect message has been posted"));
  } else {
701
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
702
703
704
705
706
        (_("This file contains no playable streams.")),
        ("no known streams found"));
  }
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
707
708
709
static GstBuffer *
_gst_buffer_new_wrapped (gpointer mem, gsize size, GFreeFunc free_func)
{
Wim Taymans's avatar
Wim Taymans committed
710
711
  return gst_buffer_new_wrapped_full (free_func ? 0 : GST_MEMORY_FLAG_READONLY,
      mem, size, 0, size, mem, free_func);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
712
713
}

714
715
716
717
718
static GstFlowReturn
gst_qtdemux_pull_atom (GstQTDemux * qtdemux, guint64 offset, guint64 size,
    GstBuffer ** buf)
{
  GstFlowReturn flow;
Wim Taymans's avatar
Wim Taymans committed
719
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
720
  gsize bsize;
721

722
  if (G_UNLIKELY (size == 0)) {
723
724
725
726
727
728
729
    GstFlowReturn ret;
    GstBuffer *tmp = NULL;

    ret = gst_qtdemux_pull_atom (qtdemux, offset, sizeof (guint32), &tmp);
    if (ret != GST_FLOW_OK)
      return ret;

Wim Taymans's avatar
Wim Taymans committed
730
731
    gst_buffer_map (tmp, &map, GST_MAP_READ);
    size = QT_UINT32 (map.data);
732
    GST_DEBUG_OBJECT (qtdemux, "size 0x%08" G_GINT64_MODIFIER "x", size);
733

Wim Taymans's avatar
Wim Taymans committed
734
    gst_buffer_unmap (tmp, &map);
735
736
737
    gst_buffer_unref (tmp);
  }

738
739
  /* Sanity check: catch bogus sizes (fuzzed/broken files) */
  if (G_UNLIKELY (size > QTDEMUX_MAX_ATOM_SIZE)) {
740
741
742
743
744
    if (qtdemux->state != QTDEMUX_STATE_MOVIE && qtdemux->got_moov) {
      /* we're pulling header but already got most interesting bits,
       * so never mind the rest (e.g. tags) (that much) */
      GST_WARNING_OBJECT (qtdemux, "atom has bogus size %" G_GUINT64_FORMAT,
          size);
745
      return GST_FLOW_EOS;
746
747
748
749
750
751
    } else {
      GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
          (_("This file is invalid and cannot be played.")),
          ("atom has bogus size %" G_GUINT64_FORMAT, size));
      return GST_FLOW_ERROR;
    }
752
753
754
755
756
757
758
  }

  flow = gst_pad_pull_range (qtdemux->sinkpad, offset, size, buf);

  if (G_UNLIKELY (flow != GST_FLOW_OK))
    return flow;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
759
  bsize = gst_buffer_get_size (*buf);
760
  /* Catch short reads - we don't want any partial atoms */
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
761
  if (G_UNLIKELY (bsize < size)) {
762
763
    GST_WARNING_OBJECT (qtdemux,
        "short read: %" G_GSIZE_FORMAT " < %" G_GUINT64_FORMAT, bsize, size);
764
765
    gst_buffer_unref (*buf);
    *buf = NULL;
766
    return GST_FLOW_EOS;
767
768
769
770
771
  }

  return flow;
}

772
#if 1
773
static gboolean
774
775
776
gst_qtdemux_src_convert (GstQTDemux * qtdemux, GstPad * pad,
    GstFormat src_format, gint64 src_value, GstFormat dest_format,
    gint64 * dest_value)
777
778
{
  gboolean res = TRUE;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
779
  QtDemuxStream *stream = gst_pad_get_element_private (pad);
780
  gint32 index;
781

Andy Wingo's avatar
Andy Wingo committed
782
783
784
785
  if (stream->subtype != FOURCC_vide) {
    res = FALSE;
    goto done;
  }
786
787
788

  switch (src_format) {
    case GST_FORMAT_TIME:
789
790
791
      switch (dest_format) {
        case GST_FORMAT_BYTES:{
          index = gst_qtdemux_find_index_linear (qtdemux, stream, src_value);
792
793
794
795
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
796
797
798
799
800
801

          *dest_value = stream->samples[index].offset;

          GST_DEBUG_OBJECT (qtdemux, "Format Conversion Time->Offset :%"
              GST_TIME_FORMAT "->%" G_GUINT64_FORMAT,
              GST_TIME_ARGS (src_value), *dest_value);
802
          break;
803
        }
804
805
806
        default:
          res = FALSE;
          break;
807
808
809
      }
      break;
    case GST_FORMAT_BYTES:
810
811
812
813
814
815
      switch (dest_format) {
        case GST_FORMAT_TIME:{
          index =
              gst_qtdemux_find_index_for_given_media_offset_linear (qtdemux,
              stream, src_value);

816
817
818
819
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
820
821

          *dest_value =
822
823
824
825
826
              QTSTREAMTIME_TO_GSTTIME (stream,
              stream->samples[index].timestamp);
          GST_DEBUG_OBJECT (qtdemux,
              "Format Conversion Offset->Time :%" G_GUINT64_FORMAT "->%"
              GST_TIME_FORMAT, src_value, GST_TIME_ARGS (*dest_value));
827
          break;
828
        }
829
830
831
        default:
          res = FALSE;
          break;
832
833
834
835
      }
      break;
    default:
      res = FALSE;
836
      break;
837
838
  }

Andy Wingo's avatar
Andy Wingo committed
839
done:
840
841
  return res;
}
842
#endif
843

844
static gboolean
845
gst_qtdemux_get_duration (GstQTDemux * qtdemux, GstClockTime * duration)
846
{
847
  gboolean res = FALSE;
848
849
850

  *duration = GST_CLOCK_TIME_NONE;

851
852
853
854
855
856
  if (qtdemux->duration != 0 &&
      qtdemux->duration != G_MAXINT64 && qtdemux->timescale != 0) {
    *duration = QTTIME_TO_GSTTIME (qtdemux, qtdemux->duration);
    res = TRUE;
  } else {
    *duration = GST_CLOCK_TIME_NONE;
857
  }
858

859
860
861
  return res;
}

862
static gboolean
Wim Taymans's avatar
Wim Taymans committed
863
864
gst_qtdemux_handle_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
865
{
866
  gboolean res = FALSE;
Wim Taymans's avatar
Wim Taymans committed
867
  GstQTDemux *qtdemux = GST_QTDEMUX (parent);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
868

869
870
  GST_LOG_OBJECT (pad, "%s query", GST_QUERY_TYPE_NAME (query));

871
  switch (GST_QUERY_TYPE (query)) {
872
873
874
875
876
877
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME
          && GST_CLOCK_TIME_IS_VALID (qtdemux->segment.position)) {
878
        gst_query_set_position (query, GST_FORMAT_TIME,
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
879
            qtdemux->segment.position);
Wim Taymans's avatar
Wim Taymans committed
880
881
        res = TRUE;
      }
882
    }
Wim Taymans's avatar
Wim Taymans committed
883
      break;
884
885
    case GST_QUERY_DURATION:{
      GstFormat fmt;
886

887
888
      gst_query_parse_duration (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME) {
889
890
891
        /* First try to query upstream */
        res = gst_pad_query_default (pad, parent, query);
        if (!res) {
892
          GstClockTime duration;
893
          if (gst_qtdemux_get_duration (qtdemux, &duration) && duration > 0) {
894
895
896
            gst_query_set_duration (query, GST_FORMAT_TIME, duration);
            res = TRUE;
          }
897
898
899
        }
      }
      break;
900
    }
901
902
    case GST_QUERY_CONVERT:{
      GstFormat src_fmt, dest_fmt;
903
      gint64 src_value, dest_value = 0;
904
905
906

      gst_query_parse_convert (query, &src_fmt, &src_value, &dest_fmt, NULL);

907
      res = gst_qtdemux_src_convert (qtdemux, pad,
908
909
910
911
912
913
914
915
916
917
918
          src_fmt, src_value, dest_fmt, &dest_value);
      if (res) {
        gst_query_set_convert (query, src_fmt, src_value, dest_fmt, dest_value);
        res = TRUE;
      }
      break;
    }
    case GST_QUERY_FORMATS:
      gst_query_set_formats (query, 2, GST_FORMAT_TIME, GST_FORMAT_BYTES);
      res = TRUE;
      break;
919
920
    case GST_QUERY_SEEKING:{
      GstFormat fmt;
921
      gboolean seekable;
922

Thiago Santos's avatar
Thiago Santos committed
923
924
925
926
927
928
      /* try upstream first */
      res = gst_pad_query_default (pad, parent, query);

      if (!res) {
        gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
        if (fmt == GST_FORMAT_TIME) {
929
          GstClockTime duration;
Thiago Santos's avatar
Thiago Santos committed
930
931
932
933
934
935
936
937
938
939
940
941
942
943

          gst_qtdemux_get_duration (qtdemux, &duration);
          seekable = TRUE;
          if (!qtdemux->pullbased) {
            GstQuery *q;

            /* we might be able with help from upstream */
            seekable = FALSE;
            q = gst_query_new_seeking (GST_FORMAT_BYTES);
            if (gst_pad_peer_query (qtdemux->sinkpad, q)) {
              gst_query_parse_seeking (q, &fmt, &seekable, NULL, NULL);
              GST_LOG_OBJECT (qtdemux, "upstream BYTE seekable %d", seekable);
            }
            gst_query_unref (q);
944
          }
Thiago Santos's avatar
Thiago Santos committed
945
946
          gst_query_set_seeking (query, GST_FORMAT_TIME, seekable, 0, duration);
          res = TRUE;
947
        }
948
      }
949
      break;
950
    }
Wim Taymans's avatar
Wim Taymans committed
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
    case GST_QUERY_SEGMENT:
    {
      GstFormat format;
      gint64 start, stop;

      format = qtdemux->segment.format;

      start =
          gst_segment_to_stream_time (&qtdemux->segment, format,
          qtdemux->segment.start);
      if ((stop = qtdemux->segment.stop) == -1)
        stop = qtdemux->segment.duration;
      else
        stop = gst_segment_to_stream_time (&qtdemux->segment, format, stop);

      gst_query_set_segment (query, qtdemux->segment.rate, format, start, stop);
      res = TRUE;
      break;
    }
970
    default:
Wim Taymans's avatar
Wim Taymans committed
971
      res = gst_pad_query_default (pad, parent, query);
972
973
974
975
976
977
      break;
  }

  return res;
}

978
979
980
981
982
983
984
static void
gst_qtdemux_push_tags (GstQTDemux * qtdemux, QtDemuxStream * stream)
{
  if (G_LIKELY (stream->pad)) {
    GST_DEBUG_OBJECT (qtdemux, "Checking pad %s:%s for tags",
        GST_DEBUG_PAD_NAME (stream->pad));

985
    if (!gst_tag_list_is_empty (stream->stream_tags)) {
986
      GST_DEBUG_OBJECT (qtdemux, "Sending tags %" GST_PTR_FORMAT,
987
          stream->stream_tags);
988
      gst_pad_push_event (stream->pad,
989
          gst_event_new_tag (gst_tag_list_ref (stream->stream_tags)));
990
991
    }

992
    if (G_UNLIKELY (stream->send_global_tags)) {
993
994
995
      GST_DEBUG_OBJECT (qtdemux, "Sending global tags %" GST_PTR_FORMAT,
          qtdemux->tag_list);
      gst_pad_push_event (stream->pad,
996
          gst_event_new_tag (gst_tag_list_ref (qtdemux->tag_list)));
997
998
999
1000
1001
      stream->send_global_tags = FALSE;
    }
  }
}

1002
/* push event on all source pads; takes ownership of the event */
1003
static void
1004
gst_qtdemux_push_event (GstQTDemux * qtdemux, GstEvent * event)
1005
1006
{
  guint n;
1007
  gboolean has_valid_stream = FALSE;
1008
  GstEventType etype = GST_EVENT_TYPE (event);
1009
1010
1011
1012
1013

  GST_DEBUG_OBJECT (qtdemux, "pushing %s event on all source pads",
      GST_EVENT_TYPE_NAME (event));

  for (n = 0; n < qtdemux->n_streams; n++) {
1014
    GstPad *pad;
1015
    QtDemuxStream *stream = qtdemux->streams[n];