qtdemux.c 447 KB
Newer Older
Artyom Baginski's avatar
Artyom Baginski committed
1
2
/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
3
 * Copyright (C) <2003> David A. Schleef <ds@schleef.org>
4
 * Copyright (C) <2006> Wim Taymans <wim@fluendo.com>
5
 * Copyright (C) <2007> Julien Moutte <julien@fluendo.com>
6
 * Copyright (C) <2009> Tim-Philipp Müller <tim centricular net>
7
 * Copyright (C) <2009> STEricsson <benjamin.gaignard@stericsson.com>
8
 * Copyright (C) <2013> Sreerenj Balachandran <sreerenj.balachandran@intel.com>
9
10
 * Copyright (C) <2013> Intel Corporation
 * Copyright (C) <2014> Centricular Ltd
11
 * Copyright (C) <2015> YouView TV Ltd.
12
 * Copyright (C) <2016> British Broadcasting Corporation
13
 *
Artyom Baginski's avatar
Artyom Baginski committed
14
15
16
17
18
19
20
21
22
23
24
25
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
26
27
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
Artyom Baginski's avatar
Artyom Baginski committed
28
29
 */

30
31
32
33
/**
 * SECTION:element-qtdemux
 *
 * Demuxes a .mov file into raw or compressed audio and/or video streams.
34
 *
35
36
 * This element supports both push and pull-based scheduling, depending on the
 * capabilities of the upstream elements.
37
38
 *
 * <refsect2>
39
 * <title>Example launch line</title>
40
 * |[
41
 * gst-launch-1.0 filesrc location=test.mov ! qtdemux name=demux  demux.audio_0 ! queue ! decodebin ! audioconvert ! audioresample ! autoaudiosink   demux.video_0 ! queue ! decodebin ! videoconvert ! videoscale ! autovideosink
42
 * ]| Play (parse and decode) a .mov file and try to output it to
43
44
45
46
47
48
 * an automatically detected soundcard and videosink. If the MOV file contains
 * compressed audio or video data, this will only work if you have the
 * right decoder elements/plugins installed.
 * </refsect2>
 */

49
50
51
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
52
53
54

#include "gst/gst-i18n-plugin.h"

55
#include <glib/gprintf.h>
56
#include <gst/tag/tag.h>
Wim Taymans's avatar
Wim Taymans committed
57
#include <gst/audio/audio.h>
58
#include <gst/video/video.h>
59

60
#include "qtatomparser.h"
61
62
#include "qtdemux_types.h"
#include "qtdemux_dump.h"
63
#include "fourcc.h"
64
#include "descriptors.h"
65
#include "qtdemux_lang.h"
66
#include "qtdemux.h"
67
#include "qtpalette.h"
68

69
70
71
#include "gst/riff/riff-media.h"
#include "gst/riff/riff-read.h"

72
73
#include <gst/pbutils/pbutils.h>

74
#include <stdio.h>
75
#include <stdlib.h>
Artyom Baginski's avatar
Artyom Baginski committed
76
#include <string.h>
77

78
79
80
#include <math.h>
#include <gst/math-compat.h>

81
82
83
#ifdef HAVE_ZLIB
# include <zlib.h>
#endif
84

85
86
87
/* max. size considered 'sane' for non-mdat atoms */
#define QTDEMUX_MAX_ATOM_SIZE (25*1024*1024)

88
89
90
/* if the sample index is larger than this, something is likely wrong */
#define QTDEMUX_MAX_SAMPLE_INDEX_SIZE (50*1024*1024)

91
92
93
94
95
96
/* For converting qt creation times to unix epoch times */
#define QTDEMUX_SECONDS_PER_DAY (60 * 60 * 24)
#define QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970 17
#define QTDEMUX_SECONDS_FROM_1904_TO_1970 (((1970 - 1904) * (guint64) 365 + \
    QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970) * QTDEMUX_SECONDS_PER_DAY)

97
#define STREAM_IS_EOS(s) (s->time_position == GST_CLOCK_TIME_NONE)
98

99
100
#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )

101
GST_DEBUG_CATEGORY (qtdemux_debug);
102

103
typedef struct _QtDemuxSegment QtDemuxSegment;
104
typedef struct _QtDemuxSample QtDemuxSample;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
105

106
107
typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
108
109
struct _QtDemuxSample
{
110
  guint32 size;
111
  gint32 pts_offset;            /* Add this value to timestamp to get the pts */
112
  guint64 offset;
113
114
  guint64 timestamp;            /* DTS In mov time */
  guint32 duration;             /* In mov time */
115
  gboolean keyframe;            /* TRUE when this packet is a keyframe */
116
117
};

118
119
120
121
122
123
124
/* Macros for converting to/from timescale */
#define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale))
#define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND))

#define QTTIME_TO_GSTTIME(qtdemux, value) (gst_util_uint64_scale((value), GST_SECOND, (qtdemux)->timescale))
#define GSTTIME_TO_QTTIME(qtdemux, value) (gst_util_uint64_scale((value), (qtdemux)->timescale, GST_SECOND))

125
/* timestamp is the DTS */
126
#define QTSAMPLE_DTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp))
127
/* timestamp + offset is the PTS */
Nicolas Dufresne's avatar
Nicolas Dufresne committed
128
#define QTSAMPLE_PTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (stream)->cslg_shift + (sample)->pts_offset))
129
/* timestamp + duration - dts is the duration */
130
#define QTSAMPLE_DUR_DTS(stream, sample, dts) (QTSTREAMTIME_TO_GSTTIME ((stream), (sample)->timestamp + (sample)->duration) - (dts))
131
132
133

#define QTSAMPLE_KEYFRAME(stream,sample) ((stream)->all_keyframe || (sample)->keyframe)

134
135
136
137
138
139
140
141
142
143
144
/*
 * Quicktime has tracks and segments. A track is a continuous piece of
 * multimedia content. The track is not always played from start to finish but
 * instead, pieces of the track are 'cut out' and played in sequence. This is
 * what the segments do.
 *
 * Inside the track we have keyframes (K) and delta frames. The track has its
 * own timing, which starts from 0 and extends to end. The position in the track
 * is called the media_time.
 *
 * The segments now describe the pieces that should be played from this track
145
 * and are basically tuples of media_time/duration/rate entries. We can have
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 * multiple segments and they are all played after one another. An example:
 *
 * segment 1: media_time: 1 second, duration: 1 second, rate 1
 * segment 2: media_time: 3 second, duration: 2 second, rate 2
 *
 * To correctly play back this track, one must play: 1 second of media starting
 * from media_time 1 followed by 2 seconds of media starting from media_time 3
 * at a rate of 2.
 *
 * Each of the segments will be played at a specific time, the first segment at
 * time 0, the second one after the duration of the first one, etc.. Note that
 * the time in resulting playback is not identical to the media_time of the
 * track anymore.
 *
 * Visually, assuming the track has 4 second of media_time:
 *
 *                (a)                   (b)          (c)              (d)
 *         .-----------------------------------------------------------.
 * track:  | K.....K.........K........K.......K.......K...........K... |
 *         '-----------------------------------------------------------'
Stefan Kost's avatar
Stefan Kost committed
166
 *         0              1              2              3              4
167
168
169
170
171
172
 *           .------------^              ^   .----------^              ^
 *          /              .-------------'  /       .------------------'
 *         /              /          .-----'       /
 *         .--------------.         .--------------.
 *         | segment 1    |         | segment 2    |
 *         '--------------'         '--------------'
Stefan Kost's avatar
Stefan Kost committed
173
 *
174
 * The challenge here is to cut out the right pieces of the track for each of
175
176
 * the playback segments. This fortunately can easily be done with the SEGMENT
 * events of GStreamer.
177
178
179
180
181
182
183
184
 *
 * For playback of segment 1, we need to provide the decoder with the keyframe
 * (a), in the above figure, but we must instruct it only to output the decoded
 * data between second 1 and 2. We do this with a SEGMENT event for 1 to 2, time
 * position set to the time of the segment: 0.
 *
 * We then proceed to push data from keyframe (a) to frame (b). The decoder
 * decodes but clips all before media_time 1.
Stefan Kost's avatar
Stefan Kost committed
185
 *
186
187
188
189
190
191
 * After finishing a segment, we push out a new SEGMENT event with the clipping
 * boundaries of the new data.
 *
 * This is a good usecase for the GStreamer accumulated SEGMENT events.
 */

192
193
194
struct _QtDemuxSegment
{
  /* global time and duration, all gst time */
195
196
197
  GstClockTime time;
  GstClockTime stop_time;
  GstClockTime duration;
198
  /* media time of trak, all gst time */
199
200
  GstClockTime media_start;
  GstClockTime media_stop;
201
  gdouble rate;
202
203
  /* Media start time in trak timescale units */
  guint32 trak_media_start;
204
205
};

206
207
#define QTSEGMENT_IS_EMPTY(s) ((s)->media_start == GST_CLOCK_TIME_NONE)

208
209
210
211
212
213
214
/* Used with fragmented MP4 files (mfra atom) */
typedef struct
{
  GstClockTime ts;
  guint64 moof_offset;
} QtDemuxRandomAccessEntry;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
215
216
struct _QtDemuxStream
{
217
218
219
  GstPad *pad;

  /* stream type */
220
221
  guint32 subtype;
  GstCaps *caps;
222
  guint32 fourcc;
223
  gboolean sparse;
224

225
226
227
228
  gboolean new_caps;            /* If TRUE, caps need to be generated (by
                                 * calling _configure_stream()) This happens
                                 * for MSS and fragmented streams */

229
  gboolean new_stream;          /* signals that a stream_start is required */
230
231
232
233
  gboolean on_keyframe;         /* if this stream last pushed buffer was a
                                 * keyframe. This is important to identify
                                 * where to stop pushing buffers after a
                                 * segment stop time */
234

235
236
237
  /* if the stream has a redirect URI in its headers, we store it here */
  gchar *redirect_uri;

238
239
240
  /* track id */
  guint track_id;

241
  /* duration/scale */
242
  guint64 duration;             /* in timescale units */
243
  guint32 timescale;
244

245
  /* language */
246
  gchar lang_id[4];             /* ISO 639-2T language code */
247

248
249
250
251
  /* our samples */
  guint32 n_samples;
  QtDemuxSample *samples;
  gboolean all_keyframe;        /* TRUE when all samples are keyframes (no stss) */
252
  guint32 first_duration;       /* duration in timescale of first sample, used for figuring out
253
                                   the framerate */
254
255
256
  guint32 n_samples_moof;       /* sample count in a moof */
  guint64 duration_moof;        /* duration in timescale of a moof, used for figure out
                                 * the framerate of fragmented format stream */
257
  guint64 duration_last_moof;
258
259
260
261
262
263
264

  guint32 offset_in_sample;     /* Offset in the current sample, used for
                                 * streams which have got exceedingly big
                                 * sample size (such as 24s of raw audio).
                                 * Only used when max_buffer_size is non-NULL */
  guint32 max_buffer_size;      /* Maximum allowed size for output buffers.
                                 * Currently only set for raw audio streams*/
265

266
267
  /* if we use chunks or samples */
  gboolean sampled;
268
  guint padding;
269
270

  /* video info */
271
272
  gint width;
  gint height;
273
274
275
276
277
  /* aspect ratio */
  gint display_width;
  gint display_height;
  gint par_w;
  gint par_h;
278
279
280
  /* Numerator/denominator framerate */
  gint fps_n;
  gint fps_d;
281
  GstVideoColorimetry colorimetry;
282
283
  guint16 bits_per_sample;
  guint16 color_table_id;
284
  GstMemory *rgb8_palette;
285
286
  guint interlace_mode;
  guint field_order;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
287

288
  /* audio info */
289
290
  gdouble rate;
  gint n_channels;
291
292
293
294
  guint samples_per_packet;
  guint samples_per_frame;
  guint bytes_per_packet;
  guint bytes_per_sample;
295
  guint bytes_per_frame;
296
  guint compression;
297

298
299
300
301
302
  /* allocation */
  gboolean use_allocator;
  GstAllocator *allocator;
  GstAllocationParams params;

303
304
  gsize alignment;

305
306
  /* when a discontinuity is pending */
  gboolean discont;
307

308
309
310
  /* list of buffers to push first */
  GSList *buffers;

311
312
313
314
  /* if we need to clip this buffer. This is only needed for uncompressed
   * data */
  gboolean need_clip;

315
316
317
  /* buffer needs some custom processing, e.g. subtitles */
  gboolean need_process;

318
319
320
  /* current position */
  guint32 segment_index;
  guint32 sample_index;
321
  GstClockTime time_position;   /* in gst time */
322
  guint64 accumulated_base;
323

324
325
326
  /* the Gst segment we are processing out, used for clipping */
  GstSegment segment;

327
328
329
  /* quicktime segments */
  guint32 n_segments;
  QtDemuxSegment *segments;
330
  gboolean dummy_segment;
331
332
  guint32 from_sample;
  guint32 to_sample;
333
334

  gboolean sent_eos;
335
336
  GstTagList *pending_tags;
  gboolean send_global_tags;
337
338

  GstEvent *pending_event;
339
340
341
342
343
344
345
346
347

  GstByteReader stco;
  GstByteReader stsz;
  GstByteReader stsc;
  GstByteReader stts;
  GstByteReader stss;
  GstByteReader stps;
  GstByteReader ctts;

348
  gboolean chunks_are_samples;  /* TRUE means treat chunks as samples */
349
  gint64 stbl_index;
350
351
  /* stco */
  guint co_size;
352
353
354
355
356
357
  GstByteReader co_chunk;
  guint32 first_chunk;
  guint32 current_chunk;
  guint32 last_chunk;
  guint32 samples_per_chunk;
  guint32 stco_sample_index;
358
359
360
  /* stsz */
  guint32 sample_size;          /* 0 means variable sizes are stored in stsz */
  /* stsc */
361
  guint32 stsc_index;
362
  guint32 n_samples_per_chunk;
363
364
365
  guint32 stsc_chunk_index;
  guint32 stsc_sample_index;
  guint64 chunk_offset;
366
  /* stts */
367
368
  guint32 stts_index;
  guint32 stts_samples;
369
  guint32 n_sample_times;
370
  guint32 stts_sample_index;
371
  guint64 stts_time;
372
  guint32 stts_duration;
373
  /* stss */
374
  gboolean stss_present;
375
  guint32 n_sample_syncs;
376
  guint32 stss_index;
377
  /* stps */
378
  gboolean stps_present;
379
  guint32 n_sample_partial_syncs;
380
  guint32 stps_index;
381
382
383
  QtDemuxRandomAccessEntry *ra_entries;
  guint n_ra_entries;

384
385
  const QtDemuxRandomAccessEntry *pending_seek;

386
387
388
  /* ctts */
  gboolean ctts_present;
  guint32 n_composition_times;
389
390
391
392
  guint32 ctts_index;
  guint32 ctts_sample_index;
  guint32 ctts_count;
  gint32 ctts_soffset;
393

Nicolas Dufresne's avatar
Nicolas Dufresne committed
394
395
396
  /* cslg */
  guint32 cslg_shift;

397
398
399
400
401
  /* fragmented */
  gboolean parsed_trex;
  guint32 def_sample_duration;
  guint32 def_sample_size;
  guint32 def_sample_flags;
402
403

  gboolean disabled;
404
405
406
407

  /* stereoscopic video streams */
  GstVideoMultiviewMode multiview_mode;
  GstVideoMultiviewFlags multiview_flags;
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424

  /* protected streams */
  gboolean protected;
  guint32 protection_scheme_type;
  guint32 protection_scheme_version;
  gpointer protection_scheme_info;      /* specific to the protection scheme */
  GQueue protection_scheme_event_queue;
};

/* Contains properties and cryptographic info for a set of samples from a
 * track protected using Common Encryption (cenc) */
struct _QtDemuxCencSampleSetInfo
{
  GstStructure *default_properties;

  /* @crypto_info holds one GstStructure per sample */
  GPtrArray *crypto_info;
425
426
};

427
428
static const gchar *
qt_demux_state_string (enum QtDemuxState state)
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
429
{
430
431
432
433
434
435
436
437
438
439
440
441
442
  switch (state) {
    case QTDEMUX_STATE_INITIAL:
      return "<INITIAL>";
    case QTDEMUX_STATE_HEADER:
      return "<HEADER>";
    case QTDEMUX_STATE_MOVIE:
      return "<MOVIE>";
    case QTDEMUX_STATE_BUFFER_MDAT:
      return "<BUFFER_MDAT>";
    default:
      return "<UNKNOWN>";
  }
}
443

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
444
static GNode *qtdemux_tree_get_child_by_type (GNode * node, guint32 fourcc);
445
static GNode *qtdemux_tree_get_child_by_type_full (GNode * node,
446
    guint32 fourcc, GstByteReader * parser);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
447
static GNode *qtdemux_tree_get_sibling_by_type (GNode * node, guint32 fourcc);
448
449
static GNode *qtdemux_tree_get_sibling_by_type_full (GNode * node,
    guint32 fourcc, GstByteReader * parser);
450

451
452
static GstFlowReturn qtdemux_add_fragmented_samples (GstQTDemux * qtdemux);

David Schleef's avatar
David Schleef committed
453
static GstStaticPadTemplate gst_qtdemux_sink_template =
454
    GST_STATIC_PAD_TEMPLATE ("sink",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
455
    GST_PAD_SINK,
456
    GST_PAD_ALWAYS,
457
458
    GST_STATIC_CAPS ("video/quicktime; video/mj2; audio/x-m4a; "
        "application/x-3gp")
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
459
    );
David Schleef's avatar
David Schleef committed
460
461

static GstStaticPadTemplate gst_qtdemux_videosrc_template =
Wim Taymans's avatar
Wim Taymans committed
462
GST_STATIC_PAD_TEMPLATE ("video_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
463
464
465
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
David Schleef's avatar
David Schleef committed
466
467

static GstStaticPadTemplate gst_qtdemux_audiosrc_template =
Wim Taymans's avatar
Wim Taymans committed
468
GST_STATIC_PAD_TEMPLATE ("audio_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
469
470
471
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
Artyom Baginski's avatar
Artyom Baginski committed
472

473
static GstStaticPadTemplate gst_qtdemux_subsrc_template =
Wim Taymans's avatar
Wim Taymans committed
474
GST_STATIC_PAD_TEMPLATE ("subtitle_%u",
475
476
477
478
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
479
480
#define gst_qtdemux_parent_class parent_class
G_DEFINE_TYPE (GstQTDemux, gst_qtdemux, GST_TYPE_ELEMENT);
Artyom Baginski's avatar
Artyom Baginski committed
481

482
static void gst_qtdemux_dispose (GObject * object);
483

484
485
static guint32
gst_qtdemux_find_index_linear (GstQTDemux * qtdemux, QtDemuxStream * str,
486
    GstClockTime media_time);
487
488
489
490
static guint32
gst_qtdemux_find_index_for_given_media_offset_linear (GstQTDemux * qtdemux,
    QtDemuxStream * str, gint64 media_offset);

491
#if 0
492
493
static void gst_qtdemux_set_index (GstElement * element, GstIndex * index);
static GstIndex *gst_qtdemux_get_index (GstElement * element);
494
#endif
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
495
496
static GstStateChangeReturn gst_qtdemux_change_state (GstElement * element,
    GstStateChange transition);
Wim Taymans's avatar
Wim Taymans committed
497
static gboolean qtdemux_sink_activate (GstPad * sinkpad, GstObject * parent);
Wim Taymans's avatar
Wim Taymans committed
498
499
static gboolean qtdemux_sink_activate_mode (GstPad * sinkpad,
    GstObject * parent, GstPadMode mode, gboolean active);
500
501

static void gst_qtdemux_loop (GstPad * pad);
Wim Taymans's avatar
Wim Taymans committed
502
503
504
505
static GstFlowReturn gst_qtdemux_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * inbuf);
static gboolean gst_qtdemux_handle_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
Thiago Santos's avatar
Thiago Santos committed
506
static gboolean gst_qtdemux_setcaps (GstQTDemux * qtdemux, GstCaps * caps);
507
508
static gboolean gst_qtdemux_configure_stream (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
509
510
static GstFlowReturn gst_qtdemux_process_adapter (GstQTDemux * demux,
    gboolean force);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
511

512
513
static gboolean qtdemux_parse_moov (GstQTDemux * qtdemux,
    const guint8 * buffer, guint length);
514
static gboolean qtdemux_parse_node (GstQTDemux * qtdemux, GNode * node,
515
    const guint8 * buffer, guint length);
516
static gboolean qtdemux_parse_tree (GstQTDemux * qtdemux);
Thiago Santos's avatar
Thiago Santos committed
517
518
static void qtdemux_parse_udta (GstQTDemux * qtdemux, GstTagList * taglist,
    GNode * udta);
519

520
static void gst_qtdemux_handle_esds (GstQTDemux * qtdemux,
521
    QtDemuxStream * stream, GNode * esds, GstTagList * list);
522
523
static GstCaps *qtdemux_video_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
524
    gchar ** codec_name);
525
526
static GstCaps *qtdemux_audio_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data, int len,
527
    gchar ** codec_name);
528
static GstCaps *qtdemux_sub_caps (GstQTDemux * qtdemux,
529
530
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data,
    gchar ** codec_name);
531
532
533
534
static GstCaps *qtdemux_generic_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
    gchar ** codec_name);

535
536
static gboolean qtdemux_parse_samples (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 n);
537
static GstFlowReturn qtdemux_expose_streams (GstQTDemux * qtdemux);
538
539
static void gst_qtdemux_stream_free (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thiago Santos's avatar
Thiago Santos committed
540
541
static void gst_qtdemux_stream_clear (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Arnaud Vrac's avatar
Arnaud Vrac committed
542
static void gst_qtdemux_remove_stream (GstQTDemux * qtdemux, int index);
543
static GstFlowReturn qtdemux_prepare_streams (GstQTDemux * qtdemux);
544
545
static void qtdemux_do_allocation (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
546
547
548
549
550
static gboolean gst_qtdemux_activate_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 seg_idx, GstClockTime offset);
static gboolean gst_qtdemux_stream_update_segment (GstQTDemux * qtdemux,
    QtDemuxStream * stream, gint seg_idx, GstClockTime offset,
    GstClockTime * _start, GstClockTime * _stop);
551
552
static void gst_qtdemux_send_gap_for_segment (GstQTDemux * demux,
    QtDemuxStream * stream, gint segment_index, GstClockTime pos);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
553

554
static gboolean qtdemux_pull_mfro_mfra (GstQTDemux * qtdemux);
555
static void check_update_duration (GstQTDemux * qtdemux, GstClockTime duration);
556

557
558
559
560
static gchar *qtdemux_uuid_bytes_to_string (gconstpointer uuid_bytes);

static GstStructure *qtdemux_get_cenc_sample_properties (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint sample_index);
561
562
static void gst_qtdemux_append_protection_system_id (GstQTDemux * qtdemux,
    const gchar * id);
563
static void qtdemux_gst_structure_free (GstStructure * gststructure);
564

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
565
566
static void
gst_qtdemux_class_init (GstQTDemuxClass * klass)
Artyom Baginski's avatar
Artyom Baginski committed
567
568
569
570
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
571
572
  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
Artyom Baginski's avatar
Artyom Baginski committed
573

574
  parent_class = g_type_class_peek_parent (klass);
575

576
577
  gobject_class->dispose = gst_qtdemux_dispose;

578
  gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state);
579
#if 0
580
581
  gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index);
  gstelement_class->get_index = GST_DEBUG_FUNCPTR (gst_qtdemux_get_index);
582
#endif
583
584

  gst_tag_register_musicbrainz_tags ();
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
585

586
587
588
589
590
591
592
593
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_sink_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_videosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_audiosrc_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_qtdemux_subsrc_template);
594
  gst_element_class_set_static_metadata (gstelement_class, "QuickTime demuxer",
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
595
596
597
598
599
600
      "Codec/Demuxer",
      "Demultiplex a QuickTime file into audio and video streams",
      "David Schleef <ds@schleef.org>, Wim Taymans <wim@fluendo.com>");

  GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin");

Artyom Baginski's avatar
Artyom Baginski committed
601
602
}

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
603
static void
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
604
gst_qtdemux_init (GstQTDemux * qtdemux)
Artyom Baginski's avatar
Artyom Baginski committed
605
{
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
606
  qtdemux->sinkpad =
607
      gst_pad_new_from_static_template (&gst_qtdemux_sink_template, "sink");
608
  gst_pad_set_activate_function (qtdemux->sinkpad, qtdemux_sink_activate);
Wim Taymans's avatar
Wim Taymans committed
609
610
  gst_pad_set_activatemode_function (qtdemux->sinkpad,
      qtdemux_sink_activate_mode);
611
612
  gst_pad_set_chain_function (qtdemux->sinkpad, gst_qtdemux_chain);
  gst_pad_set_event_function (qtdemux->sinkpad, gst_qtdemux_handle_sink_event);
613
  gst_element_add_pad (GST_ELEMENT_CAST (qtdemux), qtdemux->sinkpad);
614

615
616
  qtdemux->state = QTDEMUX_STATE_INITIAL;
  qtdemux->pullbased = FALSE;
617
  qtdemux->posted_redirect = FALSE;
618
619
620
  qtdemux->neededbytes = 16;
  qtdemux->todrop = 0;
  qtdemux->adapter = gst_adapter_new ();
621
  qtdemux->offset = 0;
622
  qtdemux->first_mdat = -1;
623
  qtdemux->got_moov = FALSE;
624
  qtdemux->mdatoffset = -1;
625
  qtdemux->mdatbuffer = NULL;
626
  qtdemux->restoredata_buffer = NULL;
627
  qtdemux->restoredata_offset = -1;
628
  qtdemux->fragment_start = -1;
629
  qtdemux->fragment_start_offset = -1;
630
631
632
  qtdemux->media_caps = NULL;
  qtdemux->exposed = FALSE;
  qtdemux->mss_mode = FALSE;
633
  qtdemux->pending_newsegment = NULL;
634
  qtdemux->upstream_format_is_time = FALSE;
635
636
  qtdemux->have_group_id = FALSE;
  qtdemux->group_id = G_MAXUINT;
637
638
639
  qtdemux->cenc_aux_info_offset = 0;
  qtdemux->cenc_aux_info_sizes = NULL;
  qtdemux->cenc_aux_sample_count = 0;
640
641
  qtdemux->protection_system_ids = NULL;
  g_queue_init (&qtdemux->protection_event_queue);
642
  gst_segment_init (&qtdemux->segment, GST_FORMAT_TIME);
Thiago Santos's avatar
Thiago Santos committed
643
  qtdemux->flowcombiner = gst_flow_combiner_new ();
Wim Taymans's avatar
Wim Taymans committed
644
645

  GST_OBJECT_FLAG_SET (qtdemux, GST_ELEMENT_FLAG_INDEXABLE);
Artyom Baginski's avatar
Artyom Baginski committed
646
647
}

648
649
650
651
652
653
654
655
656
static void
gst_qtdemux_dispose (GObject * object)
{
  GstQTDemux *qtdemux = GST_QTDEMUX (object);

  if (qtdemux->adapter) {
    g_object_unref (G_OBJECT (qtdemux->adapter));
    qtdemux->adapter = NULL;
  }
Thiago Santos's avatar
Thiago Santos committed
657
  gst_flow_combiner_free (qtdemux->flowcombiner);
658
659
660
  g_queue_foreach (&qtdemux->protection_event_queue, (GFunc) gst_event_unref,
      NULL);
  g_queue_clear (&qtdemux->protection_event_queue);
661

662
663
664
  g_free (qtdemux->cenc_aux_info_sizes);
  qtdemux->cenc_aux_info_sizes = NULL;

665
  G_OBJECT_CLASS (parent_class)->dispose (object);
666
667
}

668
669
670
671
static void
gst_qtdemux_post_no_playable_stream_error (GstQTDemux * qtdemux)
{
  if (qtdemux->posted_redirect) {
672
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
673
674
675
        (_("This file contains no playable streams.")),
        ("no known streams found, a redirect message has been posted"));
  } else {
676
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
677
678
679
680
681
        (_("This file contains no playable streams.")),
        ("no known streams found"));
  }
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
682
683
684
static GstBuffer *
_gst_buffer_new_wrapped (gpointer mem, gsize size, GFreeFunc free_func)
{
Wim Taymans's avatar
Wim Taymans committed
685
686
  return gst_buffer_new_wrapped_full (free_func ? 0 : GST_MEMORY_FLAG_READONLY,
      mem, size, 0, size, mem, free_func);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
687
688
}

689
690
691
692
693
static GstFlowReturn
gst_qtdemux_pull_atom (GstQTDemux * qtdemux, guint64 offset, guint64 size,
    GstBuffer ** buf)
{
  GstFlowReturn flow;
Wim Taymans's avatar
Wim Taymans committed
694
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
695
  gsize bsize;
696

697
  if (G_UNLIKELY (size == 0)) {
698
699
700
701
702
703
704
    GstFlowReturn ret;
    GstBuffer *tmp = NULL;

    ret = gst_qtdemux_pull_atom (qtdemux, offset, sizeof (guint32), &tmp);
    if (ret != GST_FLOW_OK)
      return ret;

Wim Taymans's avatar
Wim Taymans committed
705
706
    gst_buffer_map (tmp, &map, GST_MAP_READ);
    size = QT_UINT32 (map.data);
707
    GST_DEBUG_OBJECT (qtdemux, "size 0x%08" G_GINT64_MODIFIER "x", size);
708

Wim Taymans's avatar
Wim Taymans committed
709
    gst_buffer_unmap (tmp, &map);
710
711
712
    gst_buffer_unref (tmp);
  }

713
714
  /* Sanity check: catch bogus sizes (fuzzed/broken files) */
  if (G_UNLIKELY (size > QTDEMUX_MAX_ATOM_SIZE)) {
715
716
717
718
719
    if (qtdemux->state != QTDEMUX_STATE_MOVIE && qtdemux->got_moov) {
      /* we're pulling header but already got most interesting bits,
       * so never mind the rest (e.g. tags) (that much) */
      GST_WARNING_OBJECT (qtdemux, "atom has bogus size %" G_GUINT64_FORMAT,
          size);
720
      return GST_FLOW_EOS;
721
722
723
724
725
726
    } else {
      GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
          (_("This file is invalid and cannot be played.")),
          ("atom has bogus size %" G_GUINT64_FORMAT, size));
      return GST_FLOW_ERROR;
    }
727
728
729
730
731
732
733
  }

  flow = gst_pad_pull_range (qtdemux->sinkpad, offset, size, buf);

  if (G_UNLIKELY (flow != GST_FLOW_OK))
    return flow;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
734
  bsize = gst_buffer_get_size (*buf);
735
  /* Catch short reads - we don't want any partial atoms */
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
736
  if (G_UNLIKELY (bsize < size)) {
737
738
    GST_WARNING_OBJECT (qtdemux,
        "short read: %" G_GSIZE_FORMAT " < %" G_GUINT64_FORMAT, bsize, size);
739
740
    gst_buffer_unref (*buf);
    *buf = NULL;
741
    return GST_FLOW_EOS;
742
743
744
745
746
  }

  return flow;
}

747
#if 1
748
static gboolean
749
750
751
gst_qtdemux_src_convert (GstQTDemux * qtdemux, GstPad * pad,
    GstFormat src_format, gint64 src_value, GstFormat dest_format,
    gint64 * dest_value)
752
753
{
  gboolean res = TRUE;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
754
  QtDemuxStream *stream = gst_pad_get_element_private (pad);
755
  gint32 index;
756

Andy Wingo's avatar
Andy Wingo committed
757
758
759
760
  if (stream->subtype != FOURCC_vide) {
    res = FALSE;
    goto done;
  }
761
762
763

  switch (src_format) {
    case GST_FORMAT_TIME:
764
765
766
      switch (dest_format) {
        case GST_FORMAT_BYTES:{
          index = gst_qtdemux_find_index_linear (qtdemux, stream, src_value);
767
768
769
770
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
771
772
773
774
775
776

          *dest_value = stream->samples[index].offset;

          GST_DEBUG_OBJECT (qtdemux, "Format Conversion Time->Offset :%"
              GST_TIME_FORMAT "->%" G_GUINT64_FORMAT,
              GST_TIME_ARGS (src_value), *dest_value);
777
          break;
778
        }
779
780
781
        default:
          res = FALSE;
          break;
782
783
784
      }
      break;
    case GST_FORMAT_BYTES:
785
786
787
788
789
790
      switch (dest_format) {
        case GST_FORMAT_TIME:{
          index =
              gst_qtdemux_find_index_for_given_media_offset_linear (qtdemux,
              stream, src_value);

791
792
793
794
          if (-1 == index) {
            res = FALSE;
            goto done;
          }
795
796

          *dest_value =
797
798
799
800
801
              QTSTREAMTIME_TO_GSTTIME (stream,
              stream->samples[index].timestamp);
          GST_DEBUG_OBJECT (qtdemux,
              "Format Conversion Offset->Time :%" G_GUINT64_FORMAT "->%"
              GST_TIME_FORMAT, src_value, GST_TIME_ARGS (*dest_value));
802
          break;
803
        }
804
805
806
        default:
          res = FALSE;
          break;
807
808
809
810
      }
      break;
    default:
      res = FALSE;
811
      break;
812
813
  }

Andy Wingo's avatar
Andy Wingo committed
814
done:
815
816
  return res;
}
817
#endif
818

819
static gboolean
820
gst_qtdemux_get_duration (GstQTDemux * qtdemux, GstClockTime * duration)
821
{
822
  gboolean res = FALSE;
823
824
825

  *duration = GST_CLOCK_TIME_NONE;

826
827
828
829
830
831
  if (qtdemux->duration != 0 &&
      qtdemux->duration != G_MAXINT64 && qtdemux->timescale != 0) {
    *duration = QTTIME_TO_GSTTIME (qtdemux, qtdemux->duration);
    res = TRUE;
  } else {
    *duration = GST_CLOCK_TIME_NONE;
832
  }
833

834
835
836
  return res;
}

837
static gboolean
Wim Taymans's avatar
Wim Taymans committed
838
839
gst_qtdemux_handle_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
840
{
841
  gboolean res = FALSE;
Wim Taymans's avatar
Wim Taymans committed
842
  GstQTDemux *qtdemux = GST_QTDEMUX (parent);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
843

844
845
  GST_LOG_OBJECT (pad, "%s query", GST_QUERY_TYPE_NAME (query));

846
  switch (GST_QUERY_TYPE (query)) {
847
848
849
850
851
852
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME
          && GST_CLOCK_TIME_IS_VALID (qtdemux->segment.position)) {
853
        gst_query_set_position (query, GST_FORMAT_TIME,
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
854
            qtdemux->segment.position);
Wim Taymans's avatar
Wim Taymans committed
855
856
        res = TRUE;
      }
857
    }
Wim Taymans's avatar
Wim Taymans committed
858
      break;
859
860
    case GST_QUERY_DURATION:{
      GstFormat fmt;
861

862
863
      gst_query_parse_duration (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME) {
864
865
866
        /* First try to query upstream */
        res = gst_pad_query_default (pad, parent, query);
        if (!res) {
867
          GstClockTime duration;
868
          if (gst_qtdemux_get_duration (qtdemux, &duration) && duration > 0) {
869
870
871
            gst_query_set_duration (query, GST_FORMAT_TIME, duration);
            res = TRUE;
          }
872
873
874
        }
      }
      break;
875
    }
876
877
    case GST_QUERY_CONVERT:{
      GstFormat src_fmt, dest_fmt;
878
      gint64 src_value, dest_value = 0;
879
880
881

      gst_query_parse_convert (query, &src_fmt, &src_value, &dest_fmt, NULL);

882
      res = gst_qtdemux_src_convert (qtdemux, pad,
883
884
885
886
887
888
889
890
891
892
893
          src_fmt, src_value, dest_fmt, &dest_value);
      if (res) {
        gst_query_set_convert (query, src_fmt, src_value, dest_fmt, dest_value);
        res = TRUE;
      }
      break;
    }
    case GST_QUERY_FORMATS:
      gst_query_set_formats (query, 2, GST_FORMAT_TIME, GST_FORMAT_BYTES);
      res = TRUE;
      break;
894
895
    case GST_QUERY_SEEKING:{
      GstFormat fmt;
896
      gboolean seekable;
897

Thiago Santos's avatar
Thiago Santos committed
898
899
900
901
902
903
      /* try upstream first */
      res = gst_pad_query_default (pad, parent, query);

      if (!res) {
        gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
        if (fmt == GST_FORMAT_TIME) {
904
          GstClockTime duration;
Thiago Santos's avatar
Thiago Santos committed
905
906
907
908
909
910
911
912
913
914
915
916
917
918

          gst_qtdemux_get_duration (qtdemux, &duration);
          seekable = TRUE;
          if (!qtdemux->pullbased) {
            GstQuery *q;

            /* we might be able with help from upstream */
            seekable = FALSE;
            q = gst_query_new_seeking (GST_FORMAT_BYTES);
            if (gst_pad_peer_query (qtdemux->sinkpad, q)) {
              gst_query_parse_seeking (q, &fmt, &seekable, NULL, NULL);
              GST_LOG_OBJECT (qtdemux, "upstream BYTE seekable %d", seekable);
            }
            gst_query_unref (q);
919
          }
Thiago Santos's avatar
Thiago Santos committed
920
921
          gst_query_set_seeking (query, GST_FORMAT_TIME, seekable, 0, duration);
          res = TRUE;
922
        }
923
      }
924
      break;
925
    }
Wim Taymans's avatar
Wim Taymans committed
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
    case GST_QUERY_SEGMENT:
    {
      GstFormat format;
      gint64 start, stop;

      format = qtdemux->segment.format;

      start =
          gst_segment_to_stream_time (&qtdemux->segment, format,
          qtdemux->segment.start);
      if ((stop = qtdemux->segment.stop) == -1)
        stop = qtdemux->segment.duration;
      else
        stop = gst_segment_to_stream_time (&qtdemux->segment, format, stop);

      gst_query_set_segment (query, qtdemux->segment.rate, format, start, stop);
      res = TRUE;
      break;
    }
945
    default:
Wim Taymans's avatar
Wim Taymans committed
946
      res = gst_pad_query_default (pad, parent, query);
947
948
949
950
951
952
      break;
  }

  return res;
}

953
954
955
956
957
958
959
960
961
962
963
static void
gst_qtdemux_push_tags (GstQTDemux * qtdemux, QtDemuxStream * stream)
{
  if (G_LIKELY (stream->pad)) {
    GST_DEBUG_OBJECT (qtdemux, "Checking pad %s:%s for tags",
        GST_DEBUG_PAD_NAME (stream->pad));

    if (G_UNLIKELY (stream->pending_tags)) {
      GST_DEBUG_OBJECT (qtdemux, "Sending tags %" GST_PTR_FORMAT,
          stream->pending_tags);
      gst_pad_push_event (stream->pad,
964
          gst_event_new_tag (stream->pending_tags));
965
966
967
968
969
970
971
      stream->pending_tags = NULL;
    }

    if (G_UNLIKELY (stream->send_global_tags && qtdemux->tag_list)) {
      GST_DEBUG_OBJECT (qtdemux, "Sending global tags %" GST_PTR_FORMAT,
          qtdemux->tag_list);
      gst_pad_push_event (stream->pad,
972
          gst_event_new_tag (gst_tag_list_ref (qtdemux->tag_list)));
973
974
975
976
977
      stream->send_global_tags = FALSE;
    }
  }
}