qtdemux.c 411 KB
Newer Older
Artyom Baginski's avatar
Artyom Baginski committed
1
2
/* GStreamer
 * Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
3
 * Copyright (C) <2003> David A. Schleef <ds@schleef.org>
4
 * Copyright (C) <2006> Wim Taymans <wim@fluendo.com>
5
 * Copyright (C) <2007> Julien Moutte <julien@fluendo.com>
6
 * Copyright (C) <2009> Tim-Philipp Müller <tim centricular net>
7
 * Copyright (C) <2009> STEricsson <benjamin.gaignard@stericsson.com>
8
 * Copyright (C) <2013> Sreerenj Balachandran <sreerenj.balachandran@intel.com>
9
10
 * Copyright (C) <2013> Intel Corporation
 * Copyright (C) <2014> Centricular Ltd
11
 * Copyright (C) <2015> YouView TV Ltd.
12
 *
Artyom Baginski's avatar
Artyom Baginski committed
13
14
15
16
17
18
19
20
21
22
23
24
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
25
26
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
Artyom Baginski's avatar
Artyom Baginski committed
27
28
 */

29
30
31
32
/**
 * SECTION:element-qtdemux
 *
 * Demuxes a .mov file into raw or compressed audio and/or video streams.
33
 *
34
35
 * This element supports both push and pull-based scheduling, depending on the
 * capabilities of the upstream elements.
36
37
 *
 * <refsect2>
38
 * <title>Example launch line</title>
39
 * |[
40
 * gst-launch-1.0 filesrc location=test.mov ! qtdemux name=demux  demux.audio_0 ! queue ! decodebin ! audioconvert ! audioresample ! autoaudiosink   demux.video_0 ! queue ! decodebin ! videoconvert ! videoscale ! autovideosink
41
 * ]| Play (parse and decode) a .mov file and try to output it to
42
43
44
45
46
47
 * an automatically detected soundcard and videosink. If the MOV file contains
 * compressed audio or video data, this will only work if you have the
 * right decoder elements/plugins installed.
 * </refsect2>
 */

48
49
50
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
51
52
53

#include "gst/gst-i18n-plugin.h"

54
#include <glib/gprintf.h>
55
#include <gst/tag/tag.h>
Wim Taymans's avatar
Wim Taymans committed
56
#include <gst/audio/audio.h>
57
#include <gst/video/video.h>
58

59
#include "qtatomparser.h"
60
61
#include "qtdemux_types.h"
#include "qtdemux_dump.h"
62
#include "fourcc.h"
63
#include "descriptors.h"
64
#include "qtdemux_lang.h"
65
#include "qtdemux.h"
66
#include "qtpalette.h"
67

68
69
70
#include "gst/riff/riff-media.h"
#include "gst/riff/riff-read.h"

71
72
#include <gst/pbutils/pbutils.h>

73
#include <stdio.h>
74
#include <stdlib.h>
Artyom Baginski's avatar
Artyom Baginski committed
75
#include <string.h>
76

77
78
79
#include <math.h>
#include <gst/math-compat.h>

80
81
82
#ifdef HAVE_ZLIB
# include <zlib.h>
#endif
83

84
85
86
/* max. size considered 'sane' for non-mdat atoms */
#define QTDEMUX_MAX_ATOM_SIZE (25*1024*1024)

87
88
89
/* if the sample index is larger than this, something is likely wrong */
#define QTDEMUX_MAX_SAMPLE_INDEX_SIZE (50*1024*1024)

90
91
92
93
94
95
/* For converting qt creation times to unix epoch times */
#define QTDEMUX_SECONDS_PER_DAY (60 * 60 * 24)
#define QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970 17
#define QTDEMUX_SECONDS_FROM_1904_TO_1970 (((1970 - 1904) * (guint64) 365 + \
    QTDEMUX_LEAP_YEARS_FROM_1904_TO_1970) * QTDEMUX_SECONDS_PER_DAY)

96
#define STREAM_IS_EOS(s) (s->time_position == GST_CLOCK_TIME_NONE)
97

98
GST_DEBUG_CATEGORY (qtdemux_debug);
99

100
/*typedef struct _QtNode QtNode; */
101
typedef struct _QtDemuxSegment QtDemuxSegment;
102
typedef struct _QtDemuxSample QtDemuxSample;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
103

104
105
typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo;

106
/*struct _QtNode
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
107
{
108
  guint32 type;
109
  guint8 *data;
110
  gint len;
111
};*/
112

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
113
114
struct _QtDemuxSample
{
115
  guint32 size;
116
  gint32 pts_offset;            /* Add this value to timestamp to get the pts */
117
  guint64 offset;
118
119
  guint64 timestamp;            /* DTS In mov time */
  guint32 duration;             /* In mov time */
120
  gboolean keyframe;            /* TRUE when this packet is a keyframe */
121
122
};

123
124
125
126
127
128
129
/* Macros for converting to/from timescale */
#define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale))
#define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND))

#define QTTIME_TO_GSTTIME(qtdemux, value) (gst_util_uint64_scale((value), GST_SECOND, (qtdemux)->timescale))
#define GSTTIME_TO_QTTIME(qtdemux, value) (gst_util_uint64_scale((value), (qtdemux)->timescale, GST_SECOND))

130
/* timestamp is the DTS */
131
#define QTSAMPLE_DTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp))
132
/* timestamp + offset is the PTS */
Nicolas Dufresne's avatar
Nicolas Dufresne committed
133
#define QTSAMPLE_PTS(stream,sample) (QTSTREAMTIME_TO_GSTTIME((stream), (sample)->timestamp + (stream)->cslg_shift + (sample)->pts_offset))
134
/* timestamp + duration - dts is the duration */
135
#define QTSAMPLE_DUR_DTS(stream, sample, dts) (QTSTREAMTIME_TO_GSTTIME ((stream), (sample)->timestamp + (sample)->duration) - (dts))
136
137
138

#define QTSAMPLE_KEYFRAME(stream,sample) ((stream)->all_keyframe || (sample)->keyframe)

139
140
141
142
143
144
145
146
147
148
149
/*
 * Quicktime has tracks and segments. A track is a continuous piece of
 * multimedia content. The track is not always played from start to finish but
 * instead, pieces of the track are 'cut out' and played in sequence. This is
 * what the segments do.
 *
 * Inside the track we have keyframes (K) and delta frames. The track has its
 * own timing, which starts from 0 and extends to end. The position in the track
 * is called the media_time.
 *
 * The segments now describe the pieces that should be played from this track
150
 * and are basically tuples of media_time/duration/rate entries. We can have
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 * multiple segments and they are all played after one another. An example:
 *
 * segment 1: media_time: 1 second, duration: 1 second, rate 1
 * segment 2: media_time: 3 second, duration: 2 second, rate 2
 *
 * To correctly play back this track, one must play: 1 second of media starting
 * from media_time 1 followed by 2 seconds of media starting from media_time 3
 * at a rate of 2.
 *
 * Each of the segments will be played at a specific time, the first segment at
 * time 0, the second one after the duration of the first one, etc.. Note that
 * the time in resulting playback is not identical to the media_time of the
 * track anymore.
 *
 * Visually, assuming the track has 4 second of media_time:
 *
 *                (a)                   (b)          (c)              (d)
 *         .-----------------------------------------------------------.
 * track:  | K.....K.........K........K.......K.......K...........K... |
 *         '-----------------------------------------------------------'
Stefan Kost's avatar
Stefan Kost committed
171
 *         0              1              2              3              4
172
173
174
175
176
177
 *           .------------^              ^   .----------^              ^
 *          /              .-------------'  /       .------------------'
 *         /              /          .-----'       /
 *         .--------------.         .--------------.
 *         | segment 1    |         | segment 2    |
 *         '--------------'         '--------------'
Stefan Kost's avatar
Stefan Kost committed
178
 *
179
 * The challenge here is to cut out the right pieces of the track for each of
180
181
 * the playback segments. This fortunately can easily be done with the SEGMENT
 * events of GStreamer.
182
183
184
185
186
187
188
189
 *
 * For playback of segment 1, we need to provide the decoder with the keyframe
 * (a), in the above figure, but we must instruct it only to output the decoded
 * data between second 1 and 2. We do this with a SEGMENT event for 1 to 2, time
 * position set to the time of the segment: 0.
 *
 * We then proceed to push data from keyframe (a) to frame (b). The decoder
 * decodes but clips all before media_time 1.
Stefan Kost's avatar
Stefan Kost committed
190
 *
191
192
193
194
195
196
 * After finishing a segment, we push out a new SEGMENT event with the clipping
 * boundaries of the new data.
 *
 * This is a good usecase for the GStreamer accumulated SEGMENT events.
 */

197
198
199
struct _QtDemuxSegment
{
  /* global time and duration, all gst time */
200
201
202
  GstClockTime time;
  GstClockTime stop_time;
  GstClockTime duration;
203
  /* media time of trak, all gst time */
204
205
  GstClockTime media_start;
  GstClockTime media_stop;
206
  gdouble rate;
207
208
  /* Media start time in trak timescale units */
  guint32 trak_media_start;
209
210
};

211
212
#define QTSEGMENT_IS_EMPTY(s) ((s)->media_start == GST_CLOCK_TIME_NONE)

213
214
215
216
217
218
219
/* Used with fragmented MP4 files (mfra atom) */
typedef struct
{
  GstClockTime ts;
  guint64 moof_offset;
} QtDemuxRandomAccessEntry;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
220
221
struct _QtDemuxStream
{
222
223
224
  GstPad *pad;

  /* stream type */
225
226
  guint32 subtype;
  GstCaps *caps;
227
  guint32 fourcc;
228
  gboolean sparse;
229

230
  gboolean new_caps;
231
  gboolean new_stream;          /* signals that a stream_start is required */
232
233
234
235
  gboolean on_keyframe;         /* if this stream last pushed buffer was a
                                 * keyframe. This is important to identify
                                 * where to stop pushing buffers after a
                                 * segment stop time */
236

237
238
239
  /* if the stream has a redirect URI in its headers, we store it here */
  gchar *redirect_uri;

240
241
242
  /* track id */
  guint track_id;

243
  /* duration/scale */
244
  guint64 duration;             /* in timescale */
245
  guint32 timescale;
246

247
  /* language */
248
  gchar lang_id[4];             /* ISO 639-2T language code */
249

250
251
252
253
  /* our samples */
  guint32 n_samples;
  QtDemuxSample *samples;
  gboolean all_keyframe;        /* TRUE when all samples are keyframes (no stss) */
254
  guint32 first_duration;       /* duration in timescale of first sample, used for figuring out
255
                                   the framerate, in timescale units */
256
257
258
  guint32 n_samples_moof;       /* sample count in a moof */
  guint64 duration_moof;        /* duration in timescale of a moof, used for figure out
                                 * the framerate of fragmented format stream */
259
260
  guint32 offset_in_sample;
  guint32 max_buffer_size;
261

262
263
  /* if we use chunks or samples */
  gboolean sampled;
264
  guint padding;
265
266

  /* video info */
267
268
  gint width;
  gint height;
269
270
271
272
273
  /* aspect ratio */
  gint display_width;
  gint display_height;
  gint par_w;
  gint par_h;
274
275
276
  /* Numerator/denominator framerate */
  gint fps_n;
  gint fps_d;
277
278
  guint16 bits_per_sample;
  guint16 color_table_id;
279
  GstMemory *rgb8_palette;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
280

281
  /* audio info */
282
283
  gdouble rate;
  gint n_channels;
284
285
286
287
  guint samples_per_packet;
  guint samples_per_frame;
  guint bytes_per_packet;
  guint bytes_per_sample;
288
  guint bytes_per_frame;
289
  guint compression;
290

291
292
293
294
295
  /* allocation */
  gboolean use_allocator;
  GstAllocator *allocator;
  GstAllocationParams params;

296
297
  /* when a discontinuity is pending */
  gboolean discont;
298

299
300
301
  /* list of buffers to push first */
  GSList *buffers;

302
303
304
305
  /* if we need to clip this buffer. This is only needed for uncompressed
   * data */
  gboolean need_clip;

306
307
308
  /* buffer needs some custom processing, e.g. subtitles */
  gboolean need_process;

309
310
311
  /* current position */
  guint32 segment_index;
  guint32 sample_index;
312
  GstClockTime time_position;   /* in gst time */
313
  guint64 accumulated_base;
314

315
316
  /* the Gst segment we are processing out, used for clipping */
  GstSegment segment;
317
  guint32 segment_seqnum;       /* segment event seqnum obtained from seek */
318

319
320
321
  /* quicktime segments */
  guint32 n_segments;
  QtDemuxSegment *segments;
322
  gboolean dummy_segment;
323
324
  guint32 from_sample;
  guint32 to_sample;
325
326

  gboolean sent_eos;
327
328
  GstTagList *pending_tags;
  gboolean send_global_tags;
329
330

  GstEvent *pending_event;
331
332
333
334
335
336
337
338
339

  GstByteReader stco;
  GstByteReader stsz;
  GstByteReader stsc;
  GstByteReader stts;
  GstByteReader stss;
  GstByteReader stps;
  GstByteReader ctts;

340
  gboolean chunks_are_samples;  /* TRUE means treat chunks as samples */
341
  gint64 stbl_index;
342
343
  /* stco */
  guint co_size;
344
345
346
347
348
349
  GstByteReader co_chunk;
  guint32 first_chunk;
  guint32 current_chunk;
  guint32 last_chunk;
  guint32 samples_per_chunk;
  guint32 stco_sample_index;
350
351
352
  /* stsz */
  guint32 sample_size;          /* 0 means variable sizes are stored in stsz */
  /* stsc */
353
  guint32 stsc_index;
354
  guint32 n_samples_per_chunk;
355
356
357
  guint32 stsc_chunk_index;
  guint32 stsc_sample_index;
  guint64 chunk_offset;
358
  /* stts */
359
360
  guint32 stts_index;
  guint32 stts_samples;
361
  guint32 n_sample_times;
362
  guint32 stts_sample_index;
363
  guint64 stts_time;
364
  guint32 stts_duration;
365
  /* stss */
366
  gboolean stss_present;
367
  guint32 n_sample_syncs;
368
  guint32 stss_index;
369
  /* stps */
370
  gboolean stps_present;
371
  guint32 n_sample_partial_syncs;
372
  guint32 stps_index;
373
374
375
  QtDemuxRandomAccessEntry *ra_entries;
  guint n_ra_entries;

376
377
  const QtDemuxRandomAccessEntry *pending_seek;

378
379
380
  /* ctts */
  gboolean ctts_present;
  guint32 n_composition_times;
381
382
383
384
  guint32 ctts_index;
  guint32 ctts_sample_index;
  guint32 ctts_count;
  gint32 ctts_soffset;
385

Nicolas Dufresne's avatar
Nicolas Dufresne committed
386
387
388
  /* cslg */
  guint32 cslg_shift;

389
390
391
392
393
  /* fragmented */
  gboolean parsed_trex;
  guint32 def_sample_duration;
  guint32 def_sample_size;
  guint32 def_sample_flags;
394
395

  gboolean disabled;
396
397
398
399

  /* stereoscopic video streams */
  GstVideoMultiviewMode multiview_mode;
  GstVideoMultiviewFlags multiview_flags;
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416

  /* protected streams */
  gboolean protected;
  guint32 protection_scheme_type;
  guint32 protection_scheme_version;
  gpointer protection_scheme_info;      /* specific to the protection scheme */
  GQueue protection_scheme_event_queue;
};

/* Contains properties and cryptographic info for a set of samples from a
 * track protected using Common Encryption (cenc) */
struct _QtDemuxCencSampleSetInfo
{
  GstStructure *default_properties;

  /* @crypto_info holds one GstStructure per sample */
  GPtrArray *crypto_info;
417
418
};

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
419
420
enum QtDemuxState
{
421
422
423
  QTDEMUX_STATE_INITIAL,        /* Initial state (haven't got the header yet) */
  QTDEMUX_STATE_HEADER,         /* Parsing the header */
  QTDEMUX_STATE_MOVIE,          /* Parsing/Playing the media data */
424
  QTDEMUX_STATE_BUFFER_MDAT     /* Buffering the mdat atom */
425
426
};

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
427
static GNode *qtdemux_tree_get_child_by_type (GNode * node, guint32 fourcc);
428
static GNode *qtdemux_tree_get_child_by_type_full (GNode * node,
429
    guint32 fourcc, GstByteReader * parser);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
430
static GNode *qtdemux_tree_get_sibling_by_type (GNode * node, guint32 fourcc);
431
432
static GNode *qtdemux_tree_get_sibling_by_type_full (GNode * node,
    guint32 fourcc, GstByteReader * parser);
433

434
435
static GstFlowReturn qtdemux_add_fragmented_samples (GstQTDemux * qtdemux);

David Schleef's avatar
David Schleef committed
436
static GstStaticPadTemplate gst_qtdemux_sink_template =
437
    GST_STATIC_PAD_TEMPLATE ("sink",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
438
    GST_PAD_SINK,
439
    GST_PAD_ALWAYS,
440
441
    GST_STATIC_CAPS ("video/quicktime; video/mj2; audio/x-m4a; "
        "application/x-3gp")
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
442
    );
David Schleef's avatar
David Schleef committed
443
444

static GstStaticPadTemplate gst_qtdemux_videosrc_template =
Wim Taymans's avatar
Wim Taymans committed
445
GST_STATIC_PAD_TEMPLATE ("video_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
446
447
448
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
David Schleef's avatar
David Schleef committed
449
450

static GstStaticPadTemplate gst_qtdemux_audiosrc_template =
Wim Taymans's avatar
Wim Taymans committed
451
GST_STATIC_PAD_TEMPLATE ("audio_%u",
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
452
453
454
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);
Artyom Baginski's avatar
Artyom Baginski committed
455

456
static GstStaticPadTemplate gst_qtdemux_subsrc_template =
Wim Taymans's avatar
Wim Taymans committed
457
GST_STATIC_PAD_TEMPLATE ("subtitle_%u",
458
459
460
461
    GST_PAD_SRC,
    GST_PAD_SOMETIMES,
    GST_STATIC_CAPS_ANY);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
462
463
#define gst_qtdemux_parent_class parent_class
G_DEFINE_TYPE (GstQTDemux, gst_qtdemux, GST_TYPE_ELEMENT);
Artyom Baginski's avatar
Artyom Baginski committed
464

465
static void gst_qtdemux_dispose (GObject * object);
466

467
468
static guint32
gst_qtdemux_find_index_linear (GstQTDemux * qtdemux, QtDemuxStream * str,
469
    GstClockTime media_time);
470
471
472
473
static guint32
gst_qtdemux_find_index_for_given_media_offset_linear (GstQTDemux * qtdemux,
    QtDemuxStream * str, gint64 media_offset);

474
#if 0
475
476
static void gst_qtdemux_set_index (GstElement * element, GstIndex * index);
static GstIndex *gst_qtdemux_get_index (GstElement * element);
477
#endif
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
478
479
static GstStateChangeReturn gst_qtdemux_change_state (GstElement * element,
    GstStateChange transition);
Wim Taymans's avatar
Wim Taymans committed
480
static gboolean qtdemux_sink_activate (GstPad * sinkpad, GstObject * parent);
Wim Taymans's avatar
Wim Taymans committed
481
482
static gboolean qtdemux_sink_activate_mode (GstPad * sinkpad,
    GstObject * parent, GstPadMode mode, gboolean active);
483
484

static void gst_qtdemux_loop (GstPad * pad);
Wim Taymans's avatar
Wim Taymans committed
485
486
487
488
static GstFlowReturn gst_qtdemux_chain (GstPad * sinkpad, GstObject * parent,
    GstBuffer * inbuf);
static gboolean gst_qtdemux_handle_sink_event (GstPad * pad, GstObject * parent,
    GstEvent * event);
Thiago Santos's avatar
Thiago Santos committed
489
static gboolean gst_qtdemux_setcaps (GstQTDemux * qtdemux, GstCaps * caps);
490
491
static gboolean gst_qtdemux_configure_stream (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
492
493
static GstFlowReturn gst_qtdemux_process_adapter (GstQTDemux * demux,
    gboolean force);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
494

495
496
static gboolean qtdemux_parse_moov (GstQTDemux * qtdemux,
    const guint8 * buffer, guint length);
497
static gboolean qtdemux_parse_node (GstQTDemux * qtdemux, GNode * node,
498
    const guint8 * buffer, guint length);
499
static gboolean qtdemux_parse_tree (GstQTDemux * qtdemux);
Thiago Santos's avatar
Thiago Santos committed
500
501
static void qtdemux_parse_udta (GstQTDemux * qtdemux, GstTagList * taglist,
    GNode * udta);
502

503
static void gst_qtdemux_handle_esds (GstQTDemux * qtdemux,
504
    QtDemuxStream * stream, GNode * esds, GstTagList * list);
505
506
static GstCaps *qtdemux_video_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
507
    gchar ** codec_name);
508
509
static GstCaps *qtdemux_audio_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data, int len,
510
    gchar ** codec_name);
511
static GstCaps *qtdemux_sub_caps (GstQTDemux * qtdemux,
512
513
    QtDemuxStream * stream, guint32 fourcc, const guint8 * data,
    gchar ** codec_name);
514
515
516
517
static GstCaps *qtdemux_generic_caps (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 fourcc, const guint8 * stsd_data,
    gchar ** codec_name);

518
519
static gboolean qtdemux_parse_samples (GstQTDemux * qtdemux,
    QtDemuxStream * stream, guint32 n);
520
static GstFlowReturn qtdemux_expose_streams (GstQTDemux * qtdemux);
521
522
static void gst_qtdemux_stream_free (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thiago Santos's avatar
Thiago Santos committed
523
524
static void gst_qtdemux_stream_clear (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Arnaud Vrac's avatar
Arnaud Vrac committed
525
static void gst_qtdemux_remove_stream (GstQTDemux * qtdemux, int index);
526
static GstFlowReturn qtdemux_prepare_streams (GstQTDemux * qtdemux);
527
528
static void qtdemux_do_allocation (GstQTDemux * qtdemux,
    QtDemuxStream * stream);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
529

530
static gboolean qtdemux_pull_mfro_mfra (GstQTDemux * qtdemux);
531
static void check_update_duration (GstQTDemux * qtdemux, GstClockTime duration);
532

533
534
535
static void gst_qtdemux_append_protection_system_id (GstQTDemux * qtdemux,
    const gchar * id);

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
536
537
static void
gst_qtdemux_class_init (GstQTDemuxClass * klass)
Artyom Baginski's avatar
Artyom Baginski committed
538
539
540
541
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
542
543
  gobject_class = (GObjectClass *) klass;
  gstelement_class = (GstElementClass *) klass;
Artyom Baginski's avatar
Artyom Baginski committed
544

545
  parent_class = g_type_class_peek_parent (klass);
546

547
548
  gobject_class->dispose = gst_qtdemux_dispose;

549
  gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state);
550
#if 0
551
552
  gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index);
  gstelement_class->get_index = GST_DEBUG_FUNCPTR (gst_qtdemux_get_index);
553
#endif
554
555

  gst_tag_register_musicbrainz_tags ();
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
556
557
558
559
560
561
562
563
564

  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&gst_qtdemux_sink_template));
  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&gst_qtdemux_videosrc_template));
  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&gst_qtdemux_audiosrc_template));
  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&gst_qtdemux_subsrc_template));
565
  gst_element_class_set_static_metadata (gstelement_class, "QuickTime demuxer",
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
566
567
568
569
570
571
      "Codec/Demuxer",
      "Demultiplex a QuickTime file into audio and video streams",
      "David Schleef <ds@schleef.org>, Wim Taymans <wim@fluendo.com>");

  GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin");

Artyom Baginski's avatar
Artyom Baginski committed
572
573
}

Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
574
static void
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
575
gst_qtdemux_init (GstQTDemux * qtdemux)
Artyom Baginski's avatar
Artyom Baginski committed
576
{
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
577
  qtdemux->sinkpad =
578
      gst_pad_new_from_static_template (&gst_qtdemux_sink_template, "sink");
579
  gst_pad_set_activate_function (qtdemux->sinkpad, qtdemux_sink_activate);
Wim Taymans's avatar
Wim Taymans committed
580
581
  gst_pad_set_activatemode_function (qtdemux->sinkpad,
      qtdemux_sink_activate_mode);
582
583
  gst_pad_set_chain_function (qtdemux->sinkpad, gst_qtdemux_chain);
  gst_pad_set_event_function (qtdemux->sinkpad, gst_qtdemux_handle_sink_event);
584
  gst_element_add_pad (GST_ELEMENT_CAST (qtdemux), qtdemux->sinkpad);
585

586
587
  qtdemux->state = QTDEMUX_STATE_INITIAL;
  qtdemux->pullbased = FALSE;
588
  qtdemux->posted_redirect = FALSE;
589
  qtdemux->pending_configure = FALSE;
590
591
592
  qtdemux->neededbytes = 16;
  qtdemux->todrop = 0;
  qtdemux->adapter = gst_adapter_new ();
593
  qtdemux->offset = 0;
594
  qtdemux->first_mdat = -1;
595
  qtdemux->got_moov = FALSE;
596
  qtdemux->mdatoffset = -1;
597
  qtdemux->mdatbuffer = NULL;
598
  qtdemux->restoredata_buffer = NULL;
599
  qtdemux->restoredata_offset = -1;
600
  qtdemux->fragment_start = -1;
601
  qtdemux->fragment_start_offset = -1;
602
603
604
  qtdemux->media_caps = NULL;
  qtdemux->exposed = FALSE;
  qtdemux->mss_mode = FALSE;
605
  qtdemux->pending_newsegment = NULL;
606
  qtdemux->upstream_format_is_time = FALSE;
607
608
  qtdemux->have_group_id = FALSE;
  qtdemux->group_id = G_MAXUINT;
609
610
611
  qtdemux->cenc_aux_info_offset = 0;
  qtdemux->cenc_aux_info_sizes = NULL;
  qtdemux->cenc_aux_sample_count = 0;
612
613
  qtdemux->protection_system_ids = NULL;
  g_queue_init (&qtdemux->protection_event_queue);
614
  gst_segment_init (&qtdemux->segment, GST_FORMAT_TIME);
Thiago Santos's avatar
Thiago Santos committed
615
  qtdemux->flowcombiner = gst_flow_combiner_new ();
Wim Taymans's avatar
Wim Taymans committed
616
617

  GST_OBJECT_FLAG_SET (qtdemux, GST_ELEMENT_FLAG_INDEXABLE);
Artyom Baginski's avatar
Artyom Baginski committed
618
619
}

620
621
622
623
624
625
626
627
628
static void
gst_qtdemux_dispose (GObject * object)
{
  GstQTDemux *qtdemux = GST_QTDEMUX (object);

  if (qtdemux->adapter) {
    g_object_unref (G_OBJECT (qtdemux->adapter));
    qtdemux->adapter = NULL;
  }
Thiago Santos's avatar
Thiago Santos committed
629
  gst_flow_combiner_free (qtdemux->flowcombiner);
630
631
632
  g_queue_foreach (&qtdemux->protection_event_queue, (GFunc) gst_event_unref,
      NULL);
  g_queue_clear (&qtdemux->protection_event_queue);
633

634
635
636
  g_free (qtdemux->cenc_aux_info_sizes);
  qtdemux->cenc_aux_info_sizes = NULL;

637
  G_OBJECT_CLASS (parent_class)->dispose (object);
638
639
}

640
641
642
643
static void
gst_qtdemux_post_no_playable_stream_error (GstQTDemux * qtdemux)
{
  if (qtdemux->posted_redirect) {
644
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
645
646
647
        (_("This file contains no playable streams.")),
        ("no known streams found, a redirect message has been posted"));
  } else {
648
    GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
649
650
651
652
653
        (_("This file contains no playable streams.")),
        ("no known streams found"));
  }
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
654
655
656
static GstBuffer *
_gst_buffer_new_wrapped (gpointer mem, gsize size, GFreeFunc free_func)
{
Wim Taymans's avatar
Wim Taymans committed
657
658
  return gst_buffer_new_wrapped_full (free_func ? 0 : GST_MEMORY_FLAG_READONLY,
      mem, size, 0, size, mem, free_func);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
659
660
}

661
662
663
664
665
static GstFlowReturn
gst_qtdemux_pull_atom (GstQTDemux * qtdemux, guint64 offset, guint64 size,
    GstBuffer ** buf)
{
  GstFlowReturn flow;
Wim Taymans's avatar
Wim Taymans committed
666
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
667
  gsize bsize;
668

669
  if (G_UNLIKELY (size == 0)) {
670
671
672
673
674
675
676
    GstFlowReturn ret;
    GstBuffer *tmp = NULL;

    ret = gst_qtdemux_pull_atom (qtdemux, offset, sizeof (guint32), &tmp);
    if (ret != GST_FLOW_OK)
      return ret;

Wim Taymans's avatar
Wim Taymans committed
677
678
    gst_buffer_map (tmp, &map, GST_MAP_READ);
    size = QT_UINT32 (map.data);
679
    GST_DEBUG_OBJECT (qtdemux, "size 0x%08" G_GINT64_MODIFIER "x", size);
680

Wim Taymans's avatar
Wim Taymans committed
681
    gst_buffer_unmap (tmp, &map);
682
683
684
    gst_buffer_unref (tmp);
  }

685
686
  /* Sanity check: catch bogus sizes (fuzzed/broken files) */
  if (G_UNLIKELY (size > QTDEMUX_MAX_ATOM_SIZE)) {
687
688
689
690
691
    if (qtdemux->state != QTDEMUX_STATE_MOVIE && qtdemux->got_moov) {
      /* we're pulling header but already got most interesting bits,
       * so never mind the rest (e.g. tags) (that much) */
      GST_WARNING_OBJECT (qtdemux, "atom has bogus size %" G_GUINT64_FORMAT,
          size);
692
      return GST_FLOW_EOS;
693
694
695
696
697
698
    } else {
      GST_ELEMENT_ERROR (qtdemux, STREAM, DEMUX,
          (_("This file is invalid and cannot be played.")),
          ("atom has bogus size %" G_GUINT64_FORMAT, size));
      return GST_FLOW_ERROR;
    }
699
700
701
702
703
704
705
  }

  flow = gst_pad_pull_range (qtdemux->sinkpad, offset, size, buf);

  if (G_UNLIKELY (flow != GST_FLOW_OK))
    return flow;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
706
  bsize = gst_buffer_get_size (*buf);
707
  /* Catch short reads - we don't want any partial atoms */
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
708
  if (G_UNLIKELY (bsize < size)) {
709
710
    GST_WARNING_OBJECT (qtdemux,
        "short read: %" G_GSIZE_FORMAT " < %" G_GUINT64_FORMAT, bsize, size);
711
712
    gst_buffer_unref (*buf);
    *buf = NULL;
713
    return GST_FLOW_EOS;
714
715
716
717
718
  }

  return flow;
}

719
#if 1
720
static gboolean
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
721
gst_qtdemux_src_convert (GstPad * pad, GstFormat src_format, gint64 src_value,
722
    GstFormat dest_format, gint64 * dest_value)
723
724
{
  gboolean res = TRUE;
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
725
  QtDemuxStream *stream = gst_pad_get_element_private (pad);
726
727
  GstQTDemux *qtdemux = GST_QTDEMUX (gst_pad_get_parent (pad));
  gint32 index;
728

Andy Wingo's avatar
Andy Wingo committed
729
730
731
732
  if (stream->subtype != FOURCC_vide) {
    res = FALSE;
    goto done;
  }
733
734
735

  switch (src_format) {
    case GST_FORMAT_TIME:
736
737
738
739
740
741
742
743
744
745
746
      switch (dest_format) {
        case GST_FORMAT_BYTES:{
          index = gst_qtdemux_find_index_linear (qtdemux, stream, src_value);
          if (-1 == index)
            return FALSE;

          *dest_value = stream->samples[index].offset;

          GST_DEBUG_OBJECT (qtdemux, "Format Conversion Time->Offset :%"
              GST_TIME_FORMAT "->%" G_GUINT64_FORMAT,
              GST_TIME_ARGS (src_value), *dest_value);
747
          break;
748
        }
749
750
751
        default:
          res = FALSE;
          break;
752
753
754
      }
      break;
    case GST_FORMAT_BYTES:
755
756
757
758
759
760
761
762
763
764
      switch (dest_format) {
        case GST_FORMAT_TIME:{
          index =
              gst_qtdemux_find_index_for_given_media_offset_linear (qtdemux,
              stream, src_value);

          if (-1 == index)
            return FALSE;

          *dest_value =
765
766
767
768
769
              QTSTREAMTIME_TO_GSTTIME (stream,
              stream->samples[index].timestamp);
          GST_DEBUG_OBJECT (qtdemux,
              "Format Conversion Offset->Time :%" G_GUINT64_FORMAT "->%"
              GST_TIME_FORMAT, src_value, GST_TIME_ARGS (*dest_value));
770
          break;
771
        }
772
773
774
        default:
          res = FALSE;
          break;
775
776
777
778
779
780
      }
      break;
    default:
      res = FALSE;
  }

Andy Wingo's avatar
Andy Wingo committed
781
782
783
done:
  gst_object_unref (qtdemux);

784
785
  return res;
}
786
#endif
787

788
static gboolean
789
gst_qtdemux_get_duration (GstQTDemux * qtdemux, GstClockTime * duration)
790
791
792
793
794
795
{
  gboolean res = TRUE;

  *duration = GST_CLOCK_TIME_NONE;

  if (qtdemux->duration != 0) {
796
    if (qtdemux->duration != G_MAXINT64 && qtdemux->timescale != 0) {
797
      *duration = QTTIME_TO_GSTTIME (qtdemux, qtdemux->duration);
798
799
800
801
802
    }
  }
  return res;
}

803
static gboolean
Wim Taymans's avatar
Wim Taymans committed
804
805
gst_qtdemux_handle_src_query (GstPad * pad, GstObject * parent,
    GstQuery * query)
806
{
807
  gboolean res = FALSE;
Wim Taymans's avatar
Wim Taymans committed
808
  GstQTDemux *qtdemux = GST_QTDEMUX (parent);
Thomas Vander Stichele's avatar
Thomas Vander Stichele committed
809

810
811
  GST_LOG_OBJECT (pad, "%s query", GST_QUERY_TYPE_NAME (query));

812
  switch (GST_QUERY_TYPE (query)) {
813
814
815
816
817
818
    case GST_QUERY_POSITION:{
      GstFormat fmt;

      gst_query_parse_position (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME
          && GST_CLOCK_TIME_IS_VALID (qtdemux->segment.position)) {
819
        gst_query_set_position (query, GST_FORMAT_TIME,
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
820
            qtdemux->segment.position);
Wim Taymans's avatar
Wim Taymans committed
821
822
        res = TRUE;
      }
823
    }
Wim Taymans's avatar
Wim Taymans committed
824
      break;
825
826
    case GST_QUERY_DURATION:{
      GstFormat fmt;
827

828
829
      gst_query_parse_duration (query, &fmt, NULL);
      if (fmt == GST_FORMAT_TIME) {
830
831
832
        /* First try to query upstream */
        res = gst_pad_query_default (pad, parent, query);
        if (!res) {
833
834
          guint64 duration;
          if (gst_qtdemux_get_duration (qtdemux, &duration) && duration > 0) {
835
836
837
            gst_query_set_duration (query, GST_FORMAT_TIME, duration);
            res = TRUE;
          }
838
839
840
        }
      }
      break;
841
    }
842
843
    case GST_QUERY_CONVERT:{
      GstFormat src_fmt, dest_fmt;
844
      gint64 src_value, dest_value = 0;
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859

      gst_query_parse_convert (query, &src_fmt, &src_value, &dest_fmt, NULL);

      res = gst_qtdemux_src_convert (pad,
          src_fmt, src_value, dest_fmt, &dest_value);
      if (res) {
        gst_query_set_convert (query, src_fmt, src_value, dest_fmt, dest_value);
        res = TRUE;
      }
      break;
    }
    case GST_QUERY_FORMATS:
      gst_query_set_formats (query, 2, GST_FORMAT_TIME, GST_FORMAT_BYTES);
      res = TRUE;
      break;
860
861
    case GST_QUERY_SEEKING:{
      GstFormat fmt;
862
      gboolean seekable;
863

Thiago Santos's avatar
Thiago Santos committed
864
865
866
867
868
869
      /* try upstream first */
      res = gst_pad_query_default (pad, parent, query);

      if (!res) {
        gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL);
        if (fmt == GST_FORMAT_TIME) {
870
          GstClockTime duration = GST_CLOCK_TIME_NONE;
Thiago Santos's avatar
Thiago Santos committed
871
872
873
874
875
876
877
878
879
880
881
882
883
884

          gst_qtdemux_get_duration (qtdemux, &duration);
          seekable = TRUE;
          if (!qtdemux->pullbased) {
            GstQuery *q;

            /* we might be able with help from upstream */
            seekable = FALSE;
            q = gst_query_new_seeking (GST_FORMAT_BYTES);
            if (gst_pad_peer_query (qtdemux->sinkpad, q)) {
              gst_query_parse_seeking (q, &fmt, &seekable, NULL, NULL);
              GST_LOG_OBJECT (qtdemux, "upstream BYTE seekable %d", seekable);
            }
            gst_query_unref (q);
885
          }
Thiago Santos's avatar
Thiago Santos committed
886
887
          gst_query_set_seeking (query, GST_FORMAT_TIME, seekable, 0, duration);
          res = TRUE;
888
        }
889
      }
890
      break;
891
    }
Wim Taymans's avatar
Wim Taymans committed
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
    case GST_QUERY_SEGMENT:
    {
      GstFormat format;
      gint64 start, stop;

      format = qtdemux->segment.format;

      start =
          gst_segment_to_stream_time (&qtdemux->segment, format,
          qtdemux->segment.start);
      if ((stop = qtdemux->segment.stop) == -1)
        stop = qtdemux->segment.duration;
      else
        stop = gst_segment_to_stream_time (&qtdemux->segment, format, stop);

      gst_query_set_segment (query, qtdemux->segment.rate, format, start, stop);
      res = TRUE;
      break;
    }
911
    default:
Wim Taymans's avatar
Wim Taymans committed
912
      res = gst_pad_query_default (pad, parent, query);
913
914
915
916
917
918
      break;
  }

  return res;
}

919
920
921
922
923
924
925
926
927
928
929
static void
gst_qtdemux_push_tags (GstQTDemux * qtdemux, QtDemuxStream * stream)
{
  if (G_LIKELY (stream->pad)) {
    GST_DEBUG_OBJECT (qtdemux, "Checking pad %s:%s for tags",
        GST_DEBUG_PAD_NAME (stream->pad));

    if (G_UNLIKELY (stream->pending_tags)) {
      GST_DEBUG_OBJECT (qtdemux, "Sending tags %" GST_PTR_FORMAT,
          stream->pending_tags);
      gst_pad_push_event (stream->pad,
930
          gst_event_new_tag (stream->pending_tags));
931
932
933
934
935
936
937
      stream->pending_tags = NULL;
    }

    if (G_UNLIKELY (stream->send_global_tags && qtdemux->tag_list)) {
      GST_DEBUG_OBJECT (qtdemux, "Sending global tags %" GST_PTR_FORMAT,
          qtdemux->tag_list);
      gst_pad_push_event (stream->pad,
938
          gst_event_new_tag (gst_tag_list_ref (qtdemux->tag_list)));
939
940
941
942
943
      stream->send_global_tags = FALSE;
    }
  }
}

944
/* push event on all source pads; takes ownership of the event */
945
static void
946
gst_qtdemux_push_event (GstQTDemux * qtdemux, GstEvent * event)
947
948
{
  guint n;
949
  gboolean has_valid_stream = FALSE;
950
  GstEventType etype = GST_EVENT_TYPE (event);
951
952
953
954
955

  GST_DEBUG_OBJECT (qtdemux, "pushing %s event on all source pads",
      GST_EVENT_TYPE_NAME (event));

  for (n = 0; n < qtdemux->n_streams; n++) {
956
    GstPad *pad;
957
    QtDemuxStream *stream = qtdemux->streams[n];
Thiago Santos's avatar
Thiago Santos committed
958
    GST_DEBUG_OBJECT (qtdemux, "pushing on pad %i", n);
959

960
961
962
    if ((pad = stream->pad)) {
      has_valid_stream = TRUE;

963
964
965
966
      if (etype == GST_EVENT_EOS) {
        /* let's not send twice */
        if (stream->sent_eos)
          continue;
967
        stream->sent_eos = TRUE;
968
      }
969
970

      gst_pad_push_event (pad, gst_event_ref (event));
971
    }
972
  }
973

974
  gst_event_unref (event);
975

976
977
  /* if it is EOS and there are no pads, post an error */
  if (!has_valid_stream && etype == GST_EVENT_EOS) {
978
979
    gst_qtdemux_post_no_playable_stream_error (qtdemux);
  }
980
981
}

982
983
984
985
986
987
988
989
990
991
/* push a pending newsegment event, if any from the streaming thread */
static void
gst_qtdemux_push_pending_newsegment (GstQTDemux * qtdemux)
{
  if (qtdemux->pending_newsegment) {
    gst_qtdemux_push_event (qtdemux, qtdemux->pending_newsegment);
    qtdemux->pending_newsegment = NULL;
  }
}

992
993
994
995
996
997
typedef struct
{
  guint64 media_time;
} FindData;

static gint
998
find_func (QtDemuxSample * s1, gint64 * media_time, gpointer user_data)
999
{