gsth264parse.c 90.2 KB
Newer Older
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1
/* GStreamer H.264 Parser
2
 * Copyright (C) <2010> Collabora ltd
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
3
 * Copyright (C) <2010> Nokia Corporation
4
5
6
7
 * Copyright (C) <2011> Intel Corporation
 *
 * Copyright (C) <2010> Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>
 * Copyright (C) <2011> Thibault Saunier <thibault.saunier@collabora.com>
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
8
9
10
11
12
13
14
15
16
17
18
19
20
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
Tim-Philipp Müller's avatar
Tim-Philipp Müller committed
21
22
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
23
24
25
26
27
28
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

29
30
#include <gst/base/base.h>
#include <gst/pbutils/pbutils.h>
31
#include <gst/video/video.h>
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
32
33
34
35
36
37
38
39
40
41
42
43
#include "gsth264parse.h"

#include <string.h>

GST_DEBUG_CATEGORY (h264_parse_debug);
#define GST_CAT_DEFAULT h264_parse_debug

#define DEFAULT_CONFIG_INTERVAL      (0)

enum
{
  PROP_0,
44
  PROP_CONFIG_INTERVAL
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
45
46
47
48
49
50
};

enum
{
  GST_H264_PARSE_FORMAT_NONE,
  GST_H264_PARSE_FORMAT_AVC,
51
52
  GST_H264_PARSE_FORMAT_BYTE,
  GST_H264_PARSE_FORMAT_AVC3
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
53
54
55
56
57
58
59
60
61
};

enum
{
  GST_H264_PARSE_ALIGN_NONE = 0,
  GST_H264_PARSE_ALIGN_NAL,
  GST_H264_PARSE_ALIGN_AU
};

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
enum
{
  GST_H264_PARSE_STATE_GOT_SPS = 1 << 0,
  GST_H264_PARSE_STATE_GOT_PPS = 1 << 1,
  GST_H264_PARSE_STATE_GOT_SLICE = 1 << 2,

  GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS = (GST_H264_PARSE_STATE_GOT_SPS |
      GST_H264_PARSE_STATE_GOT_PPS),
  GST_H264_PARSE_STATE_VALID_PICTURE =
      (GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS |
      GST_H264_PARSE_STATE_GOT_SLICE)
};

#define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
  (((parse)->state & (expected_state)) == (expected_state))

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
78
79
80
static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
81
    GST_STATIC_CAPS ("video/x-h264"));
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
82
83
84
85

static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
86
    GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, "
87
        "stream-format=(string) { avc, avc3, byte-stream }, "
88
        "alignment=(string) { au, nal }"));
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
89

René Stadler's avatar
René Stadler committed
90
91
#define parent_class gst_h264_parse_parent_class
G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
92
93
94
95
96

static void gst_h264_parse_finalize (GObject * object);

static gboolean gst_h264_parse_start (GstBaseParse * parse);
static gboolean gst_h264_parse_stop (GstBaseParse * parse);
97
98
static GstFlowReturn gst_h264_parse_handle_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame, gint * skipsize);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
99
100
101
102
103
104
105
106
107
108
109
static GstFlowReturn gst_h264_parse_parse_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame);
static GstFlowReturn gst_h264_parse_pre_push_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame);

static void gst_h264_parse_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_h264_parse_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

static gboolean gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps);
René Stadler's avatar
René Stadler committed
110
111
static GstCaps *gst_h264_parse_get_caps (GstBaseParse * parse,
    GstCaps * filter);
112
113
114
static gboolean gst_h264_parse_event (GstBaseParse * parse, GstEvent * event);
static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
    GstEvent * event);
115
116
static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
    GstCaps * caps);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
117
118
119
120
121
122

static void
gst_h264_parse_class_init (GstH264ParseClass * klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstBaseParseClass *parse_class = GST_BASE_PARSE_CLASS (klass);
René Stadler's avatar
René Stadler committed
123
124
125
  GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);

  GST_DEBUG_CATEGORY_INIT (h264_parse_debug, "h264parse", 0, "h264 parser");
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

  gobject_class->finalize = gst_h264_parse_finalize;
  gobject_class->set_property = gst_h264_parse_set_property;
  gobject_class->get_property = gst_h264_parse_get_property;

  g_object_class_install_property (gobject_class, PROP_CONFIG_INTERVAL,
      g_param_spec_uint ("config-interval",
          "SPS PPS Send Interval",
          "Send SPS and PPS Insertion Interval in seconds (sprop parameter sets "
          "will be multiplexed in the data stream when detected.) (0 = disabled)",
          0, 3600, DEFAULT_CONFIG_INTERVAL,
          G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));

  /* Override BaseParse vfuncs */
  parse_class->start = GST_DEBUG_FUNCPTR (gst_h264_parse_start);
  parse_class->stop = GST_DEBUG_FUNCPTR (gst_h264_parse_stop);
142
  parse_class->handle_frame = GST_DEBUG_FUNCPTR (gst_h264_parse_handle_frame);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
143
144
145
  parse_class->pre_push_frame =
      GST_DEBUG_FUNCPTR (gst_h264_parse_pre_push_frame);
  parse_class->set_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_set_caps);
146
  parse_class->get_sink_caps = GST_DEBUG_FUNCPTR (gst_h264_parse_get_caps);
147
  parse_class->sink_event = GST_DEBUG_FUNCPTR (gst_h264_parse_event);
148
  parse_class->src_event = GST_DEBUG_FUNCPTR (gst_h264_parse_src_event);
René Stadler's avatar
René Stadler committed
149
150
151
152
153
154

  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&srctemplate));
  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&sinktemplate));

155
  gst_element_class_set_static_metadata (gstelement_class, "H.264 parser",
René Stadler's avatar
René Stadler committed
156
157
158
      "Codec/Parser/Converter/Video",
      "Parses H.264 streams",
      "Mark Nauwelaerts <mark.nauwelaerts@collabora.co.uk>");
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
159
160
161
}

static void
René Stadler's avatar
René Stadler committed
162
gst_h264_parse_init (GstH264Parse * h264parse)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
163
164
{
  h264parse->frame_out = gst_adapter_new ();
165
  gst_base_parse_set_pts_interpolation (GST_BASE_PARSE (h264parse), FALSE);
166
  GST_PAD_SET_ACCEPT_INTERSECT (GST_BASE_PARSE_SINK_PAD (h264parse));
167
  GST_PAD_SET_ACCEPT_TEMPLATE (GST_BASE_PARSE_SINK_PAD (h264parse));
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
168
169
170
171
172
173
174
175
176
}


static void
gst_h264_parse_finalize (GObject * object)
{
  GstH264Parse *h264parse = GST_H264_PARSE (object);

  g_object_unref (h264parse->frame_out);
177
178

  G_OBJECT_CLASS (parent_class)->finalize (object);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
179
180
181
182
183
}

static void
gst_h264_parse_reset_frame (GstH264Parse * h264parse)
{
184
185
  GST_DEBUG_OBJECT (h264parse, "reset frame");

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
186
  /* done parsing; reset state */
187
  h264parse->current_off = -1;
188

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
189
190
191
  h264parse->picture_start = FALSE;
  h264parse->update_caps = FALSE;
  h264parse->idr_pos = -1;
192
  h264parse->sei_pos = -1;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
193
  h264parse->keyframe = FALSE;
194
  h264parse->header = FALSE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
195
  h264parse->frame_start = FALSE;
196
  gst_adapter_clear (h264parse->frame_out);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
197
198
199
}

static void
200
gst_h264_parse_reset_stream_info (GstH264Parse * h264parse)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
201
{
202
203
  gint i;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
204
205
206
207
  h264parse->width = 0;
  h264parse->height = 0;
  h264parse->fps_num = 0;
  h264parse->fps_den = 0;
208
209
  h264parse->upstream_par_n = -1;
  h264parse->upstream_par_d = -1;
210
211
  h264parse->parsed_par_n = 0;
  h264parse->parsed_par_d = 0;
212
213
  h264parse->have_pps = FALSE;
  h264parse->have_sps = FALSE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
214

215
216
217
218
  h264parse->multiview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
  h264parse->multiview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
  h264parse->first_in_bundle = TRUE;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
219
220
221
  h264parse->align = GST_H264_PARSE_ALIGN_NONE;
  h264parse->format = GST_H264_PARSE_FORMAT_NONE;

222
223
224
  h264parse->transform = FALSE;
  h264parse->nal_length_size = 4;
  h264parse->packetized = FALSE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
225
  h264parse->push_codec = FALSE;
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241

  gst_buffer_replace (&h264parse->codec_data, NULL);
  gst_buffer_replace (&h264parse->codec_data_in, NULL);

  gst_h264_parse_reset_frame (h264parse);

  for (i = 0; i < GST_H264_MAX_SPS_COUNT; i++)
    gst_buffer_replace (&h264parse->sps_nals[i], NULL);
  for (i = 0; i < GST_H264_MAX_PPS_COUNT; i++)
    gst_buffer_replace (&h264parse->pps_nals[i], NULL);
}

static void
gst_h264_parse_reset (GstH264Parse * h264parse)
{
  h264parse->last_report = GST_CLOCK_TIME_NONE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
242

243
244
245
246
  h264parse->dts = GST_CLOCK_TIME_NONE;
  h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
  h264parse->do_ts = TRUE;

247
248
  h264parse->sent_codec_tag = FALSE;

249
  h264parse->pending_key_unit_ts = GST_CLOCK_TIME_NONE;
250
  gst_event_replace (&h264parse->force_key_unit_event, NULL);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
251

252
253
  h264parse->discont = FALSE;

254
  gst_h264_parse_reset_stream_info (h264parse);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
255
256
257
258
259
260
261
}

static gboolean
gst_h264_parse_start (GstBaseParse * parse)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);

262
  GST_DEBUG_OBJECT (parse, "start");
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
263
264
  gst_h264_parse_reset (h264parse);

265
266
267
268
269
270
271
  h264parse->nalparser = gst_h264_nal_parser_new ();

  h264parse->dts = GST_CLOCK_TIME_NONE;
  h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE;
  h264parse->sei_pic_struct_pres_flag = FALSE;
  h264parse->sei_pic_struct = 0;
  h264parse->field_pic_flag = 0;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
272

273
  gst_base_parse_set_min_frame_size (parse, 6);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
274
275
276
277
278
279
280
281
282

  return TRUE;
}

static gboolean
gst_h264_parse_stop (GstBaseParse * parse)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);

283
  GST_DEBUG_OBJECT (parse, "stop");
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
284
285
  gst_h264_parse_reset (h264parse);

286
  gst_h264_nal_parser_free (h264parse->nalparser);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
287
288
289
290
291
292
293
294
295
296
297
298
299

  return TRUE;
}

static const gchar *
gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code)
{
  if (format) {
    switch (code) {
      case GST_H264_PARSE_FORMAT_AVC:
        return "avc";
      case GST_H264_PARSE_FORMAT_BYTE:
        return "byte-stream";
300
301
      case GST_H264_PARSE_FORMAT_AVC3:
        return "avc3";
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
      default:
        return "none";
    }
  } else {
    switch (code) {
      case GST_H264_PARSE_ALIGN_NAL:
        return "nal";
      case GST_H264_PARSE_ALIGN_AU:
        return "au";
      default:
        return "none";
    }
  }
}

static void
318
gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
319
{
320
321
322
323
324
  if (format)
    *format = GST_H264_PARSE_FORMAT_NONE;

  if (align)
    *align = GST_H264_PARSE_ALIGN_NONE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
325

326
327
328
329
  g_return_if_fail (gst_caps_is_fixed (caps));

  GST_DEBUG ("parsing caps: %" GST_PTR_FORMAT, caps);

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
330
331
332
333
  if (caps && gst_caps_get_size (caps) > 0) {
    GstStructure *s = gst_caps_get_structure (caps, 0);
    const gchar *str = NULL;

334
335
336
337
338
339
    if (format) {
      if ((str = gst_structure_get_string (s, "stream-format"))) {
        if (strcmp (str, "avc") == 0)
          *format = GST_H264_PARSE_FORMAT_AVC;
        else if (strcmp (str, "byte-stream") == 0)
          *format = GST_H264_PARSE_FORMAT_BYTE;
340
341
        else if (strcmp (str, "avc3") == 0)
          *format = GST_H264_PARSE_FORMAT_AVC3;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
342
343
344
      }
    }

345
346
347
348
349
350
    if (align) {
      if ((str = gst_structure_get_string (s, "alignment"))) {
        if (strcmp (str, "au") == 0)
          *align = GST_H264_PARSE_ALIGN_AU;
        else if (strcmp (str, "nal") == 0)
          *align = GST_H264_PARSE_ALIGN_NAL;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
351
352
353
      }
    }
  }
354
355
356
357
}

/* check downstream caps to configure format and alignment */
static void
358
359
gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format,
    GstCaps * in_caps)
360
361
{
  GstCaps *caps;
362
363
  guint format = h264parse->format;
  guint align = h264parse->align;
364

365
366
  g_return_if_fail ((in_caps == NULL) || gst_caps_is_fixed (in_caps));

367
368
369
  caps = gst_pad_get_allowed_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
  GST_DEBUG_OBJECT (h264parse, "allowed caps: %" GST_PTR_FORMAT, caps);

370
  /* concentrate on leading structure, since decodebin parser
371
372
   * capsfilter always includes parser template caps */
  if (caps) {
Wim Taymans's avatar
Wim Taymans committed
373
    caps = gst_caps_truncate (caps);
374
375
376
377
    GST_DEBUG_OBJECT (h264parse, "negotiating with caps: %" GST_PTR_FORMAT,
        caps);
  }

378
379
  h264parse->can_passthrough = FALSE;

380
381
382
383
384
385
  if (in_caps && caps) {
    if (gst_caps_can_intersect (in_caps, caps)) {
      GST_DEBUG_OBJECT (h264parse, "downstream accepts upstream caps");
      gst_h264_parse_format_from_caps (in_caps, &format, &align);
      gst_caps_unref (caps);
      caps = NULL;
386
      h264parse->can_passthrough = TRUE;
387
388
    }
  }
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
389

390
391
  /* FIXME We could fail the negotiation immediatly if caps are empty */
  if (caps && !gst_caps_is_empty (caps)) {
392
    /* fixate to avoid ambiguity with lists when parsing */
Wim Taymans's avatar
Wim Taymans committed
393
    caps = gst_caps_fixate (caps);
394
    gst_h264_parse_format_from_caps (caps, &format, &align);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
395
    gst_caps_unref (caps);
396
  }
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
397
398
399
400
401
402
403
404
405
406
407
408
409

  /* default */
  if (!format)
    format = GST_H264_PARSE_FORMAT_BYTE;
  if (!align)
    align = GST_H264_PARSE_ALIGN_AU;

  GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s",
      gst_h264_parse_get_string (h264parse, TRUE, format),
      gst_h264_parse_get_string (h264parse, FALSE, align));

  h264parse->format = format;
  h264parse->align = align;
410

411
412
  h264parse->transform = in_format != h264parse->format ||
      align == GST_H264_PARSE_ALIGN_AU;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
413
414
415
416
417
418
419
}

static GstBuffer *
gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data,
    guint size)
{
  GstBuffer *buf;
420
  guint nl = h264parse->nal_length_size;
René Stadler's avatar
René Stadler committed
421
  guint32 tmp;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
422

423
  GST_DEBUG_OBJECT (h264parse, "nal length %d", size);
424

425
  buf = gst_buffer_new_allocate (NULL, 4 + size, NULL);
426
427
  if (format == GST_H264_PARSE_FORMAT_AVC
      || format == GST_H264_PARSE_FORMAT_AVC3) {
René Stadler's avatar
René Stadler committed
428
    tmp = GUINT32_TO_BE (size << (32 - 8 * nl));
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
429
  } else {
430
431
432
433
    /* HACK: nl should always be 4 here, otherwise this won't work. 
     * There are legit cases where nl in avc stream is 2, but byte-stream
     * SC is still always 4 bytes. */
    nl = 4;
René Stadler's avatar
René Stadler committed
434
    tmp = GUINT32_TO_BE (1);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
435
436
  }

René Stadler's avatar
René Stadler committed
437
438
  gst_buffer_fill (buf, 0, &tmp, sizeof (guint32));
  gst_buffer_fill (buf, nl, data, size);
439
  gst_buffer_set_size (buf, size + nl);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
440
441
442
443

  return buf;
}

444
445
446
447
448
449
450
static void
gst_h264_parser_store_nal (GstH264Parse * h264parse, guint id,
    GstH264NalUnitType naltype, GstH264NalUnit * nalu)
{
  GstBuffer *buf, **store;
  guint size = nalu->size, store_size;

451
  if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_SUBSET_SPS) {
452
453
    store_size = GST_H264_MAX_SPS_COUNT;
    store = h264parse->sps_nals;
454
    GST_DEBUG_OBJECT (h264parse, "storing sps %u", id);
455
456
457
  } else if (naltype == GST_H264_NAL_PPS) {
    store_size = GST_H264_MAX_PPS_COUNT;
    store = h264parse->pps_nals;
458
    GST_DEBUG_OBJECT (h264parse, "storing pps %u", id);
459
460
461
462
463
464
465
466
  } else
    return;

  if (id >= store_size) {
    GST_DEBUG_OBJECT (h264parse, "unable to store nal, id out-of-range %d", id);
    return;
  }

Wim Taymans's avatar
Wim Taymans committed
467
  buf = gst_buffer_new_allocate (NULL, size, NULL);
René Stadler's avatar
René Stadler committed
468
  gst_buffer_fill (buf, 0, nalu->data + nalu->offset, size);
469

470
471
472
473
  /* Indicate that buffer contain a header needed for decoding */
  if (naltype == GST_H264_NAL_SPS || naltype == GST_H264_NAL_PPS)
    GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_HEADER);

474
475
476
477
478
479
  if (store[id])
    gst_buffer_unref (store[id]);

  store[id] = buf;
}

480
#ifndef GST_DISABLE_GST_DEBUG
481
482
483
484
485
486
487
488
489
490
491
492
493
static const gchar *nal_names[] = {
  "Unknown",
  "Slice",
  "Slice DPA",
  "Slice DPB",
  "Slice DPC",
  "Slice IDR",
  "SEI",
  "SPS",
  "PPS",
  "AU delimiter",
  "Sequence End",
  "Stream End",
494
495
496
  "Filler Data",
  "SPS extension",
  "Prefix",
497
498
499
500
501
502
  "SPS Subset",
  "Depth Parameter Set",
  "Reserved", "Reserved",
  "Slice Aux Unpartitioned",
  "Slice Extension",
  "Slice Depth/3D-AVC Extension"
503
504
505
506
507
};

static const gchar *
_nal_name (GstH264NalUnitType nal_type)
{
508
  if (nal_type <= GST_H264_NAL_SLICE_DEPTH)
509
510
511
    return nal_names[nal_type];
  return "Invalid";
}
512
#endif
513

514
515
516
517
518
519
520
521
522
523
524
static void
gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
{
  GstH264SEIMessage sei;
  GstH264NalParser *nalparser = h264parse->nalparser;
  GstH264ParserResult pres;
  GArray *messages;
  guint i;

  pres = gst_h264_parser_parse_sei (nalparser, nalu, &messages);
  if (pres != GST_H264_PARSER_OK)
525
    GST_WARNING_OBJECT (h264parse, "failed to parse one or more SEI message");
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552

  /* Even if pres != GST_H264_PARSER_OK, some message could have been parsed and
   * stored in messages.
   */
  for (i = 0; i < messages->len; i++) {
    sei = g_array_index (messages, GstH264SEIMessage, i);
    switch (sei.payloadType) {
      case GST_H264_SEI_PIC_TIMING:
        h264parse->sei_pic_struct_pres_flag =
            sei.payload.pic_timing.pic_struct_present_flag;
        h264parse->sei_cpb_removal_delay =
            sei.payload.pic_timing.cpb_removal_delay;
        if (h264parse->sei_pic_struct_pres_flag)
          h264parse->sei_pic_struct = sei.payload.pic_timing.pic_struct;
        GST_LOG_OBJECT (h264parse, "pic timing updated");
        break;
      case GST_H264_SEI_BUF_PERIOD:
        if (h264parse->ts_trn_nb == GST_CLOCK_TIME_NONE ||
            h264parse->dts == GST_CLOCK_TIME_NONE)
          h264parse->ts_trn_nb = 0;
        else
          h264parse->ts_trn_nb = h264parse->dts;

        GST_LOG_OBJECT (h264parse,
            "new buffering period; ts_trn_nb updated: %" GST_TIME_FORMAT,
            GST_TIME_ARGS (h264parse->ts_trn_nb));
        break;
553
554
555
556
557
558
559
560
561
562

        /* Additional messages that are not innerly useful to the
         * element but for debugging purposes */
      case GST_H264_SEI_RECOVERY_POINT:
        GST_LOG_OBJECT (h264parse, "recovery point found: %u %u %u %u",
            sei.payload.recovery_point.recovery_frame_cnt,
            sei.payload.recovery_point.exact_match_flag,
            sei.payload.recovery_point.broken_link_flag,
            sei.payload.recovery_point.changing_slice_group_idc);
        break;
563
564
565

        /* Additional messages that are not innerly useful to the
         * element but for debugging purposes */
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
      case GST_H264_SEI_STEREO_VIDEO_INFO:{
        GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
        GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;

        GST_LOG_OBJECT (h264parse, "Stereo video information %u %u %u %u %u %u",
            sei.payload.stereo_video_info.field_views_flag,
            sei.payload.stereo_video_info.top_field_is_left_view_flag,
            sei.payload.stereo_video_info.current_frame_is_left_view_flag,
            sei.payload.stereo_video_info.next_frame_is_second_view_flag,
            sei.payload.stereo_video_info.left_view_self_contained_flag,
            sei.payload.stereo_video_info.right_view_self_contained_flag);

        if (sei.payload.stereo_video_info.field_views_flag) {
          mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
          if (!sei.payload.stereo_video_info.top_field_is_left_view_flag)
            mview_mode |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
        } else {
          mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
          if (sei.payload.stereo_video_info.next_frame_is_second_view_flag) {
            /* Mark current frame as first in bundle */
            h264parse->first_in_bundle = TRUE;
            if (!sei.payload.stereo_video_info.current_frame_is_left_view_flag)
              mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;
          }
        }
        if (mview_mode != h264parse->multiview_mode ||
            mview_flags != h264parse->multiview_flags) {
          h264parse->multiview_mode = mview_mode;
          h264parse->multiview_flags = mview_flags;
          /* output caps need to be changed */
          gst_h264_parse_update_src_caps (h264parse, NULL);
        }
598
        break;
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
      }
      case GST_H264_SEI_FRAME_PACKING:{
        GstVideoMultiviewMode mview_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
        GstVideoMultiviewFlags mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;

        GST_LOG_OBJECT (h264parse,
            "frame packing arrangement message: id %u cancelled %u "
            "type %u quincunx %u content_interpretation %d flip %u "
            "right_first %u field_views %u is_frame0 %u",
            sei.payload.frame_packing.frame_packing_id,
            sei.payload.frame_packing.frame_packing_cancel_flag,
            sei.payload.frame_packing.frame_packing_type,
            sei.payload.frame_packing.quincunx_sampling_flag,
            sei.payload.frame_packing.content_interpretation_type,
            sei.payload.frame_packing.spatial_flipping_flag,
            sei.payload.frame_packing.frame0_flipped_flag,
            sei.payload.frame_packing.field_views_flag,
            sei.payload.frame_packing.current_frame_is_frame0_flag);

        /* Only IDs from 0->255 and 512->2^31-1 are valid. Ignore others */
        if ((sei.payload.frame_packing.frame_packing_id >= 256 &&
                sei.payload.frame_packing.frame_packing_id < 512) ||
            (sei.payload.frame_packing.frame_packing_id >= (1U << 31)))
          break;                /* ignore */

        if (!sei.payload.frame_packing.frame_packing_cancel_flag) {
          /* Cancel flag sets things back to no-info */

          if (sei.payload.frame_packing.content_interpretation_type == 2)
            mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST;

          switch (sei.payload.frame_packing.frame_packing_type) {
            case 0:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD;
              break;
            case 1:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED;
              break;
            case 2:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED;
              break;
            case 3:
              if (sei.payload.frame_packing.quincunx_sampling_flag)
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX;
              else
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
              if (sei.payload.frame_packing.spatial_flipping_flag) {
                /* One of the views is flopped. */
                if (sei.payload.frame_packing.frame0_flipped_flag !=
                    ! !(mview_flags &
                        GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
                  /* the left view is flopped */
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED;
                else
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED;
              }
              break;
            case 4:
              mview_mode = GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM;
              if (sei.payload.frame_packing.spatial_flipping_flag) {
                /* One of the views is flipped, */
                if (sei.payload.frame_packing.frame0_flipped_flag !=
                    ! !(mview_flags &
                        GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST))
                  /* the left view is flipped */
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED;
                else
                  mview_flags |= GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED;
              }
              break;
            case 5:
              if (sei.payload.frame_packing.content_interpretation_type == 0)
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME;
              else
                mview_mode = GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME;
              break;
            default:
              GST_DEBUG_OBJECT (h264parse, "Invalid frame packing type %u",
                  sei.payload.frame_packing.frame_packing_type);
              break;
          }
        }

        if (mview_mode != h264parse->multiview_mode ||
            mview_flags != h264parse->multiview_flags) {
          h264parse->multiview_mode = mview_mode;
          h264parse->multiview_flags = mview_flags;
          /* output caps need to be changed */
          gst_h264_parse_update_src_caps (h264parse, NULL);
        }
689
        break;
690
      }
691
692
693
694
695
    }
  }
  g_array_free (messages, TRUE);
}

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
696
/* caller guarantees 2 bytes of nal payload */
697
static gboolean
698
gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
699
700
{
  guint nal_type;
701
702
  GstH264PPS pps = { 0, };
  GstH264SPS sps = { 0, };
703
  GstH264NalParser *nalparser = h264parse->nalparser;
704
  GstH264ParserResult pres;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
705

706
  /* nothing to do for broken input */
707
708
  if (G_UNLIKELY (nalu->size < 2)) {
    GST_DEBUG_OBJECT (h264parse, "not processing nal size %u", nalu->size);
709
    return TRUE;
710
711
  }

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
712
  /* we have a peek as well */
713
  nal_type = nalu->type;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
714

715
716
  GST_DEBUG_OBJECT (h264parse, "processing nal of type %u %s, size %u",
      nal_type, _nal_name (nal_type), nalu->size);
717

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
718
  switch (nal_type) {
719
720
721
    case GST_H264_NAL_SUBSET_SPS:
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;
722
      pres = gst_h264_parser_parse_subset_sps (nalparser, nalu, &sps, TRUE);
723
724
      goto process_sps;

725
    case GST_H264_NAL_SPS:
726
727
      /* reset state, everything else is obsolete */
      h264parse->state = 0;
728
      pres = gst_h264_parser_parse_sps (nalparser, nalu, &sps, TRUE);
729

730
    process_sps:
731
      /* arranged for a fallback sps.id, so use that one and only warn */
732
      if (pres != GST_H264_PARSER_OK) {
733
        GST_WARNING_OBJECT (h264parse, "failed to parse SPS:");
734
735
        return FALSE;
      }
736
737
738

      GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
      h264parse->update_caps = TRUE;
739
740
741
742
743
744
745
746
747
      h264parse->have_sps = TRUE;
      if (h264parse->push_codec && h264parse->have_pps) {
        /* SPS and PPS found in stream before the first pre_push_frame, no need
         * to forcibly push at start */
        GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
        h264parse->push_codec = FALSE;
        h264parse->have_sps = FALSE;
        h264parse->have_pps = FALSE;
      }
748
749

      gst_h264_parser_store_nal (h264parse, sps.id, nal_type, nalu);
750
      gst_h264_sps_clear (&sps);
751
      h264parse->state |= GST_H264_PARSE_STATE_GOT_SPS;
752
      h264parse->header |= TRUE;
753
754
      break;
    case GST_H264_NAL_PPS:
755
      /* expected state: got-sps */
756
      h264parse->state &= GST_H264_PARSE_STATE_GOT_SPS;
757
758
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;
759

760
761
      pres = gst_h264_parser_parse_pps (nalparser, nalu, &pps);
      /* arranged for a fallback pps.id, so use that one and only warn */
762
      if (pres != GST_H264_PARSER_OK) {
763
        GST_WARNING_OBJECT (h264parse, "failed to parse PPS:");
764
765
766
        if (pres != GST_H264_PARSER_BROKEN_LINK)
          return FALSE;
      }
767

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
768
      /* parameters might have changed, force caps check */
769
770
771
772
      if (!h264parse->have_pps) {
        GST_DEBUG_OBJECT (h264parse, "triggering src caps check");
        h264parse->update_caps = TRUE;
      }
773
774
775
776
777
778
779
780
781
      h264parse->have_pps = TRUE;
      if (h264parse->push_codec && h264parse->have_sps) {
        /* SPS and PPS found in stream before the first pre_push_frame, no need
         * to forcibly push at start */
        GST_INFO_OBJECT (h264parse, "have SPS/PPS in stream");
        h264parse->push_codec = FALSE;
        h264parse->have_sps = FALSE;
        h264parse->have_pps = FALSE;
      }
782
783

      gst_h264_parser_store_nal (h264parse, pps.id, nal_type, nalu);
784
      gst_h264_pps_clear (&pps);
785
      h264parse->state |= GST_H264_PARSE_STATE_GOT_PPS;
786
      h264parse->header |= TRUE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
787
      break;
788
    case GST_H264_NAL_SEI:
789
790
791
792
      /* expected state: got-sps */
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;

793
      h264parse->header |= TRUE;
794
      gst_h264_parse_process_sei (h264parse, nalu);
795
796
      /* mark SEI pos */
      if (h264parse->sei_pos == -1) {
797
        if (h264parse->transform)
798
799
          h264parse->sei_pos = gst_adapter_available (h264parse->frame_out);
        else
800
          h264parse->sei_pos = nalu->sc_offset;
801
802
803
        GST_DEBUG_OBJECT (h264parse, "marking SEI in frame at offset %d",
            h264parse->sei_pos);
      }
804
805
806
807
808
809
      break;

    case GST_H264_NAL_SLICE:
    case GST_H264_NAL_SLICE_DPA:
    case GST_H264_NAL_SLICE_DPB:
    case GST_H264_NAL_SLICE_DPC:
810
    case GST_H264_NAL_SLICE_IDR:
811
    case GST_H264_NAL_SLICE_EXT:
812
      /* expected state: got-sps|got-pps (valid picture headers) */
813
      h264parse->state &= GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS;
814
815
816
      if (!GST_H264_PARSE_STATE_VALID (h264parse,
              GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
        return FALSE;
817

818
      /* don't need to parse the whole slice (header) here */
819
      if (*(nalu->data + nalu->offset + nalu->header_bytes) & 0x80) {
820
821
822
823
        /* means first_mb_in_slice == 0 */
        /* real frame data */
        GST_DEBUG_OBJECT (h264parse, "first_mb_in_slice = 0");
        h264parse->frame_start = TRUE;
824
      }
825
      GST_DEBUG_OBJECT (h264parse, "frame start: %i", h264parse->frame_start);
826
827
      if (nal_type == GST_H264_NAL_SLICE_EXT && !GST_H264_IS_MVC_NALU (nalu))
        break;
828
829
      {
        GstH264SliceHdr slice;
830

831
832
833
834
835
        pres = gst_h264_parser_parse_slice_hdr (nalparser, nalu, &slice,
            FALSE, FALSE);
        GST_DEBUG_OBJECT (h264parse,
            "parse result %d, first MB: %u, slice type: %u",
            pres, slice.first_mb_in_slice, slice.type);
836
        if (pres == GST_H264_PARSER_OK) {
837
838
          if (GST_H264_IS_I_SLICE (&slice) || GST_H264_IS_SI_SLICE (&slice))
            h264parse->keyframe |= TRUE;
839

840
          h264parse->state |= GST_H264_PARSE_STATE_GOT_SLICE;
841
          h264parse->field_pic_flag = slice.field_pic_flag;
842
        }
843
844
845
846
      }
      if (G_LIKELY (nal_type != GST_H264_NAL_SLICE_IDR &&
              !h264parse->push_codec))
        break;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
847
848
849
850
851
      /* if we need to sneak codec NALs into the stream,
       * this is a good place, so fake it as IDR
       * (which should be at start anyway) */
      /* mark where config needs to go if interval expired */
      /* mind replacement buffer if applicable */
852
      if (h264parse->idr_pos == -1) {
853
        if (h264parse->transform)
854
855
          h264parse->idr_pos = gst_adapter_available (h264parse->frame_out);
        else
856
          h264parse->idr_pos = nalu->sc_offset;
857
858
859
        GST_DEBUG_OBJECT (h264parse, "marking IDR in frame at offset %d",
            h264parse->idr_pos);
      }
860
861
862
863
864
865
      /* if SEI preceeds (faked) IDR, then we have to insert config there */
      if (h264parse->sei_pos >= 0 && h264parse->idr_pos > h264parse->sei_pos) {
        h264parse->idr_pos = h264parse->sei_pos;
        GST_DEBUG_OBJECT (h264parse, "moved IDR mark to SEI position %d",
            h264parse->idr_pos);
      }
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
866
      break;
867
868
869
870
871
872
    case GST_H264_NAL_AU_DELIMITER:
      /* Just accumulate AU Delimiter, whether it's before SPS or not */
      pres = gst_h264_parser_parse_nal (nalparser, nalu);
      if (pres != GST_H264_PARSER_OK)
        return FALSE;
      break;
873
    default:
874
875
876
877
878
879
880
881
      /* drop anything before the initial SPS */
      if (!GST_H264_PARSE_STATE_VALID (h264parse, GST_H264_PARSE_STATE_GOT_SPS))
        return FALSE;

      pres = gst_h264_parser_parse_nal (nalparser, nalu);
      if (pres != GST_H264_PARSER_OK)
        return FALSE;
      break;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
882
883
884
885
  }

  /* if AVC output needed, collect properly prefixed nal in adapter,
   * and use that to replace outgoing buffer data later on */
886
  if (h264parse->transform) {
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
887
888
889
890
    GstBuffer *buf;

    GST_LOG_OBJECT (h264parse, "collecting NAL in AVC frame");
    buf = gst_h264_parse_wrap_nal (h264parse, h264parse->format,
891
        nalu->data + nalu->offset, nalu->size);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
892
893
    gst_adapter_push (h264parse->frame_out, buf);
  }
894
  return TRUE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
895
896
897
898
899
}

/* caller guarantees at least 2 bytes of nal payload for each nal
 * returns TRUE if next_nal indicates that nal terminates an AU */
static inline gboolean
900
901
gst_h264_parse_collect_nal (GstH264Parse * h264parse, const guint8 * data,
    guint size, GstH264NalUnit * nalu)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
902
903
{
  gboolean complete;
904
905
906
  GstH264ParserResult parse_res;
  GstH264NalUnitType nal_type = nalu->type;
  GstH264NalUnit nnalu;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
907

908
  GST_DEBUG_OBJECT (h264parse, "parsing collected nal");
909
910
  parse_res = gst_h264_parser_identify_nalu_unchecked (h264parse->nalparser,
      data, nalu->offset + nalu->size, size, &nnalu);
911

912
  if (parse_res != GST_H264_PARSER_OK)
913
914
    return FALSE;

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
915
  /* determine if AU complete */
916
  GST_LOG_OBJECT (h264parse, "nal type: %d %s", nal_type, _nal_name (nal_type));
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
917
918
  /* coded slice NAL starts a picture,
   * i.e. other types become aggregated in front of it */
919
920
  h264parse->picture_start |= (nal_type == GST_H264_NAL_SLICE ||
      nal_type == GST_H264_NAL_SLICE_DPA || nal_type == GST_H264_NAL_SLICE_IDR);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
921
922
923
924
925
926
927
928

  /* consider a coded slices (IDR or not) to start a picture,
   * (so ending the previous one) if first_mb_in_slice == 0
   * (non-0 is part of previous one) */
  /* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
   * but in practice it works in sane cases, needs not much parsing,
   * and also works with broken frame_num in NAL
   * (where spec-wise would fail) */
929
  nal_type = nnalu.type;
930
931
932
  complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
          nal_type <= GST_H264_NAL_AU_DELIMITER) ||
      (nal_type >= 14 && nal_type <= 18));
933

934
935
936
937
938
  GST_LOG_OBJECT (h264parse, "next nal type: %d %s", nal_type,
      _nal_name (nal_type));
  complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
      || nal_type == GST_H264_NAL_SLICE_DPA
      || nal_type == GST_H264_NAL_SLICE_IDR) &&
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
939
      /* first_mb_in_slice == 0 considered start of frame */
940
      (nnalu.data[nnalu.offset + nnalu.header_bytes] & 0x80);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
941
942
943
944
945
946

  GST_LOG_OBJECT (h264parse, "au complete: %d", complete);

  return complete;
}

947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987

static GstFlowReturn
gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
    GstBaseParseFrame * frame)
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);
  GstBuffer *buffer = frame->buffer;
  GstFlowReturn ret = GST_FLOW_OK;
  GstH264ParserResult parse_res;
  GstH264NalUnit nalu;
  const guint nl = h264parse->nal_length_size;
  GstMapInfo map;
  gint left;

  if (nl < 1 || nl > 4) {
    GST_DEBUG_OBJECT (h264parse, "insufficient data to split input");
    return GST_FLOW_NOT_NEGOTIATED;
  }

  /* need to save buffer from invalidation upon _finish_frame */
  if (h264parse->split_packetized)
    buffer = gst_buffer_copy (frame->buffer);

  gst_buffer_map (buffer, &map, GST_MAP_READ);

  left = map.size;

  GST_LOG_OBJECT (h264parse,
      "processing packet buffer of size %" G_GSIZE_FORMAT, map.size);

  parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
      map.data, 0, map.size, nl, &nalu);

  while (parse_res == GST_H264_PARSER_OK) {
    GST_DEBUG_OBJECT (h264parse, "AVC nal offset %d", nalu.offset + nalu.size);

    /* either way, have a look at it */
    gst_h264_parse_process_nal (h264parse, &nalu);

    /* dispatch per NALU if needed */
    if (h264parse->split_packetized) {
988
989
990
991
992
993
994
995
996
      GstBaseParseFrame tmp_frame;

      gst_base_parse_frame_init (&tmp_frame);
      tmp_frame.flags |= frame->flags;
      tmp_frame.offset = frame->offset;
      tmp_frame.overhead = frame->overhead;
      tmp_frame.buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL,
          nalu.offset, nalu.size);

997
998
999
1000
      /* note we don't need to come up with a sub-buffer, since
       * subsequent code only considers input buffer's metadata.
       * Real data is either taken from input by baseclass or
       * a replacement output buffer is provided anyway. */
1001
1002
      gst_h264_parse_parse_frame (parse, &tmp_frame);
      ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size);
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
      left -= nl + nalu.size;
    }

    parse_res = gst_h264_parser_identify_nalu_avc (h264parse->nalparser,
        map.data, nalu.offset + nalu.size, map.size, nl, &nalu);
  }

  gst_buffer_unmap (buffer, &map);

  if (!h264parse->split_packetized) {
    gst_h264_parse_parse_frame (parse, frame);
    ret = gst_base_parse_finish_frame (parse, frame, map.size);
  } else {
    gst_buffer_unref (buffer);
    if (G_UNLIKELY (left)) {
      /* should not be happening for nice AVC */
      GST_WARNING_OBJECT (parse, "skipping leftover AVC data %d", left);
      frame->flags |= GST_BASE_PARSE_FRAME_FLAG_DROP;
      ret = gst_base_parse_finish_frame (parse, frame, map.size);
    }
  }

  if (parse_res == GST_H264_PARSER_NO_NAL_END ||
      parse_res == GST_H264_PARSER_BROKEN_DATA) {

    if (h264parse->split_packetized) {
      GST_ELEMENT_ERROR (h264parse, STREAM, FAILED, (NULL),
          ("invalid AVC input data"));

      return GST_FLOW_ERROR;
    } else {
      /* do not meddle to much in this case */
      GST_DEBUG_OBJECT (h264parse, "parsing packet failed");
    }
  }

  return ret;
}

1042
1043
1044
static GstFlowReturn
gst_h264_parse_handle_frame (GstBaseParse * parse,
    GstBaseParseFrame * frame, gint * skipsize)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1045
1046
1047
{
  GstH264Parse *h264parse = GST_H264_PARSE (parse);
  GstBuffer *buffer = frame->buffer;
1048
  GstMapInfo map;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1049
  guint8 *data;
René Stadler's avatar
René Stadler committed
1050
  gsize size;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1051
  gint current_off = 0;
1052
  gboolean drain, nonext;
1053
  GstH264NalParser *nalparser = h264parse->nalparser;
1054
  GstH264NalUnit nalu;
1055
  GstH264ParserResult pres;
1056
  gint framesize;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1057

1058
1059
1060
1061
1062
  if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
              GST_BUFFER_FLAG_DISCONT))) {
    h264parse->discont = TRUE;
  }

1063
1064
1065
1066
  /* delegate in packetized case, no skipping should be needed */
  if (h264parse->packetized)
    return gst_h264_parse_handle_frame_packetized (parse, frame);

1067
1068
1069
  gst_buffer_map (buffer, &map, GST_MAP_READ);
  data = map.data;
  size = map.size;
René Stadler's avatar
René Stadler committed
1070

Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1071
  /* expect at least 3 bytes startcode == sc, and 2 bytes NALU payload */
René Stadler's avatar
René Stadler committed
1072
  if (G_UNLIKELY (size < 5)) {
1073
    gst_buffer_unmap (buffer, &map);
1074
1075
    *skipsize = 1;
    return GST_FLOW_OK;
René Stadler's avatar
René Stadler committed
1076
  }
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1077
1078
1079

  /* need to configure aggregation */
  if (G_UNLIKELY (h264parse->format == GST_H264_PARSE_FORMAT_NONE))
1080
    gst_h264_parse_negotiate (h264parse, GST_H264_PARSE_FORMAT_BYTE, NULL);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1081

1082
  /* avoid stale cached parsing state */
1083
  if (frame->flags & GST_BASE_PARSE_FRAME_FLAG_NEW_FRAME) {
1084
1085
1086
1087
1088
1089
    GST_LOG_OBJECT (h264parse, "parsing new frame");
    gst_h264_parse_reset_frame (h264parse);
  } else {
    GST_LOG_OBJECT (h264parse, "resuming frame parsing");
  }

1090
1091
  drain = GST_BASE_PARSE_DRAINING (parse);
  nonext = FALSE;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1092

1093
  current_off = h264parse->current_off;
1094
1095
  if (current_off < 0)
    current_off = 0;
1096
  g_assert (current_off < size);
1097
  GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
1098

1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
  /* check for initial skip */
  if (h264parse->current_off == -1) {
    pres =
        gst_h264_parser_identify_nalu_unchecked (nalparser, data, current_off,
        size, &nalu);
    switch (pres) {
      case GST_H264_PARSER_OK:
        if (nalu.sc_offset > 0) {
          *skipsize = nalu.sc_offset;
          goto skip;
        }
        break;
      case GST_H264_PARSER_NO_NAL:
        *skipsize = size - 3;
        goto skip;
        break;
      default:
        g_assert_not_reached ();
        break;
    }
  }
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1120

1121
  while (TRUE) {
1122
1123
1124
    pres =
        gst_h264_parser_identify_nalu (nalparser, data, current_off, size,
        &nalu);
1125
1126

    switch (pres) {
1127
      case GST_H264_PARSER_OK:
1128
1129
        GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
            nalu.offset, nalu.size);
1130
        break;
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
      case GST_H264_PARSER_NO_NAL_END:
        GST_DEBUG_OBJECT (h264parse, "not a complete nal found at offset %u",
            nalu.offset);
        /* if draining, accept it as complete nal */
        if (drain) {
          nonext = TRUE;
          nalu.size = size - nalu.offset;
          GST_DEBUG_OBJECT (h264parse, "draining, accepting with size %u",
              nalu.size);
          /* if it's not too short at least */
          if (nalu.size < 2)
            goto broken;
          break;
        }
        /* otherwise need more */
        goto more;
1147
      case GST_H264_PARSER_BROKEN_LINK:
1148
1149
1150
1151
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"),
            ("The link to structure needed for the parsing couldn't be found"));
        goto invalid_stream;
1152
      case GST_H264_PARSER_ERROR:
1153
        /* should not really occur either */
1154
1155
1156
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"), ("Invalid H.264 stream"));
        goto invalid_stream;
1157
      case GST_H264_PARSER_NO_NAL:
1158
1159
1160
        GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
            ("Error parsing H.264 stream"), ("No H.264 NAL unit found"));
        goto invalid_stream;
1161
1162
      case GST_H264_PARSER_BROKEN_DATA:
        GST_WARNING_OBJECT (h264parse, "input stream is corrupt; "
1163
1164
            "it contains a NAL unit of length %u", nalu.size);
      broken:
1165
1166
1167
        /* broken nal at start -> arrange to skip it,
         * otherwise have it terminate current au
         * (and so it will be skipped on next frame round) */
1168
        if (current_off == 0) {
1169
          GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
1170
1171
          *skipsize = nalu.offset;
          goto skip;
1172
        } else {
1173
          GST_DEBUG_OBJECT (h264parse, "terminating au");
1174
          nalu.size = 0;
1175
          nalu.offset = nalu.sc_offset;
1176
1177
          goto end;
        }
1178
1179
1180
1181
        break;
      default:
        g_assert_not_reached ();
        break;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1182
1183
    }

1184
1185
    GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
        data, nalu.offset, nalu.size);
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1186

1187
1188
1189
    /* simulate no next nal if none needed */
    nonext = nonext || (h264parse->align == GST_H264_PARSE_ALIGN_NAL);

1190
    if (!nonext) {
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
      if (nalu.offset + nalu.size + 4 + 2 > size) {
        GST_DEBUG_OBJECT (h264parse, "not enough data for next NALU");
        if (drain) {
          GST_DEBUG_OBJECT (h264parse, "but draining anyway");
          nonext = TRUE;
        } else {
          goto more;
        }
      }
    }

1202
    if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
1203
      GST_WARNING_OBJECT (h264parse,
1204
          "broken/invalid nal Type: %d %s, Size: %u will be dropped",
1205
          nalu.type, _nal_name (nalu.type), nalu.size);
1206
1207
1208
      *skipsize = nalu.size;
      goto skip;
    }
1209

1210
    if (nonext)
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1211
      break;
1212

1213
    /* if no next nal, we know it's complete here */
1214
    if (gst_h264_parse_collect_nal (h264parse, data, size, &nalu))
1215
1216
1217
      break;

    GST_DEBUG_OBJECT (h264parse, "Looking for more");
1218
    current_off = nalu.offset + nalu.size;
Mark Nauwelaerts's avatar
Mark Nauwelaerts committed
1219
1220
  }

1221
end: