gstaudioaggregator.c 61.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/* GStreamer
 * Copyright (C) 1999,2000 Erik Walthinsen <omega@cse.ogi.edu>
 *                    2001 Thomas <thomas@apestaart.org>
 *               2005,2006 Wim Taymans <wim@fluendo.com>
 *                    2013 Sebastian Dröge <sebastian@centricular.com>
 *                    2014 Collabora
 *                             Olivier Crete <olivier.crete@collabora.com>
 *
 * gstaudioaggregator.c:
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */
/**
 * SECTION: gstaudioaggregator
 * @short_description: manages a set of pads with the purpose of
 * aggregating their buffers for raw audio
 * @see_also: #GstAggregator
 *
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 * #GstAudioAggregator will perform conversion on the data arriving
 * on its sink pads, based on the format expected downstream.
 *
 * Subclasses can opt out of the conversion behaviour by setting
 * #GstAudioAggregator.convert_buffer() to %NULL.
 *
 * Subclasses that wish to use the default conversion implementation
 * should use a (subclass of) #GstAudioAggregatorConvertPad as their
 * #GstAggregatorClass.sinkpads_type, as it will cache the created
 * #GstAudioConverter and install a property allowing to configure it,
 * #GstAudioAggregatorPadClass:converter-config.
 *
 * Subclasses that wish to perform custom conversion should override
 * #GstAudioAggregator.convert_buffer().
 *
 * When conversion is enabled, #GstAudioAggregator will accept
 * any type of raw audio caps and perform conversion
 * on the data arriving on its sink pads, with whatever downstream
 * expects as the target format.
 *
 * In case downstream caps are not fully fixated, it will use
 * the first configured sink pad to finish fixating its source pad
 * caps.
 *
 * Additionally, handling audio conversion directly in the element
 * means that this base class supports safely reconfiguring its
 * source pad.
 *
 * A notable exception for now is the sample rate, sink pads must
 * have the same sample rate as either the downstream requirement,
 * or the first configured pad, or a combination of both (when
 * downstream specifies a range or a set of acceptable rates).
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 */


#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include "gstaudioaggregator.h"

#include <string.h>

GST_DEBUG_CATEGORY_STATIC (audio_aggregator_debug);
#define GST_CAT_DEFAULT audio_aggregator_debug

struct _GstAudioAggregatorPadPrivate
{
  /* All members are protected by the pad object lock */

82
  GstBuffer *buffer;            /* current buffer we're mixing, for
Stefan Sauer's avatar
Stefan Sauer committed
83
84
                                   comparison with a new input buffer from
                                   aggregator to see if we need to update our
85
                                   cached values. */
Stefan Sauer's avatar
Stefan Sauer committed
86
87
88

  guint position, size;         /* position in the input buffer and size of the
                                   input buffer in number of samples */
89

90
91
  GstBuffer *input_buffer;

92
  guint64 output_offset;        /* Sample offset in output segment relative to
Stefan Sauer's avatar
Stefan Sauer committed
93
94
                                   pad.segment.start that position refers to
                                   in the current buffer. */
95

Stefan Sauer's avatar
Stefan Sauer committed
96
97
  guint64 next_offset;          /* Next expected sample offset relative to
                                   pad.segment.start */
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

  /* Last time we noticed a discont */
  GstClockTime discont_time;

  /* A new unhandled segment event has been received */
  gboolean new_segment;
};


/*****************************************
 * GstAudioAggregatorPad implementation  *
 *****************************************/
G_DEFINE_TYPE (GstAudioAggregatorPad, gst_audio_aggregator_pad,
    GST_TYPE_AGGREGATOR_PAD);

113
114
115
116
117
118
enum
{
  PROP_PAD_0,
  PROP_PAD_CONVERTER_CONFIG,
};

119
static GstFlowReturn
120
121
122
gst_audio_aggregator_pad_flush_pad (GstAggregatorPad * aggpad,
    GstAggregator * aggregator);

123
124
125
126
127
128
static void
gst_audio_aggregator_pad_finalize (GObject * object)
{
  GstAudioAggregatorPad *pad = (GstAudioAggregatorPad *) object;

  gst_buffer_replace (&pad->priv->buffer, NULL);
129
  gst_buffer_replace (&pad->priv->input_buffer, NULL);
130
131
132
133

  G_OBJECT_CLASS (gst_audio_aggregator_pad_parent_class)->finalize (object);
}

134
135
136
static void
gst_audio_aggregator_pad_class_init (GstAudioAggregatorPadClass * klass)
{
137
  GObjectClass *gobject_class = (GObjectClass *) klass;
138
139
140
141
  GstAggregatorPadClass *aggpadclass = (GstAggregatorPadClass *) klass;

  g_type_class_add_private (klass, sizeof (GstAudioAggregatorPadPrivate));

142
  gobject_class->finalize = gst_audio_aggregator_pad_finalize;
143
144
145
146
147
148
149
150
151
152
153
154
155
  aggpadclass->flush = GST_DEBUG_FUNCPTR (gst_audio_aggregator_pad_flush_pad);
}

static void
gst_audio_aggregator_pad_init (GstAudioAggregatorPad * pad)
{
  pad->priv =
      G_TYPE_INSTANCE_GET_PRIVATE (pad, GST_TYPE_AUDIO_AGGREGATOR_PAD,
      GstAudioAggregatorPadPrivate);

  gst_audio_info_init (&pad->info);

  pad->priv->buffer = NULL;
156
  pad->priv->input_buffer = NULL;
157
158
159
160
161
162
163
164
  pad->priv->position = 0;
  pad->priv->size = 0;
  pad->priv->output_offset = -1;
  pad->priv->next_offset = -1;
  pad->priv->discont_time = GST_CLOCK_TIME_NONE;
}


165
static GstFlowReturn
166
167
168
169
170
171
172
173
174
175
gst_audio_aggregator_pad_flush_pad (GstAggregatorPad * aggpad,
    GstAggregator * aggregator)
{
  GstAudioAggregatorPad *pad = GST_AUDIO_AGGREGATOR_PAD (aggpad);

  GST_OBJECT_LOCK (aggpad);
  pad->priv->position = pad->priv->size = 0;
  pad->priv->output_offset = pad->priv->next_offset = -1;
  pad->priv->discont_time = GST_CLOCK_TIME_NONE;
  gst_buffer_replace (&pad->priv->buffer, NULL);
176
  gst_buffer_replace (&pad->priv->input_buffer, NULL);
177
178
  GST_OBJECT_UNLOCK (aggpad);

179
  return GST_FLOW_OK;
180
181
}

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
struct _GstAudioAggregatorConvertPadPrivate
{
  /* All members are protected by the pad object lock */
  GstAudioConverter *converter;
  GstStructure *converter_config;
  gboolean converter_config_changed;
};


G_DEFINE_TYPE (GstAudioAggregatorConvertPad, gst_audio_aggregator_convert_pad,
    GST_TYPE_AUDIO_AGGREGATOR_PAD);

static void
gst_audio_aggregator_convert_pad_update_converter (GstAudioAggregatorConvertPad
    * aaggcpad, GstAudioInfo * in_info, GstAudioInfo * out_info)
{
  if (!aaggcpad->priv->converter_config_changed)
    return;

  if (aaggcpad->priv->converter) {
    gst_audio_converter_free (aaggcpad->priv->converter);
    aaggcpad->priv->converter = NULL;
  }

  if (gst_audio_info_is_equal (in_info, out_info) ||
      in_info->finfo->format == GST_AUDIO_FORMAT_UNKNOWN) {
    if (aaggcpad->priv->converter) {
      gst_audio_converter_free (aaggcpad->priv->converter);
      aaggcpad->priv->converter = NULL;
    }
  } else {
    /* If we haven't received caps yet, this pad should not have
     * a buffer to convert anyway */
    aaggcpad->priv->converter =
        gst_audio_converter_new (GST_AUDIO_CONVERTER_FLAG_NONE,
        in_info, out_info,
        aaggcpad->priv->converter_config ? gst_structure_copy (aaggcpad->
            priv->converter_config) : NULL);
  }

  aaggcpad->priv->converter_config_changed = FALSE;
}

static GstBuffer *
gst_audio_aggregator_convert_pad_convert_buffer (GstAudioAggregatorConvertPad *
    aaggcpad, GstAudioInfo * in_info, GstAudioInfo * out_info,
    GstBuffer * input_buffer)
{
  GstBuffer *res;

  gst_audio_aggregator_convert_pad_update_converter (aaggcpad, in_info,
      out_info);

  if (aaggcpad->priv->converter) {
    gint insize = gst_buffer_get_size (input_buffer);
    gsize insamples = insize / in_info->bpf;
    gsize outsamples =
        gst_audio_converter_get_out_frames (aaggcpad->priv->converter,
        insamples);
    gint outsize = outsamples * out_info->bpf;
    GstMapInfo inmap, outmap;

    res = gst_buffer_new_allocate (NULL, outsize, NULL);

    /* We create a perfectly similar buffer, except obviously for
     * its converted contents */
    gst_buffer_copy_into (res, input_buffer,
        GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS |
        GST_BUFFER_COPY_META, 0, -1);

    gst_buffer_map (input_buffer, &inmap, GST_MAP_READ);
    gst_buffer_map (res, &outmap, GST_MAP_WRITE);

    gst_audio_converter_samples (aaggcpad->priv->converter,
        GST_AUDIO_CONVERTER_FLAG_NONE,
        (gpointer *) & inmap.data, insamples,
        (gpointer *) & outmap.data, outsamples);

    gst_buffer_unmap (input_buffer, &inmap);
    gst_buffer_unmap (res, &outmap);
  } else {
    res = gst_buffer_ref (input_buffer);
  }

  return res;
}

static void
gst_audio_aggregator_convert_pad_finalize (GObject * object)
{
  GstAudioAggregatorConvertPad *pad = (GstAudioAggregatorConvertPad *) object;

  if (pad->priv->converter)
    gst_audio_converter_free (pad->priv->converter);

  if (pad->priv->converter_config)
    gst_structure_free (pad->priv->converter_config);

  G_OBJECT_CLASS (gst_audio_aggregator_convert_pad_parent_class)->finalize
      (object);
}

static void
gst_audio_aggregator_convert_pad_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstAudioAggregatorConvertPad *pad = GST_AUDIO_AGGREGATOR_CONVERT_PAD (object);

  switch (prop_id) {
    case PROP_PAD_CONVERTER_CONFIG:
      GST_OBJECT_LOCK (pad);
      if (pad->priv->converter_config)
        g_value_set_boxed (value, pad->priv->converter_config);
      GST_OBJECT_UNLOCK (pad);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_audio_aggregator_convert_pad_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstAudioAggregatorConvertPad *pad = GST_AUDIO_AGGREGATOR_CONVERT_PAD (object);
308

309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  switch (prop_id) {
    case PROP_PAD_CONVERTER_CONFIG:
      GST_OBJECT_LOCK (pad);
      if (pad->priv->converter_config)
        gst_structure_free (pad->priv->converter_config);
      pad->priv->converter_config = g_value_dup_boxed (value);
      pad->priv->converter_config_changed = TRUE;
      GST_OBJECT_UNLOCK (pad);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_audio_aggregator_convert_pad_class_init (GstAudioAggregatorConvertPadClass *
    klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  g_type_class_add_private (klass,
      sizeof (GstAudioAggregatorConvertPadPrivate));

  gobject_class->set_property = gst_audio_aggregator_convert_pad_set_property;
  gobject_class->get_property = gst_audio_aggregator_convert_pad_get_property;

  g_object_class_install_property (gobject_class, PROP_PAD_CONVERTER_CONFIG,
      g_param_spec_boxed ("converter-config", "Converter configuration",
          "A GstStructure describing the configuration that should be used "
          "when converting this pad's audio buffers",
          GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  gobject_class->finalize = gst_audio_aggregator_convert_pad_finalize;
}

static void
gst_audio_aggregator_convert_pad_init (GstAudioAggregatorConvertPad * pad)
{
  pad->priv =
      G_TYPE_INSTANCE_GET_PRIVATE (pad, GST_TYPE_AUDIO_AGGREGATOR_CONVERT_PAD,
      GstAudioAggregatorConvertPadPrivate);
}
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366

/**************************************
 * GstAudioAggregator implementation  *
 **************************************/

struct _GstAudioAggregatorPrivate
{
  GMutex mutex;

  /* All three properties are unprotected, can't be modified while streaming */
  /* Size in frames that is output per buffer */
  GstClockTime output_buffer_duration;
  GstClockTime alignment_threshold;
  GstClockTime discont_wait;

  /* Protected by srcpad stream clock */
Stefan Sauer's avatar
Stefan Sauer committed
367
368
  /* Output buffer starting at offset containing blocksize frames (calculated
   * from output_buffer_duration) */
369
370
371
372
  GstBuffer *current_buffer;

  /* counters to keep track of timestamps */
  /* Readable with object lock, writable with both aag lock and object lock */
373

Stefan Sauer's avatar
Stefan Sauer committed
374
375
  /* Sample offset starting from 0 at aggregator.segment.start */
  gint64 offset;
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
};

#define GST_AUDIO_AGGREGATOR_LOCK(self)   g_mutex_lock (&(self)->priv->mutex);
#define GST_AUDIO_AGGREGATOR_UNLOCK(self) g_mutex_unlock (&(self)->priv->mutex);

static void gst_audio_aggregator_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_audio_aggregator_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
static void gst_audio_aggregator_dispose (GObject * object);

static gboolean gst_audio_aggregator_src_event (GstAggregator * agg,
    GstEvent * event);
static gboolean gst_audio_aggregator_sink_event (GstAggregator * agg,
    GstAggregatorPad * aggpad, GstEvent * event);
static gboolean gst_audio_aggregator_src_query (GstAggregator * agg,
    GstQuery * query);
393
394
395
static gboolean
gst_audio_aggregator_sink_query (GstAggregator * agg, GstAggregatorPad * aggpad,
    GstQuery * query);
396
397
398
399
400
401
static gboolean gst_audio_aggregator_start (GstAggregator * agg);
static gboolean gst_audio_aggregator_stop (GstAggregator * agg);
static GstFlowReturn gst_audio_aggregator_flush (GstAggregator * agg);

static GstBuffer *gst_audio_aggregator_create_output_buffer (GstAudioAggregator
    * aagg, guint num_frames);
402
403
static GstBuffer *gst_audio_aggregator_do_clip (GstAggregator * agg,
    GstAggregatorPad * bpad, GstBuffer * buffer);
404
405
static GstFlowReturn gst_audio_aggregator_aggregate (GstAggregator * agg,
    gboolean timeout);
406
static gboolean sync_pad_values (GstElement * aagg, GstPad * pad, gpointer ud);
407
408
static gboolean gst_audio_aggregator_negotiated_src_caps (GstAggregator * agg,
    GstCaps * caps);
409
410
411
412
413
static GstFlowReturn
gst_audio_aggregator_update_src_caps (GstAggregator * agg,
    GstCaps * caps, GstCaps ** ret);
static GstCaps *gst_audio_aggregator_fixate_src_caps (GstAggregator * agg,
    GstCaps * caps);
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435

#define DEFAULT_OUTPUT_BUFFER_DURATION (10 * GST_MSECOND)
#define DEFAULT_ALIGNMENT_THRESHOLD   (40 * GST_MSECOND)
#define DEFAULT_DISCONT_WAIT (1 * GST_SECOND)

enum
{
  PROP_0,
  PROP_OUTPUT_BUFFER_DURATION,
  PROP_ALIGNMENT_THRESHOLD,
  PROP_DISCONT_WAIT,
};

G_DEFINE_ABSTRACT_TYPE (GstAudioAggregator, gst_audio_aggregator,
    GST_TYPE_AGGREGATOR);

static GstClockTime
gst_audio_aggregator_get_next_time (GstAggregator * agg)
{
  GstClockTime next_time;

  GST_OBJECT_LOCK (agg);
436
  if (agg->segment.position == -1 || agg->segment.position < agg->segment.start)
437
438
439
    next_time = agg->segment.start;
  else
    next_time = agg->segment.position;
440
441
442
443
444
445

  if (agg->segment.stop != -1 && next_time > agg->segment.stop)
    next_time = agg->segment.stop;

  next_time =
      gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, next_time);
446
447
448
449
450
  GST_OBJECT_UNLOCK (agg);

  return next_time;
}

451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
static GstBuffer *
gst_audio_aggregator_convert_once (GstAudioAggregator * aagg, GstPad * pad,
    GstAudioInfo * in_info, GstAudioInfo * out_info, GstBuffer * buffer)
{
  GstAudioConverter *converter =
      gst_audio_converter_new (GST_AUDIO_CONVERTER_FLAG_NONE,
      in_info, out_info, NULL);
  gint insize = gst_buffer_get_size (buffer);
  gsize insamples = insize / in_info->bpf;
  gsize outsamples = gst_audio_converter_get_out_frames (converter,
      insamples);
  gint outsize = outsamples * out_info->bpf;
  GstMapInfo inmap, outmap;
  GstBuffer *converted = gst_buffer_new_allocate (NULL, outsize, NULL);

  gst_buffer_copy_into (converted, buffer,
      GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS |
      GST_BUFFER_COPY_META, 0, -1);

  gst_buffer_map (buffer, &inmap, GST_MAP_READ);
  gst_buffer_map (converted, &outmap, GST_MAP_WRITE);

  gst_audio_converter_samples (converter,
      GST_AUDIO_CONVERTER_FLAG_NONE,
      (gpointer *) & inmap.data, insamples,
      (gpointer *) & outmap.data, outsamples);

  gst_buffer_unmap (buffer, &inmap);
  gst_buffer_unmap (converted, &outmap);
  gst_audio_converter_free (converter);

  return converted;
}

static GstBuffer *
gst_audio_aggregator_default_convert_buffer (GstAudioAggregator * aagg,
    GstPad * pad, GstAudioInfo * in_info, GstAudioInfo * out_info,
    GstBuffer * buffer)
{
  if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (pad))
    return
        gst_audio_aggregator_convert_pad_convert_buffer
        (GST_AUDIO_AGGREGATOR_CONVERT_PAD (pad),
        &GST_AUDIO_AGGREGATOR_PAD (pad)->info, out_info, buffer);
  else
    return gst_audio_aggregator_convert_once (aagg, pad, in_info, out_info,
        buffer);
}

static GstBuffer *
gst_audio_aggregator_convert_buffer (GstAudioAggregator * aagg, GstPad * pad,
    GstAudioInfo * in_info, GstAudioInfo * out_info, GstBuffer * buffer)
{
  GstAudioAggregatorClass *klass = GST_AUDIO_AGGREGATOR_GET_CLASS (aagg);

  g_assert (klass->convert_buffer);

  return klass->convert_buffer (aagg, pad, in_info, out_info, buffer);
}

511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
static void
gst_audio_aggregator_class_init (GstAudioAggregatorClass * klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstAggregatorClass *gstaggregator_class = (GstAggregatorClass *) klass;

  g_type_class_add_private (klass, sizeof (GstAudioAggregatorPrivate));

  gobject_class->set_property = gst_audio_aggregator_set_property;
  gobject_class->get_property = gst_audio_aggregator_get_property;
  gobject_class->dispose = gst_audio_aggregator_dispose;

  gstaggregator_class->src_event =
      GST_DEBUG_FUNCPTR (gst_audio_aggregator_src_event);
  gstaggregator_class->sink_event =
      GST_DEBUG_FUNCPTR (gst_audio_aggregator_sink_event);
  gstaggregator_class->src_query =
      GST_DEBUG_FUNCPTR (gst_audio_aggregator_src_query);
529
  gstaggregator_class->sink_query = gst_audio_aggregator_sink_query;
530
531
532
533
534
535
536
  gstaggregator_class->start = gst_audio_aggregator_start;
  gstaggregator_class->stop = gst_audio_aggregator_stop;
  gstaggregator_class->flush = gst_audio_aggregator_flush;
  gstaggregator_class->aggregate =
      GST_DEBUG_FUNCPTR (gst_audio_aggregator_aggregate);
  gstaggregator_class->clip = GST_DEBUG_FUNCPTR (gst_audio_aggregator_do_clip);
  gstaggregator_class->get_next_time = gst_audio_aggregator_get_next_time;
537
538
539
  gstaggregator_class->update_src_caps =
      GST_DEBUG_FUNCPTR (gst_audio_aggregator_update_src_caps);
  gstaggregator_class->fixate_src_caps = gst_audio_aggregator_fixate_src_caps;
540
541
  gstaggregator_class->negotiated_src_caps =
      gst_audio_aggregator_negotiated_src_caps;
542
543

  klass->create_output_buffer = gst_audio_aggregator_create_output_buffer;
544
  klass->convert_buffer = gst_audio_aggregator_default_convert_buffer;
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647

  GST_DEBUG_CATEGORY_INIT (audio_aggregator_debug, "audioaggregator",
      GST_DEBUG_FG_MAGENTA, "GstAudioAggregator");

  g_object_class_install_property (gobject_class, PROP_OUTPUT_BUFFER_DURATION,
      g_param_spec_uint64 ("output-buffer-duration", "Output Buffer Duration",
          "Output block size in nanoseconds", 1,
          G_MAXUINT64, DEFAULT_OUTPUT_BUFFER_DURATION,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_ALIGNMENT_THRESHOLD,
      g_param_spec_uint64 ("alignment-threshold", "Alignment Threshold",
          "Timestamp alignment threshold in nanoseconds", 0,
          G_MAXUINT64 - 1, DEFAULT_ALIGNMENT_THRESHOLD,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_DISCONT_WAIT,
      g_param_spec_uint64 ("discont-wait", "Discont Wait",
          "Window of time in nanoseconds to wait before "
          "creating a discontinuity", 0,
          G_MAXUINT64 - 1, DEFAULT_DISCONT_WAIT,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
}

static void
gst_audio_aggregator_init (GstAudioAggregator * aagg)
{
  aagg->priv =
      G_TYPE_INSTANCE_GET_PRIVATE (aagg, GST_TYPE_AUDIO_AGGREGATOR,
      GstAudioAggregatorPrivate);

  g_mutex_init (&aagg->priv->mutex);

  aagg->priv->output_buffer_duration = DEFAULT_OUTPUT_BUFFER_DURATION;
  aagg->priv->alignment_threshold = DEFAULT_ALIGNMENT_THRESHOLD;
  aagg->priv->discont_wait = DEFAULT_DISCONT_WAIT;

  aagg->current_caps = NULL;
  gst_audio_info_init (&aagg->info);

  gst_aggregator_set_latency (GST_AGGREGATOR (aagg),
      aagg->priv->output_buffer_duration, aagg->priv->output_buffer_duration);
}

static void
gst_audio_aggregator_dispose (GObject * object)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (object);

  gst_caps_replace (&aagg->current_caps, NULL);

  g_mutex_clear (&aagg->priv->mutex);

  G_OBJECT_CLASS (gst_audio_aggregator_parent_class)->dispose (object);
}

static void
gst_audio_aggregator_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (object);

  switch (prop_id) {
    case PROP_OUTPUT_BUFFER_DURATION:
      aagg->priv->output_buffer_duration = g_value_get_uint64 (value);
      gst_aggregator_set_latency (GST_AGGREGATOR (aagg),
          aagg->priv->output_buffer_duration,
          aagg->priv->output_buffer_duration);
      break;
    case PROP_ALIGNMENT_THRESHOLD:
      aagg->priv->alignment_threshold = g_value_get_uint64 (value);
      break;
    case PROP_DISCONT_WAIT:
      aagg->priv->discont_wait = g_value_get_uint64 (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_audio_aggregator_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (object);

  switch (prop_id) {
    case PROP_OUTPUT_BUFFER_DURATION:
      g_value_set_uint64 (value, aagg->priv->output_buffer_duration);
      break;
    case PROP_ALIGNMENT_THRESHOLD:
      g_value_set_uint64 (value, aagg->priv->alignment_threshold);
      break;
    case PROP_DISCONT_WAIT:
      g_value_set_uint64 (value, aagg->priv->discont_wait);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
/* Caps negotiation */

/* Unref after usage */
static GstAudioAggregatorPad *
gst_audio_aggregator_get_first_configured_pad (GstAggregator * agg)
{
  GstAudioAggregatorPad *res = NULL;
  GList *l;

  GST_OBJECT_LOCK (agg);
  for (l = GST_ELEMENT (agg)->sinkpads; l; l = l->next) {
    GstAudioAggregatorPad *aaggpad = l->data;

    if (GST_AUDIO_INFO_FORMAT (&aaggpad->info) != GST_AUDIO_FORMAT_UNKNOWN) {
      res = gst_object_ref (aaggpad);
      break;
    }
  }
  GST_OBJECT_UNLOCK (agg);

  return res;
}

static GstCaps *
gst_audio_aggregator_sink_getcaps (GstPad * pad, GstAggregator * agg,
    GstCaps * filter)
{
  GstAudioAggregatorPad *first_configured_pad =
      gst_audio_aggregator_get_first_configured_pad (agg);
  GstCaps *sink_template_caps = gst_pad_get_pad_template_caps (pad);
  GstCaps *downstream_caps = gst_pad_get_allowed_caps (agg->srcpad);
  GstCaps *sink_caps;
  GstStructure *s, *s2;
  gint downstream_rate;

  sink_template_caps = gst_caps_make_writable (sink_template_caps);
  s = gst_caps_get_structure (sink_template_caps, 0);

  if (downstream_caps && !gst_caps_is_empty (downstream_caps))
    s2 = gst_caps_get_structure (downstream_caps, 0);
  else
    s2 = NULL;

  if (s2 && gst_structure_get_int (s2, "rate", &downstream_rate)) {
    gst_structure_fixate_field_nearest_int (s, "rate", downstream_rate);
  } else if (first_configured_pad) {
    gst_structure_fixate_field_nearest_int (s, "rate",
        first_configured_pad->info.rate);
  }

  if (first_configured_pad)
    gst_object_unref (first_configured_pad);

  sink_caps = filter ? gst_caps_intersect (sink_template_caps,
      filter) : gst_caps_ref (sink_template_caps);

  GST_INFO_OBJECT (pad, "Getting caps with filter %" GST_PTR_FORMAT, filter);
  GST_DEBUG_OBJECT (pad, "sink template caps : %" GST_PTR_FORMAT,
      sink_template_caps);
  GST_DEBUG_OBJECT (pad, "downstream caps %" GST_PTR_FORMAT, downstream_caps);
  GST_INFO_OBJECT (pad, "returned sink caps : %" GST_PTR_FORMAT, sink_caps);

  gst_caps_unref (sink_template_caps);

  if (downstream_caps)
    gst_caps_unref (downstream_caps);

  return sink_caps;
}

static gboolean
gst_audio_aggregator_sink_setcaps (GstAudioAggregatorPad * aaggpad,
    GstAggregator * agg, GstCaps * caps)
{
  GstAudioAggregatorPad *first_configured_pad =
      gst_audio_aggregator_get_first_configured_pad (agg);
  GstCaps *downstream_caps = gst_pad_get_allowed_caps (agg->srcpad);
  GstAudioInfo info;
  gboolean ret = TRUE;
  gint downstream_rate;
  GstStructure *s;

  if (!downstream_caps || gst_caps_is_empty (downstream_caps)) {
    ret = FALSE;
    goto done;
  }

  gst_audio_info_from_caps (&info, caps);
  s = gst_caps_get_structure (downstream_caps, 0);

  /* TODO: handle different rates on sinkpads, a bit complex
   * because offsets will have to be updated, and audio resampling
   * has a latency to take into account
   */
  if ((gst_structure_get_int (s, "rate", &downstream_rate)
          && info.rate != downstream_rate) || (first_configured_pad
          && info.rate != first_configured_pad->info.rate)) {
    gst_pad_push_event (GST_PAD (aaggpad), gst_event_new_reconfigure ());
    gst_object_unref (first_configured_pad);
    ret = FALSE;
  } else {
    GST_OBJECT_LOCK (aaggpad);
    gst_audio_info_from_caps (&aaggpad->info, caps);
    if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad))
      GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad)->
          priv->converter_config_changed = TRUE;
    GST_OBJECT_UNLOCK (aaggpad);
  }

done:
  if (downstream_caps)
    gst_caps_unref (downstream_caps);

  return ret;
}

static GstFlowReturn
gst_audio_aggregator_update_src_caps (GstAggregator * agg,
    GstCaps * caps, GstCaps ** ret)
{
  GstCaps *src_template_caps = gst_pad_get_pad_template_caps (agg->srcpad);
  GstCaps *downstream_caps =
      gst_pad_peer_query_caps (agg->srcpad, src_template_caps);

  gst_caps_unref (src_template_caps);

  *ret = gst_caps_intersect (caps, downstream_caps);

  GST_INFO ("Updated src caps to %" GST_PTR_FORMAT, *ret);

  if (downstream_caps)
    gst_caps_unref (downstream_caps);

  return GST_FLOW_OK;
}

/* At that point if the caps are not fixed, this means downstream
 * didn't have fully specified requirements, we'll just go ahead
 * and fixate raw audio fields using our first configured pad, we don't for
 * now need a more complicated heuristic
 */
static GstCaps *
gst_audio_aggregator_fixate_src_caps (GstAggregator * agg, GstCaps * caps)
{
  GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (agg);
  GstAudioAggregatorPad *first_configured_pad;

  if (!aaggclass->convert_buffer)
    return
        GST_AGGREGATOR_CLASS
        (gst_audio_aggregator_parent_class)->fixate_src_caps (agg, caps);

  first_configured_pad = gst_audio_aggregator_get_first_configured_pad (agg);

  if (first_configured_pad) {
    GstStructure *s, *s2;
    GstCaps *first_configured_caps =
        gst_audio_info_to_caps (&first_configured_pad->info);
    gint first_configured_rate, first_configured_channels;

    caps = gst_caps_make_writable (caps);
    s = gst_caps_get_structure (caps, 0);
    s2 = gst_caps_get_structure (first_configured_caps, 0);

    gst_structure_get_int (s2, "rate", &first_configured_rate);
    gst_structure_get_int (s2, "channels", &first_configured_channels);

    gst_structure_fixate_field_string (s, "format",
        gst_structure_get_string (s2, "format"));
    gst_structure_fixate_field_string (s, "layout",
        gst_structure_get_string (s2, "layout"));
    gst_structure_fixate_field_nearest_int (s, "rate", first_configured_rate);
    gst_structure_fixate_field_nearest_int (s, "channels",
        first_configured_channels);

    gst_caps_unref (first_configured_caps);
    gst_object_unref (first_configured_pad);
  }

  if (!gst_caps_is_fixed (caps))
    caps = gst_caps_fixate (caps);

  GST_INFO_OBJECT (agg, "Fixated src caps to %" GST_PTR_FORMAT, caps);

  return caps;
}

/* Must be called with OBJECT_LOCK taken */
static void
gst_audio_aggregator_update_converters (GstAudioAggregator * aagg,
    GstAudioInfo * new_info)
{
  GList *l;

  for (l = GST_ELEMENT (aagg)->sinkpads; l; l = l->next) {
    GstAudioAggregatorPad *aaggpad = l->data;

    if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad))
      GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad)->
          priv->converter_config_changed = TRUE;

    /* If we currently were mixing a buffer, we need to convert it to the new
     * format */
    if (aaggpad->priv->buffer) {
      GstBuffer *new_converted_buffer =
          gst_audio_aggregator_convert_buffer (aagg, GST_PAD (aaggpad),
          &aaggpad->info, new_info, aaggpad->priv->input_buffer);
      gst_buffer_replace (&aaggpad->priv->buffer, new_converted_buffer);
    }
  }
}

/* We now have our final output caps, we can create the required converters */
static gboolean
gst_audio_aggregator_negotiated_src_caps (GstAggregator * agg, GstCaps * caps)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);
  GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (agg);
  GstAudioInfo info;

  GST_INFO_OBJECT (agg, "src caps negotiated %" GST_PTR_FORMAT, caps);

  if (!gst_audio_info_from_caps (&info, caps)) {
    GST_WARNING_OBJECT (aagg, "Rejecting invalid caps: %" GST_PTR_FORMAT, caps);
    return FALSE;
  }

  GST_AUDIO_AGGREGATOR_LOCK (aagg);
  GST_OBJECT_LOCK (aagg);

  if (aaggclass->convert_buffer) {
    gst_audio_aggregator_update_converters (aagg, &info);

    if (aagg->priv->current_buffer
        && !gst_audio_info_is_equal (&aagg->info, &info)) {
      GstBuffer *converted =
          gst_audio_aggregator_convert_buffer (aagg, agg->srcpad, &aagg->info,
          &info, aagg->priv->current_buffer);
      gst_buffer_unref (aagg->priv->current_buffer);
      aagg->priv->current_buffer = converted;
    }
  }

  if (!gst_audio_info_is_equal (&info, &aagg->info)) {
    GST_INFO_OBJECT (aagg, "setting caps to %" GST_PTR_FORMAT, caps);
    gst_caps_replace (&aagg->current_caps, caps);

    memcpy (&aagg->info, &info, sizeof (info));
  }

  GST_OBJECT_UNLOCK (aagg);
  GST_AUDIO_AGGREGATOR_UNLOCK (aagg);

  return
      GST_AGGREGATOR_CLASS
      (gst_audio_aggregator_parent_class)->negotiated_src_caps (agg, caps);
}
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938

/* event handling */

static gboolean
gst_audio_aggregator_src_event (GstAggregator * agg, GstEvent * event)
{
  gboolean result;

  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);
  GST_DEBUG_OBJECT (agg->srcpad, "Got %s event on src pad",
      GST_EVENT_TYPE_NAME (event));

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_QOS:
      /* QoS might be tricky */
      gst_event_unref (event);
      return FALSE;
    case GST_EVENT_NAVIGATION:
      /* navigation is rather pointless. */
      gst_event_unref (event);
      return FALSE;
      break;
    case GST_EVENT_SEEK:
    {
      GstSeekFlags flags;
      gdouble rate;
      GstSeekType start_type, stop_type;
      gint64 start, stop;
      GstFormat seek_format, dest_format;

      /* parse the seek parameters */
      gst_event_parse_seek (event, &rate, &seek_format, &flags, &start_type,
          &start, &stop_type, &stop);

Stefan Sauer's avatar
Stefan Sauer committed
939
      /* Check the seeking parameters before linking up */
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
      if ((start_type != GST_SEEK_TYPE_NONE)
          && (start_type != GST_SEEK_TYPE_SET)) {
        result = FALSE;
        GST_DEBUG_OBJECT (aagg,
            "seeking failed, unhandled seek type for start: %d", start_type);
        goto done;
      }
      if ((stop_type != GST_SEEK_TYPE_NONE) && (stop_type != GST_SEEK_TYPE_SET)) {
        result = FALSE;
        GST_DEBUG_OBJECT (aagg,
            "seeking failed, unhandled seek type for end: %d", stop_type);
        goto done;
      }

      GST_OBJECT_LOCK (agg);
      dest_format = agg->segment.format;
      GST_OBJECT_UNLOCK (agg);
      if (seek_format != dest_format) {
        result = FALSE;
        GST_DEBUG_OBJECT (aagg,
            "seeking failed, unhandled seek format: %s",
            gst_format_get_name (seek_format));
        goto done;
      }
    }
      break;
    default:
      break;
  }

  return
      GST_AGGREGATOR_CLASS (gst_audio_aggregator_parent_class)->src_event (agg,
      event);

done:
  return result;
}


static gboolean
gst_audio_aggregator_sink_event (GstAggregator * agg,
    GstAggregatorPad * aggpad, GstEvent * event)
{
983
  GstAudioAggregatorPad *aaggpad = GST_AUDIO_AGGREGATOR_PAD (aggpad);
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
  gboolean res = TRUE;

  GST_DEBUG_OBJECT (aggpad, "Got %s event on sink pad",
      GST_EVENT_TYPE_NAME (event));

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_SEGMENT:
    {
      const GstSegment *segment;
      gst_event_parse_segment (event, &segment);

      if (segment->format != GST_FORMAT_TIME) {
        GST_ERROR_OBJECT (agg, "Segment of type %s are not supported,"
            " only TIME segments are supported",
            gst_format_get_name (segment->format));
        gst_event_unref (event);
        event = NULL;
        res = FALSE;
        break;
      }

      GST_OBJECT_LOCK (agg);
      if (segment->rate != agg->segment.rate) {
        GST_ERROR_OBJECT (aggpad,
            "Got segment event with wrong rate %lf, expected %lf",
            segment->rate, agg->segment.rate);
        res = FALSE;
        gst_event_unref (event);
        event = NULL;
      } else if (segment->rate < 0.0) {
        GST_ERROR_OBJECT (aggpad, "Negative rates not supported yet");
        res = FALSE;
        gst_event_unref (event);
        event = NULL;
      } else {
        GstAudioAggregatorPad *pad = GST_AUDIO_AGGREGATOR_PAD (aggpad);

        GST_OBJECT_LOCK (pad);
        pad->priv->new_segment = TRUE;
        GST_OBJECT_UNLOCK (pad);
      }
      GST_OBJECT_UNLOCK (agg);

      break;
    }
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
    case GST_EVENT_CAPS:
    {
      GstCaps *caps;

      gst_event_parse_caps (event, &caps);
      GST_INFO_OBJECT (aggpad, "Got caps %" GST_PTR_FORMAT, caps);
      res = gst_audio_aggregator_sink_setcaps (aaggpad, agg, caps);
      gst_event_unref (event);
      event = NULL;
      break;
    }
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
    default:
      break;
  }

  if (event != NULL)
    return
        GST_AGGREGATOR_CLASS (gst_audio_aggregator_parent_class)->sink_event
        (agg, aggpad, event);

  return res;
}

1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
static gboolean
gst_audio_aggregator_sink_query (GstAggregator * agg, GstAggregatorPad * aggpad,
    GstQuery * query)
{
  gboolean res = FALSE;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CAPS:
    {
      GstCaps *filter, *caps;

      gst_query_parse_caps (query, &filter);
      caps = gst_audio_aggregator_sink_getcaps (GST_PAD (aggpad), agg, filter);
      gst_query_set_caps_result (query, caps);
      gst_caps_unref (caps);
      res = TRUE;
      break;
    }
    default:
      res =
          GST_AGGREGATOR_CLASS (gst_audio_aggregator_parent_class)->sink_query
          (agg, aggpad, query);
      break;
  }

  return res;
}


1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
/* FIXME, the duration query should reflect how long you will produce
 * data, that is the amount of stream time until you will emit EOS.
 *
 * For synchronized mixing this is always the max of all the durations
 * of upstream since we emit EOS when all of them finished.
 *
 * We don't do synchronized mixing so this really depends on where the
 * streams where punched in and what their relative offsets are against
 * eachother which we can get from the first timestamps we see.
 *
 * When we add a new stream (or remove a stream) the duration might
 * also become invalid again and we need to post a new DURATION
 * message to notify this fact to the parent.
 * For now we take the max of all the upstream elements so the simple
 * cases work at least somewhat.
 */
static gboolean
gst_audio_aggregator_query_duration (GstAudioAggregator * aagg,
    GstQuery * query)
{
  gint64 max;
  gboolean res;
  GstFormat format;
  GstIterator *it;
  gboolean done;
  GValue item = { 0, };

  /* parse format */
  gst_query_parse_duration (query, &format, NULL);

  max = -1;
  res = TRUE;
  done = FALSE;

  it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (aagg));
  while (!done) {
    GstIteratorResult ires;

    ires = gst_iterator_next (it, &item);
    switch (ires) {
      case GST_ITERATOR_DONE:
        done = TRUE;
        break;
      case GST_ITERATOR_OK:
      {
        GstPad *pad = g_value_get_object (&item);
        gint64 duration;

        /* ask sink peer for duration */
        res &= gst_pad_peer_query_duration (pad, format, &duration);
        /* take max from all valid return values */
        if (res) {
          /* valid unknown length, stop searching */
          if (duration == -1) {
            max = duration;
            done = TRUE;
          }
          /* else see if bigger than current max */
          else if (duration > max)
            max = duration;
        }
        g_value_reset (&item);
        break;
      }
      case GST_ITERATOR_RESYNC:
        max = -1;
        res = TRUE;
        gst_iterator_resync (it);
        break;
      default:
        res = FALSE;
        done = TRUE;
        break;
    }
  }
  g_value_unset (&item);
  gst_iterator_free (it);

  if (res) {
    /* and store the max */
    GST_DEBUG_OBJECT (aagg, "Total duration in format %s: %"
        GST_TIME_FORMAT, gst_format_get_name (format), GST_TIME_ARGS (max));
    gst_query_set_duration (query, format, max);
  }

  return res;
}


static gboolean
gst_audio_aggregator_src_query (GstAggregator * agg, GstQuery * query)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);
  gboolean res = FALSE;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_DURATION:
      res = gst_audio_aggregator_query_duration (aagg, query);
      break;
    case GST_QUERY_POSITION:
    {
      GstFormat format;

      gst_query_parse_position (query, &format, NULL);

      GST_OBJECT_LOCK (aagg);

      switch (format) {
        case GST_FORMAT_TIME:
1190
1191
1192
          gst_query_set_position (query, format,
              gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME,
                  agg->segment.position));
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
          res = TRUE;
          break;
        case GST_FORMAT_BYTES:
          if (GST_AUDIO_INFO_BPF (&aagg->info)) {
            gst_query_set_position (query, format, aagg->priv->offset *
                GST_AUDIO_INFO_BPF (&aagg->info));
            res = TRUE;
          }
          break;
        case GST_FORMAT_DEFAULT:
          gst_query_set_position (query, format, aagg->priv->offset);
          res = TRUE;
          break;
        default:
          break;
      }

      GST_OBJECT_UNLOCK (aagg);

      break;
    }
    default:
      res =
          GST_AGGREGATOR_CLASS (gst_audio_aggregator_parent_class)->src_query
          (agg, query);
      break;
  }

  return res;
}


void
gst_audio_aggregator_set_sink_caps (GstAudioAggregator * aagg,
    GstAudioAggregatorPad * pad, GstCaps * caps)
{
1229
#ifndef G_DISABLE_ASSERT
1230
1231
  gboolean valid;

1232
  GST_OBJECT_LOCK (pad);
1233
1234
  valid = gst_audio_info_from_caps (&pad->info, caps);
  g_assert (valid);
1235
  GST_OBJECT_UNLOCK (pad);
1236
1237
1238
1239
1240
#else
  GST_OBJECT_LOCK (pad);
  (void) gst_audio_info_from_caps (&pad->info, caps);
  GST_OBJECT_UNLOCK (pad);
#endif
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
}

/* Must hold object lock and aagg lock to call */

static void
gst_audio_aggregator_reset (GstAudioAggregator * aagg)
{
  GstAggregator *agg = GST_AGGREGATOR (aagg);

  GST_AUDIO_AGGREGATOR_LOCK (aagg);
  GST_OBJECT_LOCK (aagg);
  agg->segment.position = -1;
1253
  aagg->priv->offset = -1;
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
  gst_audio_info_init (&aagg->info);
  gst_caps_replace (&aagg->current_caps, NULL);
  gst_buffer_replace (&aagg->priv->current_buffer, NULL);
  GST_OBJECT_UNLOCK (aagg);
  GST_AUDIO_AGGREGATOR_UNLOCK (aagg);
}

static gboolean
gst_audio_aggregator_start (GstAggregator * agg)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);

  gst_audio_aggregator_reset (aagg);

  return TRUE;
}

static gboolean
gst_audio_aggregator_stop (GstAggregator * agg)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);

  gst_audio_aggregator_reset (aagg);

  return TRUE;
}

static GstFlowReturn
gst_audio_aggregator_flush (GstAggregator * agg)
{
  GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);

  GST_AUDIO_AGGREGATOR_LOCK (aagg);
  GST_OBJECT_LOCK (aagg);
  agg->segment.position = -1;
1289
  aagg->priv->offset = -1;
1290
1291
1292
1293
1294
1295
1296
  gst_buffer_replace (&aagg->priv->current_buffer, NULL);
  GST_OBJECT_UNLOCK (aagg);
  GST_AUDIO_AGGREGATOR_UNLOCK (aagg);

  return GST_FLOW_OK;
}

1297
static GstBuffer *
1298
gst_audio_aggregator_do_clip (GstAggregator * agg,
1299
    GstAggregatorPad * bpad, GstBuffer * buffer)
1300
1301
1302
1303
1304
1305
1306
1307
{
  GstAudioAggregatorPad *pad = GST_AUDIO_AGGREGATOR_PAD (bpad);
  gint rate, bpf;

  rate = GST_AUDIO_INFO_RATE (&pad->info);
  bpf = GST_AUDIO_INFO_BPF (&pad->info);

  GST_OBJECT_LOCK (bpad);
1308
  buffer = gst_audio_buffer_clip (buffer, &bpad->segment, rate, bpf);
1309
1310
  GST_OBJECT_UNLOCK (bpad);

1311
  return buffer;
1312
1313
1314
1315
}

/* Called with the object lock for both the element and pad held,
 * as well as the aagg lock
1316
1317
1318
 *
 * Replace the current buffer with input and update GstAudioAggregatorPadPrivate
 * values.
1319
1320
 */
static gboolean
1321
1322
gst_audio_aggregator_fill_buffer (GstAudioAggregator * aagg,
    GstAudioAggregatorPad * pad)
1323
{
1324
  GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (aagg);
1325
1326
1327
1328
1329
  GstClockTime start_time, end_time;
  gboolean discont = FALSE;
  guint64 start_offset, end_offset;
  gint rate, bpf;

1330
  GstAggregator *agg = GST_AGGREGATOR (aagg);
1331
1332
  GstAggregatorPad *aggpad = GST_AGGREGATOR_PAD (pad);

1333
1334
1335
1336
1337
1338
1339
  if (aaggclass->convert_buffer) {
    rate = GST_AUDIO_INFO_RATE (&aagg->info);
    bpf = GST_AUDIO_INFO_BPF (&aagg->info);
  } else {
    rate = GST_AUDIO_INFO_RATE (&pad->info);
    bpf = GST_AUDIO_INFO_BPF (&pad->info);
  }
1340
1341

  pad->priv->position = 0;
1342
  pad->priv->size = gst_buffer_get_size (pad->priv->buffer) / bpf;
1343

1344
  if (pad->priv->size == 0) {
1345
1346
    if (!GST_BUFFER_DURATION_IS_VALID (pad->priv->buffer) ||
        !GST_BUFFER_FLAG_IS_SET (pad->priv->buffer, GST_BUFFER_FLAG_GAP)) {
1347
      GST_WARNING_OBJECT (pad, "Dropping 0-sized buffer missing either a"
1348
          " duration or a GAP flag: %" GST_PTR_FORMAT, pad->priv->buffer);
1349
1350
1351
      return FALSE;
    }

1352
1353
    pad->priv->size =
        gst_util_uint64_scale (GST_BUFFER_DURATION (pad->priv->buffer), rate,
1354
1355
1356
        GST_SECOND);
  }

1357
  if (!GST_BUFFER_PTS_IS_VALID (pad->priv->buffer)) {
1358
1359
1360
1361
1362
1363
1364
1365
1366
    if (pad->priv->output_offset == -1)
      pad->priv->output_offset = aagg->priv->offset;
    if (pad->priv->next_offset == -1)
      pad->priv->next_offset = pad->priv->size;
    else
      pad->priv->next_offset += pad->priv->size;
    goto done;
  }

1367
  start_time = GST_BUFFER_PTS (pad->priv->buffer);
1368
1369
1370
1371
  end_time =
      start_time + gst_util_uint64_scale_ceil (pad->priv->size, GST_SECOND,
      rate);

1372
1373
1374
1375
1376
1377
  /* Clipping should've ensured this */
  g_assert (start_time >= aggpad->segment.start);

  start_offset =
      gst_util_uint64_scale (start_time - aggpad->segment.start, rate,
      GST_SECOND);
1378
1379
  end_offset = start_offset + pad->priv->size;

1380
1381
  if (GST_BUFFER_IS_DISCONT (pad->priv->buffer)
      || GST_BUFFER_FLAG_IS_SET (pad->priv->buffer, GST_BUFFER_FLAG_RESYNC)
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
      || pad->priv->new_segment || pad->priv->next_offset == -1) {
    discont = TRUE;
    pad->priv->new_segment = FALSE;
  } else {
    guint64 diff, max_sample_diff;

    /* Check discont, based on audiobasesink */
    if (start_offset <= pad->priv->next_offset)
      diff = pad->priv->next_offset - start_offset;
    else
      diff = start_offset - pad->priv->next_offset;

    max_sample_diff =
        gst_util_uint64_scale_int (aagg->priv->alignment_threshold, rate,
        GST_SECOND);

    /* Discont! */
    if (G_UNLIKELY (diff >= max_sample_diff)) {
      if (aagg->priv->discont_wait > 0) {
        if (pad->priv->discont_time == GST_CLOCK_TIME_NONE) {
          pad->priv->discont_time = start_time;
        } else if (start_time - pad->priv->discont_time >=
            aagg->priv->discont_wait) {
          discont = TRUE;
          pad->priv->discont_time = GST_CLOCK_TIME_NONE;
        }
      } else {
        discont = TRUE;
      }
    } else if (G_UNLIKELY (pad->priv->discont_time != GST_CLOCK_TIME_NONE)) {
      /* we have had a discont, but are now back on track! */
      pad->priv->discont_time = GST_CLOCK_TIME_NONE;
    }
  }

  if (discont) {
    /* Have discont, need resync */
    if (pad->priv->next_offset != -1)
1420
      GST_DEBUG_OBJECT (pad, "Have discont. Expected %"
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
          G_GUINT64_FORMAT ", got %" G_GUINT64_FORMAT,
          pad->priv->next_offset, start_offset);
    pad->priv->output_offset = -1;
    pad->priv->next_offset = end_offset;
  } else {
    pad->priv->next_offset += pad->priv->size;
  }

  if (pad->priv->output_offset == -1) {
    GstClockTime start_running_time;
    GstClockTime end_running_time;
1432
1433
1434
    GstClockTime segment_pos;
    guint64 start_output_offset = -1;
    guint64 end_output_offset = -1;
1435
1436
1437
1438
1439
1440
1441
1442

    start_running_time =
        gst_segment_to_running_time (&aggpad->segment,
        GST_FORMAT_TIME, start_time);
    end_running_time =
        gst_segment_to_running_time (&aggpad->segment,
        GST_FORMAT_TIME, end_time);

1443
    /* Convert to position in the output segment */
1444
    segment_pos =
1445
        gst_segment_position_from_running_time (&agg->segment, GST_FORMAT_TIME,
1446
        start_running_time);
1447
    if (GST_CLOCK_TIME_IS_VALID (segment_pos))
1448
      start_output_offset =
1449
          gst_util_uint64_scale (segment_pos - agg->segment.start, rate,
1450
1451
          GST_SECOND);

1452
    segment_pos =
1453
        gst_segment_position_from_running_time (&agg->segment, GST_FORMAT_TIME,
1454
        end_running_time);
1455
    if (GST_CLOCK_TIME_IS_VALID (segment_pos))
1456
      end_output_offset =
1457
          gst_util_uint64_scale (segment_pos - agg->segment.start, rate,
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
          GST_SECOND);

    if (start_output_offset == -1 && end_output_offset == -1) {
      /* Outside output segment, drop */
      pad->priv->position = 0;
      pad->priv->size = 0;
      pad->priv->output_offset = -1;
      GST_DEBUG_OBJECT (pad, "Buffer outside output segment");
      return FALSE;
    }

    /* Calculate end_output_offset if it was outside the output segment */
    if (end_output_offset == -1)
      end_output_offset = start_output_offset + pad->priv->size;

    if (end_output_offset < aagg->priv->offset) {
1474
1475
1476
1477
1478
      pad->priv->position = 0;
      pad->priv->size = 0;
      pad->priv->output_offset = -1;
      GST_DEBUG_OBJECT (pad,
          "Buffer before segment or current position: %" G_GUINT64_FORMAT " < %"
Vineeth TM's avatar
Vineeth TM committed
1479
          G_GINT64_FORMAT, end_output_offset, aagg->priv->offset);
1480
1481
1482
      return FALSE;
    }

1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
    if (start_output_offset == -1 || start_output_offset < aagg->priv->offset) {
      guint diff;

      if (start_output_offset == -1 && end_output_offset < pad->priv->size) {
        diff = pad->priv->size - end_output_offset + aagg->priv->offset;
      } else if (start_output_offset == -1) {
        start_output_offset = end_output_offset - pad->priv->size;

        if (start_output_offset < aagg->priv->offset)
          diff = aagg->priv->offset - start_output_offset;
        else
          diff = 0;
      } else {
        diff = aagg->priv->offset - start_output_offset;
      }
1498
1499
1500
1501
1502
1503
1504
1505
1506

      pad->priv->position += diff;
      if (pad->priv->position >= pad->priv->size) {
        /* Empty buffer, drop */
        pad->priv->position = 0;
        pad->priv->size = 0;
        pad->priv->output_offset = -1;
        GST_DEBUG_OBJECT (pad,
            "Buffer before segment or current position: %" G_GUINT64_FORMAT
Vineeth TM's avatar
Vineeth TM committed
1507
            " < %" G_GINT64_FORMAT, end_output_offset, aagg->priv->offset);
1508
1509
1510
1511
        return FALSE;
      }
    }

1512
1513
1514
1515
1516
    if (start_output_offset == -1 || start_output_offset < aagg->priv->offset)
      pad->priv->output_offset = aagg->priv->offset;
    else
      pad->priv->output_offset = start_output_offset;

1517
1518
    GST_DEBUG_OBJECT (pad,
        "Buffer resynced: Pad offset %" G_GUINT64_FORMAT
Vineeth TM's avatar
Vineeth TM committed
1519
        ", current audio aggregator offset %" G_GINT64_FORMAT,
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
        pad->priv->output_offset, aagg->priv->offset);
  }

done:

  GST_LOG_OBJECT (pad,
      "Queued new buffer at offset %" G_GUINT64_FORMAT,
      pad->priv->output_offset);

  return TRUE;
}

/* Called with pad object lock held */

static gboolean
gst_audio_aggregator_mix_buffer (GstAudioAggregator * aagg,
1536
1537
    GstAudioAggregatorPad * pad, GstBuffer * inbuf, GstBuffer * outbuf,
    guint blocksize)
1538
1539
1540
1541
{
  guint overlap;
  guint out_start;
  gboolean filled;
1542
1543
  guint in_offset;
  gboolean pad_changed = FALSE;
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557

  /* Overlap => mix */
  if (aagg->priv->offset < pad->priv->output_offset)
    out_start = pad->priv->output_offset - aagg->priv->offset;
  else
    out_start = 0;

  overlap = pad->priv->size - pad->priv->position;
  if (overlap > blocksize - out_start)
    overlap = blocksize - out_start;

  if (GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_GAP)) {
    /* skip gap buffer */
    GST_LOG_OBJECT (pad, "skipping GAP buffer");
1558
    pad->priv->output_offset += pad->priv->size - pad->priv->position;
1559
1560
1561
    pad->priv->position = pad->priv->size;

    gst_buffer_replace (&pad->priv->buffer, NULL);
1562
    gst_buffer_replace (&pad->priv->input_buffer, NULL);
1563
1564
1565
    return FALSE;
  }

1566
1567
1568
1569
1570
  gst_buffer_ref (inbuf);
  in_offset = pad->priv->position;
  GST_OBJECT_UNLOCK (pad);
  GST_OBJECT_UNLOCK (aagg);

1571
  filled = GST_AUDIO_AGGREGATOR_GET_CLASS (aagg)->aggregate_one_buffer (aagg,
1572
1573
1574
1575
1576
1577
1578
      pad, inbuf, in_offset, outbuf, out_start, overlap);

  GST_OBJECT_LOCK (aagg);
  GST_OBJECT_LOCK (pad);

  pad_changed = (inbuf != pad->priv->buffer);
  gst_buffer_unref (inbuf);
1579
1580
1581
1582

  if (filled)
    GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_GAP);

1583
1584
1585
  if (pad_changed)
    return FALSE;

1586
1587
1588
1589
1590
1591
  pad->priv->position += overlap;
  pad->priv->output_offset += overlap;

  if (pad->priv->position == pad->priv->size) {
    /* Buffer done, drop it */
    gst_buffer_replace (&pad->priv->buffer, NULL);
1592
    gst_buffer_replace (&pad->priv->input_buffer, NULL);
1593
    GST_LOG_OBJECT (pad, "Finished mixing buffer, waiting for next");
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
    return FALSE;
  }

  return TRUE;
}

static GstBuffer *
gst_audio_aggregator_create_output_buffer (GstAudioAggregator * aagg,
    guint num_frames)
{
1604
1605
1606
  GstAllocator *allocator;
  GstAllocationParams params;
  GstBuffer *outbuf;
1607
1608
  GstMapInfo outmap;

1609
1610
  gst_aggregator_get_allocator (GST_AGGREGATOR (aagg), &allocator, &params);

1611
1612
1613
  GST_DEBUG ("Creating output buffer with size %d",
      num_frames * GST_AUDIO_INFO_BPF (&aagg->info));

1614
1615
1616
1617
1618
1619
  outbuf = gst_buffer_new_allocate (allocator, num_frames *
      GST_AUDIO_INFO_BPF (&aagg->info), &params);

  if (allocator)
    gst_object_unref (allocator);

1620
1621
1622
1623
1624
1625
1626
  gst_buffer_map (outbuf, &outmap, GST_MAP_WRITE);
  gst_audio_format_fill_silence (aagg->info.finfo, outmap.data, outmap.size);
  gst_buffer_unmap (outbuf, &outmap);

  return outbuf;
}

1627
static gboolean
1628
sync_pad_values (GstElement * aagg, GstPad * pad, gpointer user_data)
1629
{
1630
1631
  GstAudioAggregatorPad *aapad = GST_AUDIO_AGGREGATOR_PAD (pad);
  GstAggregatorPad *bpad = GST_AGGREGATOR_PAD_CAST (pad);
1632
1633
  GstClockTime timestamp, stream_time;

1634
  if (aapad->priv->buffer == NULL)
1635
1636
    return TRUE;

1637
  timestamp = GST_BUFFER_PTS (aapad->priv->buffer);
1638
1639
1640
1641
1642
1643
1644
1645
  GST_OBJECT_LOCK (bpad);
  stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME,
      timestamp);
  GST_OBJECT_UNLOCK (bpad);

  /* sync object properties on stream time */
  /* TODO: Ideally we would want to do that on every sample */
  if (GST_CLOCK_TIME_IS_VALID (stream_time))
1646
    gst_object_sync_values (GST_OBJECT_CAST (pad), stream_time);
1647
1648
1649
1650

  return TRUE;
}

1651
1652
1653
static GstFlowReturn
gst_audio_aggregator_aggregate (GstAggregator * agg, gboolean timeout)
{
1654
1655
  /* Calculate the current output offset/timestamp and offset_end/timestamp_end.
   * Allocate a silence buffer for this and store it.
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
   *
   * For all pads:
   * 1) Once per input buffer (cached)
   *   1) Check discont (flag and timestamp with tolerance)
   *   2) If discont or new, resync. That means:
   *     1) Drop all start data of the buffer that comes before
   *        the current position/offset.
   *     2) Calculate the offset (output segment!) that the first
   *        frame of the input buffer corresponds to. Base this on
   *        the running time.
   *
   * 2) If the current pad's offset/offset_end overlaps with the output
   *    offset/offset_end, mix it at the appropiate position in the output
   *    buffer and advance the pad's position. Remember if this pad needs
   *    a new buffer to advance behind the output offset_end.
   *
1672
   * If we had no pad with a buffer, go EOS.
1673
   *
1674
1675
1676
   * If we had at least one pad that did not advance behind output
   * offset_end, let aggregate be called again for the current
   * output offset/offset_end.
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
   */
  GstElement *element;
  GstAudioAggregator *aagg;
  GList *iter;
  GstFlowReturn ret;
  GstBuffer *outbuf = NULL;
  gint64 next_offset;
  gint64 next_timestamp;
  gint rate, bpf;
  gboolean dropped = FALSE;
  gboolean is_eos = TRUE;
  gboolean is_done = TRUE;
  guint blocksize;

  element = GST_ELEMENT (agg);
  aagg = GST_AUDIO_AGGREGATOR (agg);

1694
  /* Sync pad properties to the stream time */
1695
  gst_element_foreach_sink_pad (element, sync_pad_values, NULL);
1696

1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
  GST_AUDIO_AGGREGATOR_LOCK (aagg);
  GST_OBJECT_LOCK (agg);

  /* Update position from the segment start/stop if needed */
  if (agg->segment.position == -1) {
    if (agg->segment.rate > 0.0)
      agg->segment.position = agg->segment.start;
    else
      agg->segment.position = agg->segment.stop;
  }

  if (G_UNLIKELY (aagg->info.finfo->format == GST_AUDIO_FORMAT_UNKNOWN)) {
    if (timeout) {
      GST_DEBUG_OBJECT (aagg,
          "Got timeout before receiving any caps, don't output anything");

      /* Advance position */
      if (agg->segment.rate > 0.0)
        agg->segment.position += aagg->priv->output_buffer_duration;
      else if (agg->segment.position > aagg->priv->output_buffer_duration)
        agg->segment.position -= aagg->priv->output_buffer_duration;
      else
        agg->segment.position = 0;

      GST_OBJECT_UNLOCK (agg);
      GST_AUDIO_AGGREGATOR_UNLOCK (aagg);
1723
      return GST_AGGREGATOR_FLOW_NEED_DATA;
1724
1725
1726
1727
1728
1729
1730
1731
1732
    } else {
      GST_OBJECT_UNLOCK (agg);
      goto not_negotiated;
    }
  }

  rate = GST_AUDIO_INFO_RATE (&aagg->info);
  bpf = GST_AUDIO_INFO_BPF (&aagg->info);

1733
1734
1735
1736
  if (aagg->priv->offset == -1) {
    aagg->priv->offset =
        gst_util_uint64_scale (agg->segment.position - agg->segment.start, rate,
        GST_SECOND);
Vineeth TM's avatar
Vineeth TM committed
1737
1738
    GST_DEBUG_OBJECT (aagg, "Starting at offset %" G_GINT64_FORMAT,
        aagg->priv->offset);
1739
1740
  }

1741
  blocksize = gst_util_uint64_scale (aagg->priv->output_buffer_duration,
1742
      rate, GST_SECOND);
1743
  blocksize = MAX (1, blocksize);
1744
1745
1746
1747
1748
1749
1750
1751

  /* FIXME: Reverse mixing does not work at all yet */
  if (agg->segment.rate > 0.0) {
    next_offset = aagg->priv->offset + blocksize;
  } else {
    next_offset = aagg->priv->offset - blocksize;
  }

1752
  /* Use the sample counter, which will never accumulate rounding errors */
1753
1754
1755
  next_timestamp =
      agg->segment.start + gst_util_uint64_scale (next_offset, GST_SECOND,
      rate);
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768

  if (aagg->priv->current_buffer == NULL) {
    GST_OBJECT_UNLOCK (agg);
    aagg->priv->current_buffer =
        GST_AUDIO_AGGREGATOR_GET_CLASS (aagg)->create_output_buffer (aagg,
        blocksize);
    /* Be careful, some things could have changed ? */
    GST_OBJECT_LOCK (agg);
    GST_BUFFER_FLAG_SET (aagg->priv->current_buffer, GST_BUFFER_FLAG_GAP);
  }
  outbuf = aagg->priv->current_buffer;

  GST_LOG_OBJECT (agg,
Vineeth TM's avatar
Vineeth TM committed
1769
      "Starting to mix %u samples for offset %" G_GINT64_FORMAT
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
      " with timestamp %" GST_TIME_FORMAT, blocksize,
      aagg->priv->offset, GST_TIME_ARGS (agg->segment.position));

  for (iter = element->sinkpads; iter; iter = iter->next) {
    GstAudioAggregatorPad *pad = (GstAudioAggregatorPad *) iter->data;
    GstAggregatorPad *aggpad = (GstAggregatorPad *) iter->data;
    gboolean pad_eos = gst_aggregator_pad_is_eos (aggpad);

    if (!pad_eos)
      is_eos = FALSE;

1781
    pad->priv->input_buffer = gst_aggregator_pad_get_buffer (aggpad);
1782
1783

    GST_OBJECT_LOCK (pad);
1784
    if (!pad->priv->input_buffer) {
1785
1786
1787
      if (timeout) {
        if (pad->priv->output_offset < next_offset) {
          gint64 diff = next_offset - pad->priv->output_offset;
1788
1789
          GST_DEBUG_OBJECT (pad, "Timeout, missing %" G_GINT64_FORMAT
              " frames (%" GST_TIME_FORMAT ")", diff,
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
              GST_TIME_ARGS (gst_util_uint64_scale (diff, GST_SECOND,
                      GST_AUDIO_INFO_RATE (&aagg->info))));
        }
      } else if (!pad_eos) {
        is_done = FALSE;
      }
      GST_OBJECT_UNLOCK (pad);
      continue;
    }

    /* New buffer? */
    if (!pad->priv->buffer) {
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
      if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (pad))
        pad->priv->buffer =
            gst_audio_aggregator_convert_buffer
            (aagg, GST_PAD (pad), &pad->info, &aagg->info,
            pad->priv->input_buffer);
      else
        pad->priv->buffer = gst_buffer_ref (pad->priv->input_buffer);

      if (!gst_audio_aggregator_fill_buffer (aagg, pad)) {
        gst_buffer_replace (&pad->priv->buffer, NULL);
        gst_buffer_replace (&pad->priv->input_buffer, NULL);
        pad->priv->buffer = NULL;
1814
1815
        dropped = TRUE;
        GST_OBJECT_UNLOCK (pad);
1816

1817
1818
1819
1820
        gst_aggregator_pad_drop_buffer (aggpad);
        continue;
      }
    } else {
1821
      gst_buffer_unref (pad->priv->input_buffer);
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
    }

    if (!pad->priv->buffer && !dropped && pad_eos) {
      GST_DEBUG_OBJECT (aggpad, "Pad is in EOS state");
      GST_OBJECT_UNLOCK (pad);
      continue;
    }

    g_assert (pad->priv->buffer);

1832
    /* This pad is lagging behind, we need to update the offset
1833
1834
1835
     * and maybe drop the current buffer */
    if (pad->priv->output_offset < aagg->priv->offset) {
      gint64 diff = aagg->priv->offset - pad->priv->output_offset;
1836
      gint64 odiff = diff;
1837
1838
1839
1840
1841
1842
1843

      if (pad->priv->position + diff > pad->priv->size)
        diff = pad->priv->size - pad->priv->position;
      pad->priv->position += diff;
      pad->priv->output_offset += diff;

      if (pad->priv->position == pad->priv->size) {
1844
        GST_DEBUG_OBJECT (pad, "Buffer was late by %" GST_TIME_FORMAT
1845
1846
1847
            ", dropping %" GST_PTR_FORMAT,
            GST_TIME_ARGS (gst_util_uint64_scale (odiff, GST_SECOND,
                    GST_AUDIO_INFO_RATE (&aagg->info))), pad->priv->buffer);
1848
1849
        /* Buffer done, drop it */
        gst_buffer_replace (&pad->priv->buffer, NULL);
1850
        gst_buffer_replace (&pad->priv->input_buffer, NULL);
1851
1852
1853
1854
1855
1856
1857
        dropped = TRUE;
        GST_OBJECT_UNLOCK (pad);
        gst_aggregator_pad_drop_buffer (aggpad);
        continue;
      }
    }

1858
    g_assert (pad->priv->buffer);
1859
1860

    if (pad->priv->output_offset >= aagg->priv->offset
1861
1862
        && pad->priv->output_offset < aagg->priv->offset + blocksize) {
      gboolean drop_buf;
1863

1864
1865
      GST_LOG_OBJECT (aggpad, "Mixing buffer for current offset");
      drop_buf = !gst_audio_aggregator_mix_buffer (aagg, pad, pad->priv->buffer,
1866
          outbuf, blocksize);
1867
      if (pad->priv->output_offset >= next_offset) {
1868
1869
        GST_LOG_OBJECT (pad,
            "Pad is at or after current offset: %" G_GUINT64_FORMAT " >= %"
Vineeth TM's avatar
Vineeth TM committed
1870
            G_GINT64_FORMAT, pad->priv->output_offset, next_offset);
1871
1872
1873
      } else {
        is_done = FALSE;
      }
1874
1875
1876
1877
1878
      if (drop_buf) {
        GST_OBJECT_UNLOCK (pad);
        gst_aggregator_pad_drop_buffer (aggpad);
        continue;
      }
1879
1880
1881
1882
1883
1884
1885
1886
    }

    GST_OBJECT_UNLOCK (pad);
  }
  GST_OBJECT_UNLOCK (agg);

  if (dropped) {
    /* We dropped a buffer, retry */