(Committing on behalf of Chris Rogers -- original CL http://codereview.chromium.org/5550006/ )

Implement WebKitClientImpl::loadAudioResource() to decode in-memory audio file data for use by WebKit.
Most of the interesting low-level code is being added in the media directory.

BUG=NONE
TEST=NONE (tested locally with web audio API loading files of format .wav .aif .mp3 .m4a 16bit 24bit
In the longer term, WebKit layout tests will comprehensively exercise this code)

Review URL: http://codereview.chromium.org/5880002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@69458 0039d316-1c4b-4281-b951-d872f2087c98

media/audio/audio_util.cc

@@ -158,4 +158,50 @@ bool FoldChannels(void* buf,
   return false;
 }
 
+bool DeinterleaveAudioChannel(void* source,
+                              float* destination,
+                              int channels,
+                              int channel_index,
+                              int bytes_per_sample,
+                              size_t number_of_frames) {
+  switch (bytes_per_sample) {
+    case 1:
+    {
+      uint8* source8 = static_cast<uint8*>(source) + channel_index;
+      const float kScale = 1.0f / 128.0f;
+      for (unsigned i = 0; i < number_of_frames; ++i) {
+        destination[i] = kScale * static_cast<int>(*source8 + 128);
+        source8 += channels;
+      }
+      return true;
+    }
+
+    case 2:
+    {
+      int16* source16 = static_cast<int16*>(source) + channel_index;
+      const float kScale = 1.0f / 32768.0f;
+      for (unsigned i = 0; i < number_of_frames; ++i) {
+        destination[i] = kScale * *source16;
+        source16 += channels;
+      }
+      return true;
+    }
+
+    case 4:
+    {
+      int32* source32 = static_cast<int32*>(source) + channel_index;
+      const float kScale = 1.0f / (1L << 31);
+      for (unsigned i = 0; i < number_of_frames; ++i) {
+        destination[i] = kScale * *source32;
+        source32 += channels;
+      }
+      return true;
+    }
+
+    default:
+     break;
+  }
+  return false;
+}
+
 }  // namespace media

media/audio/audio_util.h

-// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -49,6 +49,19 @@ bool FoldChannels(void* buf,
                   int bytes_per_sample,
                   float volume);
 
+// DeinterleaveAudioChannel() takes interleaved audio buffer |source|
+// of the given |sample_fmt| and |number_of_channels| and extracts
+// |number_of_frames| data for the given |channel_index| and
+// puts it in the floating point |destination|.
+// It returns |true| on success, or |false| if the |sample_fmt| is
+// not recognized.
+bool DeinterleaveAudioChannel(void* source,
+                              float* destination,
+                              int channels,
+                              int channel_index,
+                              int bytes_per_sample,
+                              size_t number_of_frames);
+
 }  // namespace media
 
 #endif  // MEDIA_AUDIO_AUDIO_UTIL_H_

media/filters/audio_file_reader.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/filters/audio_file_reader.h"
+
+#include <string>
+#include "base/basictypes.h"
+#include "base/string_util.h"
+#include "base/time.h"
+#include "media/audio/audio_util.h"
+#include "media/base/filters.h"
+#include "media/ffmpeg/ffmpeg_common.h"
+#include "media/ffmpeg/ffmpeg_util.h"
+#include "media/filters/ffmpeg_glue.h"
+
+namespace media {
+
+AudioFileReader::AudioFileReader(FFmpegURLProtocol* protocol)
+    : protocol_(protocol),
+      format_context_(NULL),
+      codec_context_(NULL),
+      codec_(NULL) {
+}
+
+AudioFileReader::~AudioFileReader() {
+  Close();
+}
+
+int AudioFileReader::channels() const {
+  return codec_context_->channels;
+}
+
+int AudioFileReader::sample_rate() const {
+  return codec_context_->sample_rate;
+}
+
+base::TimeDelta AudioFileReader::duration() const {
+  const AVRational av_time_base = {1, AV_TIME_BASE};
+  return ConvertTimestamp(av_time_base, format_context_->duration);
+}
+
+int64 AudioFileReader::number_of_frames() const {
+  return static_cast<int64>(duration().InSecondsF() * sample_rate());
+}
+
+bool AudioFileReader::Open() {
+  // Add our data reader to the protocol list and get our unique key.
+  std::string key = FFmpegGlue::GetInstance()->AddProtocol(protocol_);
+
+  // Open FFmpeg AVFormatContext.
+  DCHECK(!format_context_);
+  AVFormatContext* context = NULL;
+
+  int result = av_open_input_file(&context, key.c_str(), NULL, 0, NULL);
+
+  // Remove our data reader from protocol list since av_open_input_file() setup
+  // the AVFormatContext with the data reader.
+  FFmpegGlue::GetInstance()->RemoveProtocol(protocol_);
+
+  if (result) {
+    DLOG(WARNING)
+        << "AudioFileReader::Open() : error in av_open_input_file() -"
+        << " result: " << result;
+    return false;
+  }
+
+  DCHECK(context);
+  format_context_ = context;
+
+  // Get the codec context.
+  codec_context_ = NULL;
+  for (size_t i = 0; i < format_context_->nb_streams; ++i) {
+    AVCodecContext* c = format_context_->streams[i]->codec;
+    if (c->codec_type == CODEC_TYPE_AUDIO) {
+      codec_context_ = c;
+      break;
+    }
+  }
+
+  // Get the codec.
+  if (!codec_context_)
+    return false;
+
+  av_find_stream_info(format_context_);
+  codec_ = avcodec_find_decoder(codec_context_->codec_id);
+  if (codec_) {
+    if ((result = avcodec_open(codec_context_, codec_)) < 0) {
+      DLOG(WARNING) << "AudioFileReader::Open() : could not open codec -"
+          << " result: " << result;
+      return false;
+    }
+
+    result = av_seek_frame(format_context_, 0, 0, 0);
+  }
+
+  return true;
+}
+
+void AudioFileReader::Close() {
+  if (codec_context_ && codec_)
+    avcodec_close(codec_context_);
+
+  codec_context_ = NULL;
+  codec_ = NULL;
+
+  if (format_context_) {
+    av_close_input_file(format_context_);
+    format_context_ = NULL;
+  }
+}
+
+bool AudioFileReader::Read(const std::vector<float*>& audio_data,
+                           size_t number_of_frames) {
+  size_t channels = this->channels();
+  DCHECK_EQ(audio_data.size(), channels);
+  if (audio_data.size() != channels)
+    return false;
+
+  DCHECK(format_context_ && codec_context_);
+  if (!format_context_ || !codec_context_) {
+    DLOG(WARNING) << "AudioFileReader::Read() : reader is not opened!";
+    return false;
+  }
+
+  scoped_ptr_malloc<int16, ScopedPtrAVFree> output_buffer(
+      static_cast<int16*>(av_malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE)));
+
+  // Read until we hit EOF or we've read the requested number of frames.
+  AVPacket avpkt;
+  av_init_packet(&avpkt);
+
+  int result = 0;
+  size_t current_frame = 0;
+
+  while (current_frame < number_of_frames &&
+      (result = av_read_frame(format_context_, &avpkt)) >= 0) {
+    int out_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
+    result = avcodec_decode_audio3(codec_context_,
+                                   output_buffer.get(),
+                                   &out_size,
+                                   &avpkt);
+
+    if (result < 0) {
+      DLOG(WARNING)
+          << "AudioFileReader::Read() : error in avcodec_decode_audio3() -"
+          << result;
+      return false;
+    }
+
+    // Determine the number of sample-frames we just decoded.
+    size_t bytes_per_sample =
+        av_get_bits_per_sample_fmt(codec_context_->sample_fmt) >> 3;
+    size_t frames_read = out_size / (channels * bytes_per_sample);
+
+    // Truncate, if necessary, if the destination isn't big enough.
+    if (current_frame + frames_read > number_of_frames)
+      frames_read = number_of_frames - current_frame;
+
+    // Deinterleave each channel and convert to 32bit floating-point
+    // with nominal range -1.0 -> +1.0.
+    for (size_t channel_index = 0; channel_index < channels;
+         ++channel_index) {
+      if (!DeinterleaveAudioChannel(output_buffer.get(),
+                                    audio_data[channel_index] + current_frame,
+                                    channels,
+                                    channel_index,
+                                    bytes_per_sample,
+                                    frames_read)) {
+        DLOG(WARNING)
+            << "AudioFileReader::Read() : Unsupported sample format : "
+            << codec_context_->sample_fmt
+            << " codec_->id : " << codec_->id;
+        return false;
+      }
+    }
+
+    current_frame += frames_read;
+  }
+
+  return true;
+}
+
+InMemoryDataReader::InMemoryDataReader(const char* data, int64 size)
+    : data_(data),
+      size_(size),
+      position_(0) {
+}
+
+int InMemoryDataReader::Read(int size, uint8* data) {
+  if (size < 0)
+    return -1;
+
+  int available_bytes = static_cast<int>(size_ - position_);
+  if (size > available_bytes)
+    size = available_bytes;
+
+  memcpy(data, data_ + position_, size);
+  position_ += size;
+  return size;
+}
+
+bool InMemoryDataReader::GetPosition(int64* position_out) {
+  if (position_out)
+    *position_out = position_;
+  return true;
+}
+
+bool InMemoryDataReader::SetPosition(int64 position) {
+  if (position >= size_)
+    return false;
+  position_ = position;
+  return true;
+}
+
+bool InMemoryDataReader::GetSize(int64* size_out) {
+  if (size_out)
+    *size_out = size_;
+  return true;
+}
+
+bool InMemoryDataReader::IsStreaming() {
+  return false;
+}
+
+}  // namespace media

media/filters/audio_file_reader.h

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_FILTERS_AUDIO_FILE_READER_H_
+#define MEDIA_FILTERS_AUDIO_FILE_READER_H_
+
+#include <vector>
+#include "media/filters/ffmpeg_glue.h"
+
+struct AVCodec;
+struct AVCodecContext;
+struct AVFormatContext;
+
+namespace base { class TimeDelta; }
+
+namespace media {
+
+class FFmpegURLProtocol;
+
+class AudioFileReader {
+ public:
+  // Audio file data will be read using the given protocol.
+  // The AudioFileReader does not take ownership of |protocol| and
+  // simply maintains a weak reference to it.
+  explicit AudioFileReader(FFmpegURLProtocol* protocol);
+  virtual ~AudioFileReader();
+
+  // Open() reads the audio data format so that the sample_rate(),
+  // channels(), duration(), and number_of_frames() methods can be called.
+  // It returns |true| on success.
+  bool Open();
+  void Close();
+
+  // After a call to Open(), reads |number_of_frames| into |audio_data|.
+  // |audio_data| must be of the same size as channels().
+  // The audio data will be decoded as floating-point linear PCM with
+  // a nominal range of -1.0 -> +1.0.
+  // Returns |true| on success.
+  bool Read(const std::vector<float*>& audio_data, size_t number_of_frames);
+
+  // These methods can be called once Open() has been called.
+  int channels() const;
+  int sample_rate() const;
+  base::TimeDelta duration() const;
+  int64 number_of_frames() const;
+
+ private:
+  FFmpegURLProtocol* protocol_;
+  AVFormatContext* format_context_;
+  AVCodecContext* codec_context_;
+  AVCodec* codec_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioFileReader);
+};
+
+class InMemoryDataReader : public FFmpegURLProtocol {
+ public:
+  // Ownership of |data| is not taken, instead it simply maintains
+  // a weak reference.
+  InMemoryDataReader(const char* data, int64 size);
+
+  virtual int Read(int size, uint8* data);
+  virtual bool GetPosition(int64* position_out);
+  virtual bool SetPosition(int64 position);
+  virtual bool GetSize(int64* size_out);
+  virtual bool IsStreaming();
+
+ private:
+  const char* data_;
+  int64 size_;
+  int64 position_;
+
+  DISALLOW_COPY_AND_ASSIGN(InMemoryDataReader);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_FILTERS_AUDIO_FILE_READER_H_

media/media.gyp

@@ -115,6 +115,8 @@
         'ffmpeg/ffmpeg_util.h',
         'ffmpeg/file_protocol.cc',
         'ffmpeg/file_protocol.h',
+        'filters/audio_file_reader.cc',
+        'filters/audio_file_reader.h',
         'filters/audio_renderer_algorithm_base.cc',
         'filters/audio_renderer_algorithm_base.h',
         'filters/audio_renderer_algorithm_default.cc',

webkit/glue/media/audio_decoder.cc

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "webkit/glue/media/audio_decoder.h"
+
+#include <vector>
+#include "base/basictypes.h"
+#include "base/string_util.h"
+#include "base/time.h"
+#include "media/filters/audio_file_reader.h"
+#include "third_party/WebKit/WebKit/chromium/public/WebAudioBus.h"
+
+using media::AudioFileReader;
+using media::InMemoryDataReader;
+using std::vector;
+using WebKit::WebAudioBus;
+
+namespace webkit_glue {
+
+// Decode in-memory audio file data.
+bool DecodeAudioFileData(
+    WebKit::WebAudioBus* destination_bus,
+    const char* data, size_t data_size, double sample_rate) {
+  DCHECK(destination_bus);
+  if (!destination_bus)
+    return false;
+
+  // Uses the FFmpeg library for audio file reading.
+  InMemoryDataReader data_reader(data, data_size);
+  AudioFileReader reader(&data_reader);
+
+  if (!reader.Open())
+    return false;
+
+  size_t number_of_channels = reader.channels();
+  double file_sample_rate = reader.sample_rate();
+  double duration = reader.duration().InSecondsF();
+  size_t number_of_frames = static_cast<size_t>(reader.number_of_frames());
+
+  // TODO(crogers) : do sample-rate conversion with FFmpeg.
+  // For now, we're ignoring the requested 'sample_rate' and returning
+  // the WebAudioBus at the file's sample-rate.
+  // double destination_sample_rate =
+  //   (sample_rate != 0.0) ? sample_rate : file_sample_rate;
+  double destination_sample_rate = file_sample_rate;
+
+  DLOG(INFO) << "Decoding file data -"
+      << " data: " << data
+      << " data size: " << data_size
+      << " duration: " << duration
+      << " number of frames: " << number_of_frames
+      << " sample rate: " << file_sample_rate
+      << " number of channels: " << number_of_channels;
+
+  // Change to destination sample-rate.
+  number_of_frames = static_cast<size_t>(number_of_frames *
+      (destination_sample_rate / file_sample_rate));
+
+  // Allocate and configure the output audio channel data.
+  destination_bus->initialize(number_of_channels,
+                              number_of_frames,
+                              destination_sample_rate);
+
+  // Wrap the channel pointers which will receive the decoded PCM audio.
+  vector<float*> audio_data;
+  audio_data.reserve(number_of_channels);
+  for (size_t i = 0; i < number_of_channels; ++i) {
+    audio_data.push_back(destination_bus->channelData(i));
+  }
+
+  // Decode the audio file data.
+  return reader.Read(audio_data, number_of_frames);
+}
+
+}  // namespace webkit_glue

webkit/glue/media/audio_decoder.h

+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef WEBKIT_GLUE_MEDIA_AUDIO_DECODER_H_
+#define WEBKIT_GLUE_MEDIA_AUDIO_DECODER_H_
+
+#include "base/basictypes.h"
+
+namespace WebKit { class WebAudioBus; }
+
+namespace webkit_glue {
+
+// Decode in-memory audio file data.
+bool DecodeAudioFileData(WebKit::WebAudioBus* destination_bus, const char* data,
+                         size_t data_size, double sample_rate);
+
+}  // namespace webkit_glue
+
+#endif  // WEBKIT_GLUE_MEDIA_AUDIO_DECODER_H_

webkit/glue/webkit_glue.gypi

@@ -250,6 +250,8 @@
         '../plugins/ppapi/var.h',
         '../plugins/ppapi/var_object_class.cc',
         '../plugins/ppapi/var_object_class.h',
+        'media/audio_decoder.cc',
+        'media/audio_decoder.h',
         'media/buffered_data_source.cc',
         'media/buffered_data_source.h',
         'media/buffered_resource_loader.cc',

webkit/glue/webkitclient_impl.cc

@@ -34,6 +34,7 @@
 #include "third_party/WebKit/WebKit/chromium/public/WebString.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebVector.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebURL.h"
+#include "webkit/glue/media/audio_decoder.h"
 #include "webkit/glue/plugins/plugin_instance.h"
 #include "webkit/glue/plugins/webplugininfo.h"
 #include "webkit/glue/webkit_glue.h"
@@ -44,6 +45,7 @@
 #include "v8/include/v8.h"
 #endif
 
+using WebKit::WebAudioBus;
 using WebKit::WebCookie;
 using WebKit::WebData;
 using WebKit::WebLocalizedString;
@@ -352,6 +354,15 @@ WebData WebKitClientImpl::loadResource(const char* name) {
   return WebData();
 }
 
+bool WebKitClientImpl::loadAudioResource(
+    WebKit::WebAudioBus* destination_bus, const char* audio_file_data,
+    size_t data_size, double sample_rate) {
+  return DecodeAudioFileData(destination_bus,
+                             audio_file_data,
+                             data_size,
+                             sample_rate);
+}
+
 WebString WebKitClientImpl::queryLocalizedString(
     WebLocalizedString::Name name) {
   int message_id = ToMessageID(name);

webkit/glue/webkitclient_impl.h

@@ -52,6 +52,9 @@ class WebKitClientImpl : public WebKit::WebKitClient {
   virtual void traceEventBegin(const char* name, void* id, const char* extra);
   virtual void traceEventEnd(const char* name, void* id, const char* extra);
   virtual WebKit::WebData loadResource(const char* name);
+  virtual bool loadAudioResource(
+      WebKit::WebAudioBus* destination_bus, const char* audio_file_data,
+      size_t data_size, double sample_rate);
   virtual WebKit::WebString queryLocalizedString(
       WebKit::WebLocalizedString::Name name);
   virtual WebKit::WebString queryLocalizedString(