/* UltraStar Deluxe - Karaoke Game
*
* UltraStar Deluxe is the legal property of its developers, whose names
* are too numerous to list here. Please refer to the COPYRIGHT
* file distributed with this source distribution.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file COPYING. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
* $URL$
* $Id$
*/
/*******************************************************************************
*
* This code is primarily based upon -
* http://www.dranger.com/ffmpeg/ffmpegtutorial_all.html
*
* and tutorial03.c
*
* http://www.inb.uni-luebeck.de/~boehme/using_libavcodec.html
*
*******************************************************************************/
#include "ffmpeg_audio_decode.h"
#include <string>
#include <sstream>
#include <cmath>
// show FFmpeg specific debug output
//#define DEBUG_FFMPEG_DECODE
// FFmpeg is very verbose and shows a bunch of errors.
// Those errors (they can be considered as warnings by us) can be ignored
// as they do not give any useful information.
// There is no solution to fix this except for turning them off.
//#define ENABLE_FFMPEG_ERROR_OUTPUT
#define MAX_AUDIOQ_SIZE (5 * 16 * 1024)
/* FFmpegDecodeStream */
FFmpegAudioDecodeStream::FFmpegAudioDecodeStream() :
_eofState(false),
_errorState(false),
_quitRequest(false),
_parserLocked(false),
_parserPauseRequestCount(0),
_seekRequest(false),
_seekFlags(0),
_seekPos(0),
_seekFlush(false),
_loop(false),
_formatCtx(NULL),
_codecCtx(NULL),
_codec(NULL),
_audioStreamIndex(-1),
_audioStream(NULL),
_audioStreamPos(0.0),
_decoderLocked(false),
_decoderPauseRequestCount(0),
_audioPaketSilence(0),
_audioBufferPos(0),
_audioBufferSize(0),
_filename("")
{
memset(&_audioPaket, 0, sizeof(_audioPaket));
memset(&_audioPaketTemp, 0, sizeof(_audioPaketTemp));
}
FFmpegAudioDecodeStream* FFmpegAudioDecodeStream::open(const IPath &filename) {
FFmpegAudioDecodeStream *stream = new FFmpegAudioDecodeStream();
if (!stream->_open(filename)) {
delete stream;
return 0;
}
return stream;
}
bool FFmpegAudioDecodeStream::_open(const IPath &filename) {
_filename = filename;
if (!filename.isFile()) {
logger.error("Audio-file does not exist: '" + filename.toNative() + "'", "UAudio_FFmpeg");
return false;
}
// use custom 'ufile' protocol for UTF-8 support
if (av_open_input_file(&_formatCtx,
("ufile:" + filename.toUTF8()).c_str(), NULL, 0, NULL) != 0)
{
logger.error("av_open_input_file failed: '" + filename.toNative() + "'", "UAudio_FFmpeg");
return false;
}
// generate PTS values if they do not exist
_formatCtx->flags |= AVFMT_FLAG_GENPTS;
// retrieve stream information
if (av_find_stream_info(_formatCtx) < 0) {
logger.error("av_find_stream_info failed: '" + filename.toNative() + "'", "UAudio_FFmpeg");
return false;
}
// FIXME: hack used by ffplay. Maybe should not use url_feof() to test for the end
_formatCtx->pb->eof_reached = 0;
#ifdef DEBUG_FFMPEG_DECODE
dump_format(_formatCtx, 0, filename.toNative(), 0);
#endif
_audioStreamIndex = ffmpegCore->findAudioStreamIndex(_formatCtx);
if (_audioStreamIndex < 0) {
logger.error("FindAudioStreamIndex: No Audio-stream found '" + filename.toNative() + "'", "UAudio_FFmpeg");
return false;
}
//std::stringstream s;
//s << _audioStreamIndex;
//logger.status("AudioStreamIndex is: " + s.str(), "UAudio_FFmpeg");
_audioStream = _formatCtx->streams[_audioStreamIndex];
_audioStreamPos = 0;
_codecCtx = _audioStream->codec;
#ifdef REQUEST_CHANNELS
// TODO: should we use this or not? Should we allow 5.1 channel audio?
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(51,42,0)
if (_codecCtx->channels > 0)
_codecCtx->request_channels = std::min(2, _codecCtx->channels);
else
_codecCtx->request_channels = 2;
#endif
#endif
_codec = avcodec_find_decoder(_codecCtx->codec_id);
if (!_codec) {
logger.error("Unsupported codec!", "UAudio_FFmpeg");
_codecCtx = NULL;
return false;
}
// set debug options
_codecCtx->debug_mv = 0;
_codecCtx->debug = 0;
// detect bug-workarounds automatically
_codecCtx->workaround_bugs = FF_BUG_AUTODETECT;
// error resilience strategy (careful/compliant/agressive/very_aggressive)
//CodecCtx->error_resilience = FF_ER_CAREFUL; //FF_ER_COMPLIANT;
// allow non spec compliant speedup tricks.
//CodecCtx->flags2 |= CODEC_FLAG2_FAST;
// Note: avcodec_open() and avcodec_close() are not thread-safe and will
// fail if called concurrently by different threads.
int avResult;
{
MediaCore_FFmpeg::AVCodecLock codecLock;
avResult = avcodec_open(_codecCtx, _codec);
}
if (avResult < 0) {
logger.error("avcodec_open failed!", "UAudio_FFmpeg");
return false;
}
// now initialize the audio-format
audioSampleFormat_t sampleFormat;
if (!ffmpegCore->convertFFmpegToAudioFormat(_codecCtx->sample_fmt, &sampleFormat)) {
// try standard format
sampleFormat = asfS16;
}
if (_codecCtx->channels > 255) {
logger.status("Error: _codecCtx->channels > 255", "TFFmpegDecodeStream.Open");
}
_formatInfo = AudioFormatInfo(
_codecCtx->channels,
_codecCtx->sample_rate,
sampleFormat
);
// finally start the decode thread
start();
return true;
}
void FFmpegAudioDecodeStream::close() {
// wake threads waiting for packet-queue data
// Note: normally, there are no waiting threads. If there were waiting
// ones, they would block the audio-callback thread.
_packetQueue.abort();
// send quit request (to parse-thread etc)
{
Mutex::RegionLock lock(_stateLock);
_quitRequest = true;
_parserIdleCond.broadcast();
}
// abort parse-thread
// and wait until it terminates
wait();
// Close the codec
if (_codecCtx) {
// avcodec_close() is not thread-safe
MediaCore_FFmpeg::AVCodecLock codecLock;
avcodec_close(_codecCtx);
}
// Close the video file
if (_formatCtx) {
av_close_input_file(_formatCtx);
}
}
double FFmpegAudioDecodeStream::getLength() {
// do not forget to consider the start_time value here
// there is a type size mismatch warnign because start_time and duration are cint64.
// So, in principle there could be an overflow when doing the sum.
return (double)(_formatCtx->start_time + _formatCtx->duration) / AV_TIME_BASE;
}
double FFmpegAudioDecodeStream::getPosition() {
DecoderPauser decoderPauser(this);
// ReadData() does not return all of the buffer retrieved by DecodeFrame().
// Determine the size of the unused part of the decode-buffer.
double bufferSizeSec = (double)(_audioBufferSize - _audioBufferPos) /
_formatInfo.getBytesPerSec();
// subtract the size of unused buffer-data from the audio clock.
return _audioStreamPos - bufferSizeSec;
}
void FFmpegAudioDecodeStream::setPosition(double time) {
setPositionIntern(time, true, true);
}
/********************************************
* Parser section
********************************************/
void FFmpegAudioDecodeStream::setPositionIntern(double time, bool flush, bool blocking) {
// - Pause the parser first to prevent it from putting obsolete packages
// into the queue after the queue was flushed and before seeking is done.
// Otherwise we will hear fragments of old data, if the stream was seeked
// in stopped mode and resumed afterwards (applies to non-blocking mode only).
// - Pause the decoder to avoid race-condition that might occur otherwise.
// - Last lock the state lock because we are manipulating some shared state-vars.
{
ParserPauser parserPauser(this);
DecoderPauser decoderPauser(this);
Mutex::RegionLock lock(_stateLock);
_eofState = false;
_errorState = false;
// do not seek if we are already at the correct position.
// This is important especially for seeking to position 0 if we already are
// at the beginning. Although seeking with AVSEEK_FLAG_BACKWARD for pos 0 works,
// it is still a bit choppy (although much better than w/o AVSEEK_FLAG_BACKWARD).
if (time == _audioStreamPos)
return;
// configure seek parameters
_seekPos = time;
_seekFlush = flush;
_seekFlags = AVSEEK_FLAG_ANY;
_seekRequest = true;
// Note: the BACKWARD-flag seeks to the first position <= the position
// searched for. Otherwise e.g. position 0 might not be seeked correct.
// For some reason ffmpeg sometimes doesn't use position 0 but the key-frame
// following. In streams with few key-frames (like many flv-files) the next
// key-frame after 0 might be 5secs ahead.
if (time <= _audioStreamPos)
_seekFlags |= AVSEEK_FLAG_BACKWARD;
// send a reuse signal in case the parser was stopped (e.g. because of an EOF)
_parserIdleCond.signal();
}
// in blocking mode, wait until seeking is done
if (blocking) {
Mutex::RegionLock lock(_stateLock);
while (_seekRequest)
_seekFinishedCond.wait(_stateLock);
}
}
int FFmpegAudioDecodeStream::run() {
// reuse thread as long as the stream is not terminated
while (parseLoop()) {
// wait for reuse or destruction of stream
Mutex::RegionLock lock(_stateLock);
while (!(_seekRequest || _quitRequest)) {
_parserIdleCond.wait(_stateLock);
}
}
return 0;
}
/**
* Parser main loop.
* Will not return until parsing of the stream is finished.
* Reasons for the parser to return are:
* - the end-of-file is reached
* - an error occured
* - the stream was quited (received a quit-request)
* Returns true if the stream can be resumed or false if the stream has to
* be terminated.
*/
bool FFmpegAudioDecodeStream::parseLoop() {
AVPacket packet;
while (true) {
ParserLock parserLock(this);
if (isQuit()) {
return false;
}
// handle seek-request (Note: no need to lock SeekRequest here)
if (_seekRequest) {
// first try: seek on the audio stream
int64_t seekTarget = llround(_seekPos / av_q2d(_audioStream->time_base));
// duration of silence at start of stream
double startSilence = 0;
if (seekTarget < _audioStream->start_time)
startSilence = (double)(_audioStream->start_time - seekTarget) * av_q2d(_audioStream->time_base);
int errorCode = av_seek_frame(_formatCtx, _audioStreamIndex, seekTarget, _seekFlags);
if (errorCode < 0) {
// second try: seek on the default stream (necessary for flv-videos and some ogg-files)
seekTarget = llround(_seekPos * AV_TIME_BASE);
startSilence = 0;
if (seekTarget < _formatCtx->start_time)
startSilence = (double)(_formatCtx->start_time - seekTarget) / AV_TIME_BASE;
errorCode = av_seek_frame(_formatCtx, -1, seekTarget, _seekFlags);
}
// pause decoder and lock state (keep the lock-order to avoid deadlocks).
// Note that the decoder does not block in the packet-queue in seeking state,
// so locking the decoder here does not cause a dead-lock.
{
DecoderPauser pause(this);
Mutex::RegionLock lock(_stateLock);
if (errorCode < 0) {
// seeking failed
_errorState = true;
std::stringstream s;
s << "Seek Error in '" << _formatCtx->filename << "'";
logger.error(s.str(), "UAudioDecoder_FFmpeg");
} else {
if (_seekFlush) {
// flush queue (we will send a Flush-Packet when seeking is finished)
_packetQueue.flush();
// flush the decode buffers
_audioBufferSize = 0;
_audioBufferPos = 0;
_audioPaketSilence = 0;
memset(&_audioPaketTemp, 0, sizeof(_audioPaketTemp));
flushCodecBuffers();
// Set preliminary stream position. The position will be set to
// the correct value as soon as the first packet is decoded.
_audioStreamPos = _seekPos;
} else {
// request avcodec buffer flush
_packetQueue.putStatus(PKT_STATUS_FLAG_FLUSH, NULL);
}
// fill the gap between position 0 and start_time with silence
// but not if we are in loop mode
if (startSilence > 0 && !_loop) {
// pointer for the EMPTY status packet
double *startSilencePtr = (double*)malloc(sizeof(startSilence));
*startSilencePtr = startSilence;
_packetQueue.putStatus(PKT_STATUS_FLAG_EMPTY, startSilencePtr);
}
}
_seekRequest = false;
_seekFinishedCond.broadcast();
}
}
if (_packetQueue.getSize() > MAX_AUDIOQ_SIZE) {
Thread::sleep(10);
continue;
}
if (av_read_frame(_formatCtx, &packet) < 0) {
// failed to read a frame, check reason
ByteIOContext *byteIOCtx;
#if LIBAVFORMAT_VERSION_MAJOR >= 52
byteIOCtx = _formatCtx->pb;
#else
byteIOCtx = &_formatCtx->pb;
#endif
// check for end-of-file (eof is not an error)
if (url_feof(byteIOCtx) != 0) {
if (getLoop()) {
// rewind stream (but do not flush)
setPositionIntern(0, false, false);
continue;
} else {
// signal end-of-file
_packetQueue.putStatus(PKT_STATUS_FLAG_EOF, NULL);
return true;
}
}
// check for errors
if (url_ferror(byteIOCtx) != 0) {
// an error occured -> abort and wait for repositioning or termination
_packetQueue.putStatus(PKT_STATUS_FLAG_ERROR, NULL);
return true;
}
// url_feof() does not detect an EOF for some files
// so we have to do it this way.
if ((_formatCtx->file_size != 0) && (byteIOCtx->pos >= _formatCtx->file_size)) {
_packetQueue.putStatus(PKT_STATUS_FLAG_EOF, NULL);
return true;
}
// unknown error occured, exit
_packetQueue.putStatus(PKT_STATUS_FLAG_ERROR, NULL);
return true;
}
if (packet.stream_index == _audioStreamIndex) {
_packetQueue.put(&packet);
} else {
av_free_packet(&packet);
}
}
}
/********************************************
* Decoder section
********************************************/
void FFmpegAudioDecodeStream::flushCodecBuffers() {
// if no flush operation is specified, avcodec_flush_buffers will not do anything.
if (_codecCtx->codec->flush) {
// flush buffers used by avcodec_decode_audio, etc.
avcodec_flush_buffers(_codecCtx);
} else {
// we need a Workaround to avoid plopping noise with ogg-vorbis and
// mp3 (in older versions of FFmpeg).
// We will just reopen the codec.
MediaCore_FFmpeg::AVCodecLock codecLock;
avcodec_close(_codecCtx);
avcodec_open(_codecCtx, _codec);
}
}
int FFmpegAudioDecodeStream::decodeFrame(uint8_t *buffer, int bufferSize) {
if (isEOF())
return -1;
while (true) {
// for titles with start_time > 0 we have to generate silence
// until we reach the pts of the first data packet.
if (_audioPaketSilence > 0) {
int dataSize = std::min(_audioPaketSilence, bufferSize);
memset(buffer, 0, dataSize);
_audioPaketSilence -= dataSize;
_audioStreamPos += dataSize / _formatInfo.getBytesPerSec();
return dataSize;
}
// read packet data
while (_audioPaketTemp.size > 0) {
// size of output data decoded by FFmpeg
int dataSize = bufferSize;
int paketDecodedSize; // size of packet data used for decoding
paketDecodedSize = avcodec_decode_audio3(_codecCtx, (int16_t*)buffer,
&dataSize, &_audioPaketTemp);
if (paketDecodedSize < 0) {
// if error, skip frame
#ifdef DEBUG_FFMPEG_DECODE
logger.status("Skip audio frame", "");
#endif
_audioPaketTemp.size = 0;
break;
}
_audioPaketTemp.data += paketDecodedSize;
_audioPaketTemp.size -= paketDecodedSize;
// check if avcodec_decode_audio returned data, otherwise fetch more frames
if (dataSize <= 0)
continue;
// update stream position by the amount of fetched data
_audioStreamPos += dataSize / _formatInfo.getBytesPerSec();
// we have data, return it and come back for more later
return dataSize;
}
// free old packet data
if (_audioPaket.data)
av_free_packet(&_audioPaket);
// do not block queue on seeking (to avoid deadlocks on the DecoderLock)
bool blockQueue = !isSeeking();
// request a new packet and block if none available.
// If this fails, the queue was aborted.
if (_packetQueue.get(&_audioPaket, blockQueue) <= 0)
return -1;
// handle Status-packet
if (_audioPaket.data == STATUS_PACKET) {
_audioPaket.data = NULL;
memset(&_audioPaketTemp, 0, sizeof(_audioPaketTemp));
switch (_audioPaket.flags) {
case PKT_STATUS_FLAG_FLUSH:
// just used if SetPositionIntern was called without the flush flag.
flushCodecBuffers();
break;
case PKT_STATUS_FLAG_EOF: // end-of-file
// ignore EOF while seeking
if (!isSeeking())
setEOF(true);
// buffer contains no data
return -1;
case PKT_STATUS_FLAG_ERROR:
setError(true);
logger.status("I/O Error", "TFFmpegDecodeStream.DecodeFrame");
return -1;
case PKT_STATUS_FLAG_EMPTY: {
double silenceDuration = *(double*) (_packetQueue.getStatusInfo(&_audioPaket));
_audioPaketSilence = lround(silenceDuration * _formatInfo.getSampleRate()) *
_formatInfo.getFrameSize();
_packetQueue.freeStatusInfo(&_audioPaket);
break;
}
default:
logger.status("Unknown status", "TFFmpegDecodeStream.DecodeFrame");
}
continue;
}
_audioPaketTemp.data = _audioPaket.data;
_audioPaketTemp.size = _audioPaket.size;
// if available, update the stream position to the presentation time of this package
if (_audioPaket.pts != (int64_t)AV_NOPTS_VALUE) {
#ifdef DEBUG_FFMPEG_DECODE
double tmpPos = _audioStreamPos;
#endif
_audioStreamPos = av_q2d(_audioStream->time_base) * _audioPaket.pts;
#ifdef DEBUG_FFMPEG_DECODE
stringstream s;
s << "Timestamp: " << _audioStreamPos << " "
<< "(Calc: " << tmpPos << "), "
<< "Diff: " << (_audioStreamPos-TmpPos);
logger.status(s.str(), "");
#endif
}
}
}
int FFmpegAudioDecodeStream::readData(uint8_t *buffer, int bufferSize) {
// set number of bytes to copy to the output buffer
int bufferPos = 0;
{
DecoderLock decoderLock(this);
// leave if end-of-file is reached
if (isEOF()) {
return -1;
}
// copy data to output buffer
while (bufferPos < bufferSize) {
// check if we need more data
if (_audioBufferPos >= _audioBufferSize) {
_audioBufferPos = 0;
// we have already sent all our data; get more
_audioBufferSize = decodeFrame(_audioBuffer, AUDIO_BUFFER_SIZE);
if (_audioBufferSize < 0) {
// error or EOF occurred
return bufferPos;
}
}
// calc number of new bytes in the decode-buffer (number of bytes to copy)
int copyByteCount = _audioBufferSize - _audioBufferPos;
// number of bytes left (remain) to read
int remainByteCount = bufferSize - bufferPos;
// resize copy-count if more bytes available than needed (remaining bytes are used the next time)
if (copyByteCount > remainByteCount)
copyByteCount = remainByteCount;
memcpy(&buffer[bufferPos], &_audioBuffer[_audioBufferPos], copyByteCount);
bufferPos += copyByteCount;
_audioBufferPos += copyByteCount;
}
}
return bufferSize;
}
/************************************
* C Interface
************************************/
#define DecodeStreamObj(ptr) reinterpret_cast<PluginDecodeStream*>(ptr)
static BOOL PLUGIN_CALL ffmpegAudioDecoder_init() {
return TRUE;
}
static BOOL PLUGIN_CALL ffmpegAudioDecoder_finalize() {
return TRUE;
}
static audioDecodeStream_t* PLUGIN_CALL ffmpegAudioDecoder_open(const char *filename) {
return (audioDecodeStream_t*)FFmpegAudioDecodeStream::open(filename);
}
static void PLUGIN_CALL ffmpegAudioDecoder_close(audioDecodeStream_t *stream) {
delete DecodeStreamObj(stream);
}
static double PLUGIN_CALL ffmpegAudioDecoder_getLength(audioDecodeStream_t *stream) {
return DecodeStreamObj(stream)->getLength();
}
static void PLUGIN_CALL ffmpegAudioDecoder_getAudioFormatInfo(audioDecodeStream_t *stream, audioFormatInfo_t *info) {
DecodeStreamObj(stream)->getAudioFormatInfo().toCStruct(info);
}
static double PLUGIN_CALL ffmpegAudioDecoder_getPosition(audioDecodeStream_t *stream) {
return DecodeStreamObj(stream)->getPosition();
}
static void PLUGIN_CALL ffmpegAudioDecoder_setPosition(audioDecodeStream_t *stream, double time) {
DecodeStreamObj(stream)->setPosition(time);
}
static BOOL PLUGIN_CALL ffmpegAudioDecoder_getLoop(audioDecodeStream_t *stream) {
return (BOOL)DecodeStreamObj(stream)->getLoop();
}
static void PLUGIN_CALL ffmpegAudioDecoder_setLoop(audioDecodeStream_t *stream, BOOL enabled) {
DecodeStreamObj(stream)->setLoop(enabled);
}
static BOOL PLUGIN_CALL ffmpegAudioDecoder_isEOF(audioDecodeStream_t *stream) {
return (BOOL)DecodeStreamObj(stream)->isEOF();
}
static BOOL PLUGIN_CALL ffmpegAudioDecoder_isError(audioDecodeStream_t *stream) {
return (BOOL)DecodeStreamObj(stream)->isError();
}
static int PLUGIN_CALL ffmpegAudioDecoder_readData(audioDecodeStream_t *stream, uint8_t *buffer, int bufferSize) {
return DecodeStreamObj(stream)->readData(buffer, bufferSize);
}
/************************************
* Module information
************************************/
const audioDecoderInfo_t audioDecoderInfo = {
50,
ffmpegAudioDecoder_init,
ffmpegAudioDecoder_finalize,
ffmpegAudioDecoder_open,
ffmpegAudioDecoder_close,
ffmpegAudioDecoder_getLength,
ffmpegAudioDecoder_getAudioFormatInfo,
ffmpegAudioDecoder_getPosition,
ffmpegAudioDecoder_setPosition,
ffmpegAudioDecoder_getLoop,
ffmpegAudioDecoder_setLoop,
ffmpegAudioDecoder_isEOF,
ffmpegAudioDecoder_isError,
ffmpegAudioDecoder_readData
};
