path: root/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.cpp

/* UltraStar Deluxe - Karaoke Game
 *
 * UltraStar Deluxe is the legal property of its developers, whose names
 * are too numerous to list here. Please refer to the COPYRIGHT
 * file distributed with this source distribution.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; see the file COPYING. If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 *
 * $URL$
 * $Id$
 */

/*
 * based on 'An ffmpeg and SDL Tutorial' (http://www.dranger.com/ffmpeg/)
 */

#include "ffmpeg_video_decode.h"
#include <sstream>
#include <math.h>

// These are called whenever we allocate a frame buffer.
// We use this to store the global_pts in a frame at the time it is allocated.
int CDECL ptsGetBuffer(AVCodecContext *codecCtx, AVFrame *frame) {
	int result = avcodec_default_get_buffer(codecCtx, frame);
	int64_t *videoPktPts = (int64_t*)codecCtx->opaque;
	if (videoPktPts) {
		// Note: we must copy the pts instead of passing a pointer, because the packet
		// (and with it the pts) might change before a frame is returned by av_decode_video.
		int64_t *pts = (int64_t*)av_malloc(sizeof(int64_t));
		*pts = *videoPktPts;
		frame->opaque = pts;
	}
	return result;
}

void CDECL ptsReleaseBuffer(AVCodecContext *codecCtx, AVFrame *frame) {
	if (frame)
		av_freep(&frame->opaque);
	avcodec_default_release_buffer(codecCtx, frame);
}

/*
 * TVideoDecoder_FFmpeg
 */

FFmpegVideoDecodeStream::FFmpegVideoDecodeStream() :
		_opened(false),
		_eof(false),
		_loop(false),
		_stream(NULL),
		_streamIndex(-1),
		_formatContext(NULL),
		_codecContext(NULL),
		_codec(NULL),
		_avFrame(NULL),
		_avFrameRGB(NULL),
		_frameFormat(FRAME_FORMAT_UNKNOWN),
		_pixelFormat(PIX_FMT_NONE),
		_frameBuffer(NULL),
		_frameTexValid(false),
#ifdef USE_SWSCALE
		_swScaleContext(NULL),
#endif
		_aspect(0),
		_frameDuration(0),
		_frameTime(0),
		_loopTime(0) {}

FFmpegVideoDecodeStream* FFmpegVideoDecodeStream::open(const Path &filename,
	videoFrameFormat_t format) 
{
	FFmpegVideoDecodeStream *stream = new FFmpegVideoDecodeStream();
	if (!stream->_open(filename, format)) {
		delete stream;
		return 0;
	}
	return stream;
}

bool FFmpegVideoDecodeStream::_open(const Path &filename, videoFrameFormat_t format) {
	std::stringstream ss;

	// use custom 'ufile' protocol for UTF-8 support
	int errnum = av_open_input_file(&_formatContext,
			("ufile:" + filename.toUTF8()).c_str(), NULL, 0, NULL);
	if (errnum != 0) {
		logger.error("Failed to open file '" + filename.toNative() + "' ("
				+ ffmpegCore->getErrorString(errnum) + ")",
				"VideoDecodeStream_FFmpeg::Open");
		return false;
	}

	// update video info
	if (av_find_stream_info(_formatContext) < 0) {
		logger.error("No stream info found", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	ss.str("");
	ss << "VideoStreamIndex: " << _streamIndex;
	logger.info(ss.str(), "VideoPlayback_ffmpeg.Open");

	// find video stream
	int audioStreamIndex;
	ffmpegCore->findStreamIDs(_formatContext, &_streamIndex, &audioStreamIndex);
	if (_streamIndex < 0) {
		logger.error("No video stream found", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	_stream = _formatContext->streams[_streamIndex];
	_codecContext = _stream->codec;

	_codec = avcodec_find_decoder(_codecContext->codec_id);
	if (!_codec) {
		logger.error("No matching codec found", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	// set debug options
	_codecContext->debug_mv = 0;
	_codecContext->debug = 0;

	// detect bug-workarounds automatically
	_codecContext->workaround_bugs = FF_BUG_AUTODETECT;
	// error resilience strategy (careful/compliant/agressive/very_aggressive)
	//fCodecContext->error_resilience = FF_ER_CAREFUL; //FF_ER_COMPLIANT;
	// allow non spec compliant speedup tricks.
	//fCodecContext->flags2 = fCodecContext->flags2 | CODEC_FLAG2_FAST;

	// Note: avcodec_open() and avcodec_close() are not thread-safe and will
	// fail if called concurrently by different threads.
	{
		MediaCore_FFmpeg::AVCodecLock codecLock;
		errnum = avcodec_open(_codecContext, _codec);
	}
	if (errnum < 0) {
		logger.error("No matching codec found", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	// register custom callbacks for pts-determination
	_codecContext->get_buffer = ptsGetBuffer;
	_codecContext->release_buffer = ptsReleaseBuffer;

#ifdef DEBUG_DISPLAY
	ss.str("");
	ss << "Found a matching Codec: " << _codecContext->codec->name << std::endl
	   << std::endl
	   << "  Width = "     << _codecContext->width
	   << ", Height="      << _codecContext->height << std::endl
	   << "  Aspect : "    << _codecContext->sample_aspect_ratio.num << "/"
	                       << _codecContext->sample_aspect_ratio.den << std::endl
	   << "  Framerate : " << _codecContext->time_base.num << "/"
	                       << _codecContext->time_base.den;
	logger.status(ss.str(), "");
#endif

	_frameFormat = format;
	// choose default (use BGR-format for accelerated colorspace conversion with swscale)
	if (_frameFormat == FRAME_FORMAT_UNKNOWN)
		_frameFormat = FRAME_FORMAT_BGR;
	if (!ffmpegCore->convertVideoFrameFormatToFFmpeg(_frameFormat, &_pixelFormat)) {
		logger.error("Invalid pixel format", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	// allocate space for decoded frame and rgb frame
	_avFrame = avcodec_alloc_frame();
	_avFrameRGB = avcodec_alloc_frame();
	_frameBuffer = (uint8_t*) av_malloc(avpicture_get_size(_pixelFormat,
			_codecContext->width, _codecContext->height));

	if (!_avFrame || !_avFrameRGB || !_frameBuffer) {
		logger.error("Failed to allocate buffers", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	// TODO: pad data for OpenGL to GL_UNPACK_ALIGNMENT
	// (otherwise video will be distorted if width/height is not a multiple of the alignment)
	errnum = avpicture_fill((AVPicture*)_avFrameRGB, _frameBuffer,
			_pixelFormat, _codecContext->width, _codecContext->height);
	if (errnum < 0) {
		logger.error("avpicture_fill failed: " + ffmpegCore->getErrorString(errnum),
				"VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}

	// calculate some information for video display
	_aspect = av_q2d(_codecContext->sample_aspect_ratio);
	if (_aspect == 0) {
		_aspect = (double)_codecContext->width / _codecContext->height;
	} else {
		_aspect *= (double)_codecContext->width / _codecContext->height;
	}

	_frameDuration = 1.0 / av_q2d(_stream->r_frame_rate);

	// hack to get reasonable framerate (for divx and others)
	if (_frameDuration < 0.02) { // 0.02 <-> 50 fps
		_frameDuration = av_q2d(_stream->r_frame_rate);
		while (_frameDuration > 50.0)
			_frameDuration /= 10.0;
		_frameDuration = 1.0 / _frameDuration;
	}

	ss.str("");
	ss << "Framerate: " << (int)(1.0 / _frameDuration) << "fps";
	logger.info(ss.str(), "VideoPlayback_ffmpeg.Open");

#ifdef USE_SWSCALE
	// if available get a SWScale-context -> faster than the deprecated img_convert().
	// SWScale has accelerated support for PIX_FMT_RGB32/PIX_FMT_BGR24/PIX_FMT_BGR565/PIX_FMT_BGR555.
	// Note: PIX_FMT_RGB32 is a BGR- and not an RGB-format (maybe a bug)!!!
	// The BGR565-formats (GL_UNSIGNED_SHORT_5_6_5) is way too slow because of its
	// bad OpenGL support. The BGR formats have MMX(2) implementations but no speed-up
	// could be observed in comparison to the RGB versions.
	_swScaleContext = sws_getCachedContext(NULL,
			_codecContext->width, _codecContext->height, _codecContext->pix_fmt,
			_codecContext->width, _codecContext->height, _pixelFormat,
			SWS_FAST_BILINEAR,
			NULL, NULL, NULL);
	if (!_swScaleContext) {
		logger.error("Failed to get swscale context", "VideoPlayback_ffmpeg.Open");
		close();
		return false;
	}
#endif

	_opened = true;
	return true;
}

void FFmpegVideoDecodeStream::close() {
	if (_frameBuffer)
		av_free(_frameBuffer);
	if (_avFrameRGB)
		av_free(_avFrameRGB);
	if (_avFrame)
		av_free(_avFrame);

	_avFrame = NULL;
	_avFrameRGB = NULL;
	_frameBuffer = NULL;

	if (_codecContext) {
		// avcodec_close() is not thread-safe
		MediaCore_FFmpeg::AVCodecLock codecLock;
		avcodec_close(_codecContext);
	}

	if (_formatContext)
		av_close_input_file(_formatContext);

	_codecContext = NULL;
	_formatContext = NULL;

	_opened = false;
}

void FFmpegVideoDecodeStream::synchronizeTime(AVFrame *frame, double &pts) {
	if (pts != 0) {
		// if we have pts, set video clock to it
		_frameTime = pts;
	} else {
		// if we aren't given a pts, set it to the clock
		pts = _frameTime;
	}
	// update the video clock
	double frameDelay = av_q2d(_codecContext->time_base);
	// if we are repeating a frame, adjust clock accordingly
	frameDelay = frameDelay + frame->repeat_pict * (frameDelay * 0.5);
	_frameTime = _frameTime + frameDelay;
}

/**
 * Decode a new frame from the video stream.
 * The decoded frame is stored in fAVFrame. fFrameTime is updated to the new frame's
 * time.
 * @param pts will be updated to the presentation time of the decoded frame.
 * returns true if a frame could be decoded. False if an error or EOF occured.
 */
bool FFmpegVideoDecodeStream::decodeFrame() {
	int64_t videoPktPts;
	ByteIOContext *pbIOCtx;
	int errnum;
	AVPacket packet;
	double pts;

	if (_eof)
		return false;

	// read packets until we have a finished frame (or there are no more packets)
	int frameFinished = 0;
	while (frameFinished == 0) {
		errnum = av_read_frame(_formatContext, &packet);
		if (errnum < 0) {
			// failed to read a frame, check reason

#if LIBAVFORMAT_VERSION_MAJOR >= 52
			pbIOCtx = _formatContext->pb;
#else
			pbIOCtx = &_formatContext->pb;
#endif

			// check for end-of-file (EOF is not an error)
			if (url_feof(pbIOCtx) != 0) {
				_eof = true;
				return false;
			}

			// check for errors
			if (url_ferror(pbIOCtx) != 0) {
				logger.error("Video decoding file error", "TVideoPlayback_FFmpeg.DecodeFrame");
				return false;
			}

			// url_feof() does not detect an EOF for some mov-files (e.g. deluxe.mov)
			// so we have to do it this way.
			if ((_formatContext->file_size != 0) &&
					(pbIOCtx->pos >= _formatContext->file_size))
			{
				_eof = true;
				return false;
			}

			// error occured, log and exit
			logger.error("Video decoding error", "TVideoPlayback_FFmpeg.DecodeFrame");
			return false;
		}

		// if we got a packet from the video stream, then decode it
		if (packet.stream_index == _streamIndex) {
			// save pts to be stored in pFrame in first call of PtsGetBuffer()
			videoPktPts = packet.pts;
			// FIXME: is the pointer valid when it is used?
			_codecContext->opaque = &videoPktPts;

			// decode packet
			avcodec_decode_video2(_codecContext, _avFrame, &frameFinished, &packet);

			// reset opaque data
			_codecContext->opaque = NULL;

			// update pts
			if (packet.dts != (int64_t)AV_NOPTS_VALUE) {
				pts = packet.dts;
			} else if (_avFrame->opaque &&
					(*((int64_t*)_avFrame->opaque) != (int64_t)AV_NOPTS_VALUE))
			{
				pts = *((int64_t*)_avFrame->opaque);
			} else {
				pts = 0;
			}

			if (_stream->start_time != (int64_t)AV_NOPTS_VALUE)
				pts -= _stream->start_time;

			pts *= av_q2d(_stream->time_base);

			// synchronize time on each complete frame
			if (frameFinished != 0)
				synchronizeTime(_avFrame, pts);
		}

		// free the packet from av_read_frame
		av_free_packet(&packet);
	}

	return true;
}

#ifdef DEBUG_FRAMES
void spawnGoldenRec(double x, double y, int screen, uint8_t live, int startFrame,
		int recArrayIndex, unsigned player)
{
	//GoldenRec.Spawn(x, y, screen, live, startFrame, recArrayIndex, ColoredStar, player)
}
#endif

uint8_t* FFmpegVideoDecodeStream::getFrame(long double time) {
	const long double SKIP_FRAME_DIFF = 0.010; // start skipping if we are >= 10ms too late
	std::stringstream ss;

	if (!_opened)
		return NULL;

	/*
	 * Synchronization - begin
	 */

	// requested stream position (relative to the last loop's start)
	long double currentTime;
	if (_loop)
		currentTime = time - _loopTime;
	else
		currentTime = time;

	// check if current texture still contains the active frame
	if (_frameTexValid) {
		// time since the last frame was returned
		long double timeDiff = currentTime - _frameTime;

#ifdef DEBUG_DISPLAY
		ss.str("");
		ss << "time: " << floor(time*1000) << std::endl
		   << "VideoTime: " << floor(_frameTime*1000) << std::endl
		   << "TimeBase: " << floor(_frameDuration*1000) << std::endl
		   << "timeDiff: " << floor(timeDiff*1000);
		logger.status(ss.str(), "");
#endif

		// check if time has reached the next frame
		if (timeDiff < _frameDuration) {
#ifdef DEBUG_FRAMES
			// frame delay debug display
			spawnGoldenRec(200, 15, 1, 16, 0, -1, 0x00ff00);
#endif

#ifdef DEBUG_DISPLAY
			ss.str("");
			ss << "not getting new frame" << std::endl
			   << "time: " << floor(time*1000) << std::endl
			   << "VideoTime: " << floor(_frameTime*1000) << std::endl
			   << "TimeBase: " << floor(_frameDuration*1000) << std::endl
			   << "timeDiff: " << floor(timeDiff*1000);
			logger.status(ss.str(), "");
#endif

			// we do not need a new frame now
			return NULL;
		}
	}

	// fetch new frame (updates fFrameTime)
	bool success = decodeFrame();
	long double timeDiff = currentTime - _frameTime;

	// check if we have to skip frames
	// Either if we are one frame behind or if the skip threshold has been reached.
	// Do not skip if the difference is less than fFrameDuration as there is no next frame.
	// Note: We assume that fFrameDuration is the length of one frame.
	if (timeDiff >= std::max(_frameDuration, SKIP_FRAME_DIFF)) {
#ifdef DEBUG_FRAMES
		//frame drop debug display
		spawnGoldenRec(200, 55, 1, 16, 0, -1, 0xff0000);
#endif
#ifdef DEBUG_DISPLAY
		ss.str("");
		ss << "skipping frames" << std::endl
		   << "TimeBase: " << floor(_frameDuration*1000) << std::endl
		   << "timeDiff: " << floor(timeDiff*1000);
		logger.status(ss.str(), "");
#endif

		// update video-time
		int dropFrameCount = (int)(timeDiff / _frameDuration);
		_frameTime = _frameTime + dropFrameCount * _frameDuration;

		// skip frames
		for (int i = 1; i <= dropFrameCount; ++i)
			success = decodeFrame();
	}

	// check if we got an EOF or error
	if (!success) {
		if (_loop) {
			// we have to loop, so rewind
			setPosition(0);
			// record the start-time of the current loop, so we can
			// determine the position in the stream (fFrameTime-fLoopTime) later.
			_loopTime = time;
		}
		return NULL;
	}

	/*
	 * Synchronization - end
	 */

	// TODO: support for pan&scan
	//if (_avFrame->pan_scan) {
	//	printf("PanScan: %d/%d", _avFrame->pan_scan->width, _avFrame->pan_scan->height);
	//}

	// otherwise we convert the pixeldata from YUV to RGB
	int errnum;
#ifdef USE_SWSCALE
	errnum = sws_scale(_swScaleContext,
			(uint8_t**)_avFrame->data, _avFrame->linesize,
			0, _codecContext->height,
			(uint8_t**)_avFrameRGB->data, _avFrameRGB->linesize);
#else
	// img_convert from lib/ffmpeg/avcodec.pas is actually deprecated.
	// If ./configure does not find SWScale then this gives the error
	// that the identifier img_convert is not known or similar.
	// I think this should be removed, but am not sure whether there should
	// be some other replacement or a warning, Therefore, I leave it for now.
	// April 2009, mischi
	errnum = img_convert((AVPicture*)_avFrameRGB, _pixelFormat,
			(AVPicture*)_avFrame, _codecContext->pix_fmt,
			_codecContext->width, _codecContext->height);
#endif

	if (errnum < 0) {
		logger.error("Image conversion failed", "TVideoPlayback_ffmpeg.GetFrame");
		return NULL;
	}

	if (!_frameTexValid)
		_frameTexValid = true;

	return _avFrameRGB->data[0];
}

void FFmpegVideoDecodeStream::setLoop(bool enable) {
	_loop = enable;
	_loopTime = 0;
}

bool FFmpegVideoDecodeStream::getLoop() {
	return _loop;
}

/**
 * Sets the stream's position.
 * The stream is set to the first keyframe with timestamp <= Time.
 * Note that fFrameTime is set to Time no matter if the actual position seeked to is
 * at Time or the time of a preceding keyframe. fFrameTime will be updated to the
 * actual frame time when GetFrame() is called the next time.
 * @param Time new position in seconds
 */
void FFmpegVideoDecodeStream::setPosition(double time) {
	int seekFlags;

	if (!_opened)
		return;

	if (time < 0)
		time = 0;

	// TODO: handle fLoop-times
	//time %= videoDuration;

	// Do not use the AVSEEK_FLAG_ANY here. It will seek to any frame, even
	// non keyframes (P-/B-frames). It will produce corrupted video frames as
	// FFmpeg does not use the information of the preceding I-frame.
	// The picture might be gray or green until the next keyframe occurs.
	// Instead seek the first keyframe smaller than the requested time
	// (AVSEEK_FLAG_BACKWARD). As this can be some seconds earlier than the
	// requested time, let the sync in GetFrame() do its job.
	seekFlags = AVSEEK_FLAG_BACKWARD;

	_frameTime = time;
	_eof = false;
	_frameTexValid = false;

	if (av_seek_frame(_formatContext, _streamIndex,
			llround(time / av_q2d(_stream->time_base)), seekFlags) < 0)
	{
		logger.error("av_seek_frame() failed", "TVideoPlayback_ffmpeg.SetPosition");
		return;
	}

	avcodec_flush_buffers(_codecContext);
}

double FFmpegVideoDecodeStream::getPosition() {
	return _frameTime;
}

int FFmpegVideoDecodeStream::getFrameWidth() {
	return _codecContext->width;
}

int FFmpegVideoDecodeStream::getFrameHeight() {
	return _codecContext->height;
}

double FFmpegVideoDecodeStream::getFrameAspect() {
	return _aspect;
}

videoFrameFormat_t FFmpegVideoDecodeStream::getFrameFormat() {
	return _frameFormat;
}

/************************************
 * C Interface
 ************************************/

#define VideoDecodeStreamObj(ptr) reinterpret_cast<FFmpegVideoDecodeStream*>(ptr)

static BOOL PLUGIN_CALL ffmpegVideoDecoder_init() {
	return TRUE;
}

static BOOL PLUGIN_CALL ffmpegVideoDecoder_finalize() {
	return TRUE;
}

static videoDecodeStream_t* PLUGIN_CALL ffmpegVideoDecoder_open(const char *filename,
	videoFrameFormat_t format) 
{
	return (videoDecodeStream_t*)FFmpegVideoDecodeStream::open(filename, format);
}

static void PLUGIN_CALL ffmpegVideoDecoder_close(videoDecodeStream_t *stream) {
	delete VideoDecodeStreamObj(stream);
}

static void PLUGIN_CALL ffmpegVideoDecoder_setLoop(videoDecodeStream_t *stream, BOOL enable) {
	VideoDecodeStreamObj(stream)->setLoop(enable);
}

static BOOL PLUGIN_CALL ffmpegVideoDecoder_getLoop(videoDecodeStream_t *stream) {
	return (BOOL)VideoDecodeStreamObj(stream)->getLoop();
}

static void PLUGIN_CALL ffmpegVideoDecoder_setPosition(videoDecodeStream_t *stream, double time) {
	VideoDecodeStreamObj(stream)->setPosition(time);
}

static double PLUGIN_CALL ffmpegVideoDecoder_getPosition(videoDecodeStream_t *stream) {
	return VideoDecodeStreamObj(stream)->getPosition();
}

static void PLUGIN_CALL ffmpegVideoDecoder_getFrameInfo(videoDecodeStream_t *stream,
	videoFrameInfo_t *info)
{
	FFmpegVideoDecodeStream* s = VideoDecodeStreamObj(stream);
	info->width = s->getFrameWidth();
	info->height = s->getFrameHeight();
	info->aspect = s->getFrameAspect();
	info->format = s->getFrameFormat();
}

static uint8_t* PLUGIN_CALL ffmpegVideoDecoder_getFrame(videoDecodeStream_t *stream, long double time) {
	return VideoDecodeStreamObj(stream)->getFrame(time);
}

/************************************
 * Module information
 ************************************/

const videoDecoderInfo_t videoDecoderInfo = {
		80,
		ffmpegVideoDecoder_init,
		ffmpegVideoDecoder_finalize,
		ffmpegVideoDecoder_open,
		ffmpegVideoDecoder_close,
		ffmpegVideoDecoder_setLoop,
		ffmpegVideoDecoder_getLoop,
		ffmpegVideoDecoder_setPosition,
		ffmpegVideoDecoder_getPosition,
		ffmpegVideoDecoder_getFrameInfo,
		ffmpegVideoDecoder_getFrame
};