{* UltraStar Deluxe - Karaoke Game
*
* UltraStar Deluxe is the legal property of its developers, whose names
* are too numerous to list here. Please refer to the COPYRIGHT
* file distributed with this source distribution.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file COPYING. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
* $URL$
* $Id$
*}
unit UVideoDecoder_FFmpeg;
{*
* based on 'An ffmpeg and SDL Tutorial' (http://www.dranger.com/ffmpeg/)
*}
// uncomment if you want to see the debug stuff
{.$define DebugDisplay}
{.$define DebugFrames}
interface
{$IFDEF FPC}
{$MODE Delphi}
{$ENDIF}
{$I switches.inc}
// use BGR-format for accelerated colorspace conversion with swscale
{$IFDEF UseSWScale}
{$DEFINE PIXEL_FMT_BGR}
{$ENDIF}
implementation
uses
SysUtils,
Math,
avcodec,
avformat,
avutil,
avio,
rational,
{$IFDEF UseSWScale}
swscale,
{$ENDIF}
UMediaCore_FFmpeg,
UCommon,
UConfig,
ULog,
UMusic,
UPath;
{$DEFINE PIXEL_FMT_BGR}
const
{$IFDEF PIXEL_FMT_BGR}
PIXEL_FMT_FFMPEG = PIX_FMT_BGR24;
PIXEL_FMT_SIZE = 3;
// looks strange on linux:
//PIXEL_FMT_FFMPEG = PIX_FMT_BGR32;
//PIXEL_FMT_SIZE = 4;
{$ELSE}
// looks strange on linux:
PIXEL_FMT_FFMPEG = PIX_FMT_RGB24;
PIXEL_FMT_SIZE = 3;
{$ENDIF}
type
TVideoDecodeStream_FFmpeg = class (TVideoDecodeStream)
private
fOpened: boolean; //**< stream successfully opened
fEOF: boolean; //**< end-of-file state
fLoop: boolean; //**< looping enabled
fStream: PAVStream;
fStreamIndex : integer;
fFormatContext: PAVFormatContext;
fCodecContext: PAVCodecContext;
fCodec: PAVCodec;
fAVFrame: PAVFrame;
fAVFrameRGB: PAVFrame;
fFrameBuffer: PByte; //**< stores a FFmpeg video frame
fFrameTexValid: boolean; //**< if true, fFrameTex contains the current frame
{$IFDEF UseSWScale}
fSwScaleContext: PSwsContext;
{$ENDIF}
fAspect: real; //**< width/height ratio
fFrameDuration: extended; //**< duration of a video frame in seconds (= 1/fps)
fFrameTime: extended; //**< video time position (absolute)
fLoopTime: extended; //**< start time of the current loop
procedure Reset();
function DecodeFrame(): boolean;
procedure SynchronizeTime(Frame: PAVFrame; var pts: double);
public
constructor Create;
destructor Destroy; override;
function Open(const FileName: IPath): boolean; override;
procedure Close; override;
procedure SetLoop(Enable: boolean); override;
function GetLoop(): boolean; override;
procedure SetPosition(Time: real); override;
function GetPosition: real; override;
function GetFrameWidth(): integer; override;
function GetFrameHeight(): integer; override;
function GetFrameAspect(): real; override;
function GetFrame(Time: Extended): PByteArray; override;
end;
TVideoDecoder_FFmpeg = class( TInterfacedObject, IVideoDecoder )
private
fInitialized: boolean;
public
function GetName: String;
function InitializeDecoder(): boolean;
function FinalizeDecoder: boolean;
function Open(const FileName: IPath): TVideoDecodeStream;
end;
var
FFmpegCore: TMediaCore_FFmpeg;
// These are called whenever we allocate a frame buffer.
// We use this to store the global_pts in a frame at the time it is allocated.
function PtsGetBuffer(CodecCtx: PAVCodecContext; Frame: PAVFrame): integer; cdecl;
var
pts: Pint64;
VideoPktPts: Pint64;
begin
Result := avcodec_default_get_buffer(CodecCtx, Frame);
VideoPktPts := CodecCtx^.opaque;
if (VideoPktPts <> nil) then
begin
// Note: we must copy the pts instead of passing a pointer, because the packet
// (and with it the pts) might change before a frame is returned by av_decode_video.
pts := av_malloc(sizeof(int64));
pts^ := VideoPktPts^;
Frame^.opaque := pts;
end;
end;
procedure PtsReleaseBuffer(CodecCtx: PAVCodecContext; Frame: PAVFrame); cdecl;
begin
if (Frame <> nil) then
av_freep(@Frame^.opaque);
avcodec_default_release_buffer(CodecCtx, Frame);
end;
{*------------------------------------------------------------------------------
* TVideoPlayback_ffmpeg
*------------------------------------------------------------------------------}
function TVideoDecoder_FFmpeg.GetName: String;
begin
result := 'FFmpeg_VideoDecoder';
end;
function TVideoDecoder_FFmpeg.InitializeDecoder(): boolean;
begin
Result := true;
if (fInitialized) then
Exit;
fInitialized := true;
FFmpegCore := TMediaCore_FFmpeg.GetInstance();
av_register_all();
end;
function TVideoDecoder_FFmpeg.FinalizeDecoder(): boolean;
begin
Result := true;
end;
function TVideoDecoder_FFmpeg.Open(const FileName : IPath): TVideoDecodeStream;
var
Stream: TVideoDecodeStream_FFmpeg;
begin
Result := nil;
Stream := TVideoDecodeStream_FFmpeg.Create;
if (not Stream.Open(FileName)) then
begin
Stream.Free;
Exit;
end;
Result := Stream
end;
{* TVideoDecoder_FFmpeg *}
constructor TVideoDecodeStream_FFmpeg.Create;
begin
Reset();
end;
destructor TVideoDecodeStream_FFmpeg.Destroy;
begin
Close();
end;
function TVideoDecodeStream_FFmpeg.Open(const FileName: IPath): boolean;
var
errnum: Integer;
AudioStreamIndex: integer;
begin
Result := false;
Reset();
// use custom 'ufile' protocol for UTF-8 support
errnum := av_open_input_file(fFormatContext, PAnsiChar('ufile:'+FileName.ToUTF8), nil, 0, nil);
if (errnum <> 0) then
begin
Log.LogError('Failed to open file "'+ FileName.ToNative +'" ('+FFmpegCore.GetErrorString(errnum)+')');
Exit;
end;
// update video info
if (av_find_stream_info(fFormatContext) < 0) then
begin
Log.LogError('No stream info found', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
Log.LogInfo('VideoStreamIndex : ' + inttostr(fStreamIndex), 'TVideoPlayback_ffmpeg.Open');
// find video stream
FFmpegCore.FindStreamIDs(fFormatContext, fStreamIndex, AudioStreamIndex);
if (fStreamIndex < 0) then
begin
Log.LogError('No video stream found', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
fStream := fFormatContext^.streams[fStreamIndex];
fCodecContext := fStream^.codec;
fCodec := avcodec_find_decoder(fCodecContext^.codec_id);
if (fCodec = nil) then
begin
Log.LogError('No matching codec found', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
// set debug options
fCodecContext^.debug_mv := 0;
fCodecContext^.debug := 0;
// detect bug-workarounds automatically
fCodecContext^.workaround_bugs := FF_BUG_AUTODETECT;
// error resilience strategy (careful/compliant/agressive/very_aggressive)
//fCodecContext^.error_resilience := FF_ER_CAREFUL; //FF_ER_COMPLIANT;
// allow non spec compliant speedup tricks.
//fCodecContext^.flags2 := fCodecContext^.flags2 or CODEC_FLAG2_FAST;
// Note: avcodec_open() and avcodec_close() are not thread-safe and will
// fail if called concurrently by different threads.
FFmpegCore.LockAVCodec();
try
errnum := avcodec_open(fCodecContext, fCodec);
finally
FFmpegCore.UnlockAVCodec();
end;
if (errnum < 0) then
begin
Log.LogError('No matching codec found', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
// register custom callbacks for pts-determination
fCodecContext^.get_buffer := PtsGetBuffer;
fCodecContext^.release_buffer := PtsReleaseBuffer;
{$ifdef DebugDisplay}
DebugWriteln('Found a matching Codec: '+ fCodecContext^.Codec.Name + sLineBreak +
sLineBreak +
' Width = '+inttostr(fCodecContext^.width) +
', Height='+inttostr(fCodecContext^.height) + sLineBreak +
' Aspect : '+inttostr(fCodecContext^.sample_aspect_ratio.num) + '/' +
inttostr(fCodecContext^.sample_aspect_ratio.den) + sLineBreak +
' Framerate : '+inttostr(fCodecContext^.time_base.num) + '/' +
inttostr(fCodecContext^.time_base.den));
{$endif}
// allocate space for decoded frame and rgb frame
fAVFrame := avcodec_alloc_frame();
fAVFrameRGB := avcodec_alloc_frame();
fFrameBuffer := av_malloc(avpicture_get_size(PIXEL_FMT_FFMPEG,
fCodecContext^.width, fCodecContext^.height));
if ((fAVFrame = nil) or (fAVFrameRGB = nil) or (fFrameBuffer = nil)) then
begin
Log.LogError('Failed to allocate buffers', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
// TODO: pad data for OpenGL to GL_UNPACK_ALIGNMENT
// (otherwise video will be distorted if width/height is not a multiple of the alignment)
errnum := avpicture_fill(PAVPicture(fAVFrameRGB), fFrameBuffer, PIXEL_FMT_FFMPEG,
fCodecContext^.width, fCodecContext^.height);
if (errnum < 0) then
begin
Log.LogError('avpicture_fill failed: ' + FFmpegCore.GetErrorString(errnum), 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
// calculate some information for video display
fAspect := av_q2d(fCodecContext^.sample_aspect_ratio);
if (fAspect = 0) then
fAspect := fCodecContext^.width /
fCodecContext^.height
else
fAspect := fAspect * fCodecContext^.width /
fCodecContext^.height;
fFrameDuration := 1/av_q2d(fStream^.r_frame_rate);
// hack to get reasonable framerate (for divx and others)
if (fFrameDuration < 0.02) then // 0.02 <-> 50 fps
begin
fFrameDuration := av_q2d(fStream^.r_frame_rate);
while (fFrameDuration > 50) do
fFrameDuration := fFrameDuration/10;
fFrameDuration := 1/fFrameDuration;
end;
Log.LogInfo('Framerate: '+inttostr(floor(1/fFrameDuration))+'fps', 'TVideoPlayback_ffmpeg.Open');
{$IFDEF UseSWScale}
// if available get a SWScale-context -> faster than the deprecated img_convert().
// SWScale has accelerated support for PIX_FMT_RGB32/PIX_FMT_BGR24/PIX_FMT_BGR565/PIX_FMT_BGR555.
// Note: PIX_FMT_RGB32 is a BGR- and not an RGB-format (maybe a bug)!!!
// The BGR565-formats (GL_UNSIGNED_SHORT_5_6_5) is way too slow because of its
// bad OpenGL support. The BGR formats have MMX(2) implementations but no speed-up
// could be observed in comparison to the RGB versions.
fSwScaleContext := sws_getContext(
fCodecContext^.width, fCodecContext^.height,
fCodecContext^.pix_fmt,
fCodecContext^.width, fCodecContext^.height,
PIXEL_FMT_FFMPEG,
SWS_FAST_BILINEAR, nil, nil, nil);
if (fSwScaleContext = nil) then
begin
Log.LogError('Failed to get swscale context', 'TVideoPlayback_ffmpeg.Open');
Close();
Exit;
end;
{$ENDIF}
fOpened := true;
Result := true;
end;
procedure TVideoDecodeStream_FFmpeg.Reset();
begin
// close previously opened video
Close();
fOpened := False;
fFrameDuration := 0;
fFrameTime := 0;
fStream := nil;
fStreamIndex := -1;
fFrameTexValid := false;
fEOF := false;
fLoop := false;
fLoopTime := 0;
end;
procedure TVideoDecodeStream_FFmpeg.Close;
begin
if (fFrameBuffer <> nil) then
av_free(fFrameBuffer);
if (fAVFrameRGB <> nil) then
av_free(fAVFrameRGB);
if (fAVFrame <> nil) then
av_free(fAVFrame);
fAVFrame := nil;
fAVFrameRGB := nil;
fFrameBuffer := nil;
if (fCodecContext <> nil) then
begin
// avcodec_close() is not thread-safe
FFmpegCore.LockAVCodec();
try
avcodec_close(fCodecContext);
finally
FFmpegCore.UnlockAVCodec();
end;
end;
if (fFormatContext <> nil) then
av_close_input_file(fFormatContext);
fCodecContext := nil;
fFormatContext := nil;
fOpened := False;
end;
procedure TVideoDecodeStream_FFmpeg.SynchronizeTime(Frame: PAVFrame; var pts: double);
var
FrameDelay: double;
begin
if (pts <> 0) then
begin
// if we have pts, set video clock to it
fFrameTime := pts;
end else
begin
// if we aren't given a pts, set it to the clock
pts := fFrameTime;
end;
// update the video clock
FrameDelay := av_q2d(fCodecContext^.time_base);
// if we are repeating a frame, adjust clock accordingly
FrameDelay := FrameDelay + Frame^.repeat_pict * (FrameDelay * 0.5);
fFrameTime := fFrameTime + FrameDelay;
end;
{**
* Decode a new frame from the video stream.
* The decoded frame is stored in fAVFrame. fFrameTime is updated to the new frame's
* time.
* @param pts will be updated to the presentation time of the decoded frame.
* returns true if a frame could be decoded. False if an error or EOF occured.
*}
function TVideoDecodeStream_FFmpeg.DecodeFrame(): boolean;
var
FrameFinished: Integer;
VideoPktPts: int64;
pbIOCtx: PByteIOContext;
errnum: integer;
AVPacket: TAVPacket;
pts: double;
begin
Result := false;
FrameFinished := 0;
if fEOF then
Exit;
// read packets until we have a finished frame (or there are no more packets)
while (FrameFinished = 0) do
begin
errnum := av_read_frame(fFormatContext, AVPacket);
if (errnum < 0) then
begin
// failed to read a frame, check reason
{$IF (LIBAVFORMAT_VERSION_MAJOR >= 52)}
pbIOCtx := fFormatContext^.pb;
{$ELSE}
pbIOCtx := @fFormatContext^.pb;
{$IFEND}
// check for end-of-file (EOF is not an error)
if (url_feof(pbIOCtx) <> 0) then
begin
fEOF := true;
Exit;
end;
// check for errors
if (url_ferror(pbIOCtx) <> 0) then
begin
Log.LogError('Video decoding file error', 'TVideoPlayback_FFmpeg.DecodeFrame');
Exit;
end;
// url_feof() does not detect an EOF for some mov-files (e.g. deluxe.mov)
// so we have to do it this way.
if ((fFormatContext^.file_size <> 0) and
(pbIOCtx^.pos >= fFormatContext^.file_size)) then
begin
fEOF := true;
Exit;
end;
// error occured, log and exit
Log.LogError('Video decoding error', 'TVideoPlayback_FFmpeg.DecodeFrame');
Exit;
end;
// if we got a packet from the video stream, then decode it
if (AVPacket.stream_index = fStreamIndex) then
begin
// save pts to be stored in pFrame in first call of PtsGetBuffer()
VideoPktPts := AVPacket.pts;
fCodecContext^.opaque := @VideoPktPts;
// decode packet
avcodec_decode_video(fCodecContext, fAVFrame,
frameFinished, AVPacket.data, AVPacket.size);
// reset opaque data
fCodecContext^.opaque := nil;
// update pts
if (AVPacket.dts <> AV_NOPTS_VALUE) then
begin
pts := AVPacket.dts;
end
else if ((fAVFrame^.opaque <> nil) and
(Pint64(fAVFrame^.opaque)^ <> AV_NOPTS_VALUE)) then
begin
pts := Pint64(fAVFrame^.opaque)^;
end
else
begin
pts := 0;
end;
if fStream^.start_time <> AV_NOPTS_VALUE then
pts := pts - fStream^.start_time;
pts := pts * av_q2d(fStream^.time_base);
// synchronize time on each complete frame
if (frameFinished <> 0) then
SynchronizeTime(fAVFrame, pts);
end;
// free the packet from av_read_frame
av_free_packet( @AVPacket );
end;
Result := true;
end;
function TVideoDecodeStream_FFmpeg.GetFrame(Time: Extended): PByteArray;
var
errnum: Integer;
CurrentTime: Extended;
TimeDiff: Extended;
DropFrameCount: Integer;
i: Integer;
Success: boolean;
const
SKIP_FRAME_DIFF = 0.010; // start skipping if we are >= 10ms too late
begin
Result := nil;
if not fOpened then
Exit;
{*
* Synchronization - begin
*}
// requested stream position (relative to the last loop's start)
if (fLoop) then
CurrentTime := Time - fLoopTime
else
CurrentTime := Time;
// check if current texture still contains the active frame
if (fFrameTexValid) then
begin
// time since the last frame was returned
TimeDiff := CurrentTime - fFrameTime;
{$IFDEF DebugDisplay}
DebugWriteln('Time: '+inttostr(floor(Time*1000)) + sLineBreak +
'VideoTime: '+inttostr(floor(fFrameTime*1000)) + sLineBreak +
'TimeBase: '+inttostr(floor(fFrameDuration*1000)) + sLineBreak +
'TimeDiff: '+inttostr(floor(TimeDifference*1000)));
{$endif}
// check if time has reached the next frame
if (TimeDiff < fFrameDuration) then
begin
{$ifdef DebugFrames}
// frame delay debug display
GoldenRec.Spawn(200,15,1,16,0,-1,ColoredStar,$00ff00);
{$endif}
{$IFDEF DebugDisplay}
DebugWriteln('not getting new frame' + sLineBreak +
'Time: '+inttostr(floor(Time*1000)) + sLineBreak +
'VideoTime: '+inttostr(floor(fFrameTime*1000)) + sLineBreak +
'TimeBase: '+inttostr(floor(fFrameDuration*1000)) + sLineBreak +
'TimeDiff: '+inttostr(floor(TimeDifference*1000)));
{$endif}
// we do not need a new frame now
Exit;
end;
end;
// fetch new frame (updates fFrameTime)
Success := DecodeFrame();
TimeDiff := CurrentTime - fFrameTime;
// check if we have to skip frames
// Either if we are one frame behind or if the skip threshold has been reached.
// Do not skip if the difference is less than fFrameDuration as there is no next frame.
// Note: We assume that fFrameDuration is the length of one frame.
if (TimeDiff >= Max(fFrameDuration, SKIP_FRAME_DIFF)) then
begin
{$IFDEF DebugFrames}
//frame drop debug display
GoldenRec.Spawn(200,55,1,16,0,-1,ColoredStar,$ff0000);
{$ENDIF}
{$IFDEF DebugDisplay}
DebugWriteln('skipping frames' + sLineBreak +
'TimeBase: '+inttostr(floor(fFrameDuration*1000)) + sLineBreak +
'TimeDiff: '+inttostr(floor(TimeDifference*1000)));
{$endif}
// update video-time
DropFrameCount := Trunc(TimeDiff / fFrameDuration);
fFrameTime := fFrameTime + DropFrameCount*fFrameDuration;
// skip frames
for i := 1 to DropFrameCount do
Success := DecodeFrame();
end;
// check if we got an EOF or error
if (not Success) then
begin
if fLoop then
begin
// we have to loop, so rewind
SetPosition(0);
// record the start-time of the current loop, so we can
// determine the position in the stream (fFrameTime-fLoopTime) later.
fLoopTime := Time;
end;
Exit;
end;
{*
* Synchronization - end
*}
// TODO: support for pan&scan
//if (fAVFrame.pan_scan <> nil) then
//begin
// Writeln(Format('PanScan: %d/%d', [fAVFrame.pan_scan.width, fAVFrame.pan_scan.height]));
//end;
// otherwise we convert the pixeldata from YUV to RGB
{$IFDEF UseSWScale}
errnum := sws_scale(fSwScaleContext, @fAVFrame.data, @fAVFrame.linesize,
0, fCodecContext^.Height,
@fAVFrameRGB.data, @fAVFrameRGB.linesize);
{$ELSE}
// img_convert from lib/ffmpeg/avcodec.pas is actually deprecated.
// If ./configure does not find SWScale then this gives the error
// that the identifier img_convert is not known or similar.
// I think this should be removed, but am not sure whether there should
// be some other replacement or a warning, Therefore, I leave it for now.
// April 2009, mischi
errnum := img_convert(PAVPicture(fAVFrameRGB), PIXEL_FMT_FFMPEG,
PAVPicture(fAVFrame), fCodecContext^.pix_fmt,
fCodecContext^.width, fCodecContext^.height);
{$ENDIF}
if (errnum < 0) then
begin
Log.LogError('Image conversion failed', 'TVideoPlayback_ffmpeg.GetFrame');
Exit;
end;
if (not fFrameTexValid) then
fFrameTexValid := true;
Result := PByteArray(fAVFrameRGB^.data[0]);
end;
procedure TVideoDecodeStream_FFmpeg.SetLoop(Enable: boolean);
begin
fLoop := Enable;
fLoopTime := 0;
end;
function TVideoDecodeStream_FFmpeg.GetLoop(): boolean;
begin
Result := fLoop;
end;
{**
* Sets the stream's position.
* The stream is set to the first keyframe with timestamp <= Time.
* Note that fFrameTime is set to Time no matter if the actual position seeked to is
* at Time or the time of a preceding keyframe. fFrameTime will be updated to the
* actual frame time when GetFrame() is called the next time.
* @param Time new position in seconds
*}
procedure TVideoDecodeStream_FFmpeg.SetPosition(Time: real);
var
SeekFlags: integer;
begin
if not fOpened then
Exit;
if (Time < 0) then
Time := 0;
// TODO: handle fLoop-times
//Time := Time mod VideoDuration;
// Do not use the AVSEEK_FLAG_ANY here. It will seek to any frame, even
// non keyframes (P-/B-frames). It will produce corrupted video frames as
// FFmpeg does not use the information of the preceding I-frame.
// The picture might be gray or green until the next keyframe occurs.
// Instead seek the first keyframe smaller than the requested time
// (AVSEEK_FLAG_BACKWARD). As this can be some seconds earlier than the
// requested time, let the sync in GetFrame() do its job.
SeekFlags := AVSEEK_FLAG_BACKWARD;
fFrameTime := Time;
fEOF := false;
fFrameTexValid := false;
if (av_seek_frame(fFormatContext,
fStreamIndex,
Round(Time / av_q2d(fStream^.time_base)),
SeekFlags) < 0) then
begin
Log.LogError('av_seek_frame() failed', 'TVideoPlayback_ffmpeg.SetPosition');
Exit;
end;
avcodec_flush_buffers(fCodecContext);
end;
function TVideoDecodeStream_FFmpeg.GetPosition: real;
begin
Result := fFrameTime;
end;
function TVideoDecodeStream_FFmpeg.GetFrameWidth(): integer;
begin
Result := fCodecContext^.width;
end;
function TVideoDecodeStream_FFmpeg.GetFrameHeight(): integer;
begin
Result := fCodecContext^.height;
end;
function TVideoDecodeStream_FFmpeg.GetFrameAspect(): real;
begin
Result := fAspect;
end;
initialization
MediaManager.Add(TVideoDecoder_FFmpeg.Create);
end.