From 1e1849ae3ce2d5e78eb1d20681e385ad7b502092 Mon Sep 17 00:00:00 2001 From: tobigun Date: Tue, 13 May 2008 18:39:34 +0000 Subject: - speedup patch for video playback. Should be a little faster than before. - use of video pts-info for better sync - some cleanup git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/trunk@1087 b956fd51-792f-4845-bead-9b4dfca2ff2c --- Game/Code/Classes/UVideo.pas | 1078 ++++++++++++++++++++++++------------------ 1 file changed, 610 insertions(+), 468 deletions(-) (limited to 'Game/Code/Classes') diff --git a/Game/Code/Classes/UVideo.pas b/Game/Code/Classes/UVideo.pas index 185a693d..fbe3ec32 100644 --- a/Game/Code/Classes/UVideo.pas +++ b/Game/Code/Classes/UVideo.pas @@ -1,17 +1,20 @@ +{############################################################################## + # FFmpeg support for UltraStar deluxe # + # # + # Created by b1indy # + # based on 'An ffmpeg and SDL Tutorial' (http://www.dranger.com/ffmpeg/) # + # with modifications by Jay Binks # + # # + # http://www.mail-archive.com/fpc-pascal@lists.freepascal.org/msg09949.html # + # http://www.nabble.com/file/p11795857/mpegpas01.zip # + # # + ##############################################################################} + unit UVideo; -{< ############################################################################# -# FFmpeg support for UltraStar deluxe # -# # -# Created by b1indy # -# based on 'An ffmpeg and SDL Tutorial' (http://www.dranger.com/ffmpeg/) # -# # -# http://www.mail-archive.com/fpc-pascal@lists.freepascal.org/msg09949.html # -# http://www.nabble.com/file/p11795857/mpegpas01.zip # -# # -############################################################################## } //{$define DebugDisplay} // uncomment if u want to see the debug stuff //{$define DebugFrames} +//{$define VideoBenchmark} //{$define Info} interface @@ -23,355 +26,505 @@ interface {$I switches.inc} (* - - look into - av_read_play - + TODO: look into av_read_play *) -implementation +// use BGR-format for accelerated colorspace conversion with swscale +{.$DEFINE PIXEL_FMT_BGR} -uses SDL, - UGraphicClasses, - textgl, - avcodec, - avformat, - avutil, - {$IFDEF UseSWScale} - swscale, - {$ENDIF} - math, - gl, - SysUtils, - {$ifdef DebugDisplay} - {$ifdef win32} - dialogs, - {$endif} - {$ENDIF} - UCommon, - UIni, - ULog, - UMusic, - UGraphic; +implementation +uses + SDL, + textgl, + avcodec, + avformat, + avutil, + avio, + rational, + {$IFDEF UseSWScale} + swscale, + {$ENDIF} + math, + gl, + SysUtils, + UCommon, + UConfig, + ULog, + UMusic, + UGraphicClasses, + UGraphic; -var - singleton_VideoFFMpeg : IVideoPlayback; +const +{$IFDEF PIXEL_FMT_BGR} + PIXEL_FMT_OPENGL = GL_BGR; + PIXEL_FMT_FFMPEG = PIX_FMT_BGR24; +{$ELSE} + PIXEL_FMT_OPENGL = GL_RGB; + PIXEL_FMT_FFMPEG = PIX_FMT_RGB24; +{$ENDIF} type - TVideoPlayback_ffmpeg = class( TInterfacedObject, IVideoPlayback ) - private - fVideoOpened , - fVideoPaused : Boolean; - - fVideoTex : glUint; - fVideoSkipTime : Single; - - VideoFormatContext: PAVFormatContext; - - VideoStreamIndex , - AudioStreamIndex : Integer; - VideoCodecContext: PAVCodecContext; - VideoCodec: PAVCodec; - AVFrame: PAVFrame; - AVFrameRGB: PAVFrame; - myBuffer: pByte; - - {$IFDEF UseSWScale} - SoftwareScaleContext: PSwsContext; - {$ENDIF} + TVideoPlayback_ffmpeg = class( TInterfacedObject, IVideoPlayback ) + private + fVideoOpened, + fVideoPaused: Boolean; + + VideoStream: PAVStream; + VideoStreamIndex : Integer; + VideoFormatContext: PAVFormatContext; + VideoCodecContext: PAVCodecContext; + VideoCodec: PAVCodec; + + AVFrame: PAVFrame; + AVFrameRGB: PAVFrame; + FrameBuffer: PByte; + + {$IFDEF UseSWScale} + SoftwareScaleContext: PSwsContext; + {$ENDIF} + + fVideoTex: GLuint; + TexWidth, TexHeight: Cardinal; + ScaledVideoWidth, ScaledVideoHeight: Real; + + VideoAspect: Real; + VideoTimeBase, VideoTime: Extended; + fLoopTime: Extended; + + EOF: boolean; + Loop: boolean; + + procedure Reset(); + function DecodeFrame(var AVPacket: TAVPacket; out pts: double): boolean; + function FindStreamIDs( const aFormatCtx : PAVFormatContext; out aFirstVideoStream, aFirstAudioStream : integer ): boolean; + procedure SynchronizeVideo(pFrame: PAVFrame; var pts: double); + public + constructor Create(); + function GetName: String; + procedure Init(); + + function Open(const aFileName : string): boolean; // true if succeed + procedure Close; + + procedure Play; + procedure Pause; + procedure Stop; + + procedure SetPosition(Time: real); + function GetPosition: real; + + procedure GetFrame(Time: Extended); + procedure DrawGL(Screen: integer); - TexX, TexY, dataX, dataY: Cardinal; - - ScaledVideoWidth, ScaledVideoHeight: Real; - VideoAspect: Real; - VideoTextureU, VideoTextureV: Real; - VideoTimeBase, VideoTime, LastFrameTime, TimeDifference, flooptime: Extended; - - - WantedAudioCodecContext, - AudioCodecContext : PSDL_AudioSpec; - aCodecCtx : PAVCodecContext; - - function find_stream_ids( const aFormatCtx : PAVFormatContext; Out aFirstVideoStream, aFirstAudioStream : integer ): boolean; + end; - public - constructor create(); - function GetName: String; - procedure init(); +var + singleton_VideoFFMpeg : IVideoPlayback; - function Open(const aFileName : string): boolean; // true if succeed - procedure Close; - procedure Play; - procedure Pause; - procedure Stop; - procedure SetPosition(Time: real); - function GetPosition: real; +function FFMpegErrorString(Errnum: integer): string; +begin + case Errnum of + AVERROR_IO: Result := 'AVERROR_IO'; + AVERROR_NUMEXPECTED: Result := 'AVERROR_NUMEXPECTED'; + AVERROR_INVALIDDATA: Result := 'AVERROR_INVALIDDATA'; + AVERROR_NOMEM: Result := 'AVERROR_NOMEM'; + AVERROR_NOFMT: Result := 'AVERROR_NOFMT'; + AVERROR_NOTSUPP: Result := 'AVERROR_NOTSUPP'; + AVERROR_NOENT: Result := 'AVERROR_NOENT'; + else Result := 'AVERROR_#'+inttostr(Errnum); + end; +end; - procedure GetFrame(Time: Extended); - procedure DrawGL(Screen: integer); +// These are called whenever we allocate a frame buffer. +// We use this to store the global_pts in a frame at the time it is allocated. +function PtsGetBuffer(pCodecCtx: PAVCodecContext; pFrame: PAVFrame): integer; cdecl; +var + pts: Pint64; + VideoPktPts: Pint64; +begin + Result := avcodec_default_get_buffer(pCodecCtx, pFrame); + VideoPktPts := pCodecCtx^.opaque; + if (VideoPktPts <> nil) then + begin + // Note: we must copy the pts instead of passing a pointer, because the packet + // (and with it the pts) might change before a frame is returned by av_decode_video. + pts := av_malloc(sizeof(int64)); + pts^ := VideoPktPts^; + pFrame^.opaque := pts; + end; +end; - end; +procedure PtsReleaseBuffer(pCodecCtx: PAVCodecContext; pFrame: PAVFrame); cdecl; +begin + if (pFrame <> nil) then + av_freep(@pFrame^.opaque); + avcodec_default_release_buffer(pCodecCtx, pFrame); +end; - const - SDL_AUDIO_BUFFER_SIZE = 1024; -{ ------------------------------------------------------------------------------ -asdf ------------------------------------------------------------------------------- } +{*------------------------------------------------------------------------------ + * TVideoPlayback_ffmpeg + *------------------------------------------------------------------------------} function TVideoPlayback_ffmpeg.GetName: String; begin - result := 'FFMpeg'; + result := 'FFMpeg_Video'; end; { - @author(Jay Binks ) - @created(2007-10-09) - @lastmod(2007-10-09) - @param(aFormatCtx is a PAVFormatContext returned from av_open_input_file ) @param(aFirstVideoStream is an OUT value of type integer, this is the index of the video stream) @param(aFirstAudioStream is an OUT value of type integer, this is the index of the audio stream) @returns(@true on success, @false otherwise) - - translated from "Setting Up the Audio" section at - http://www.dranger.com/ffmpeg/ffmpegtutorial_all.html } -function TVideoPlayback_ffmpeg.find_stream_ids( const aFormatCtx : PAVFormatContext; Out aFirstVideoStream, aFirstAudioStream : integer ): boolean; +function TVideoPlayback_ffmpeg.FindStreamIDs(const aFormatCtx: PAVFormatContext; out aFirstVideoStream, aFirstAudioStream: integer): boolean; var i : integer; - st : pAVStream; + st : PAVStream; begin // Find the first video stream aFirstAudioStream := -1; aFirstVideoStream := -1; - debugwriteln( ' aFormatCtx.nb_streams : ' + inttostr( aFormatCtx.nb_streams ) ); - debugwriteln( ' length( aFormatCtx.streams ) : ' + inttostr( length(aFormatCtx.streams) ) ); + {$IFDEF DebugDisplay} + debugwriteln('aFormatCtx.nb_streams : ' + inttostr(aFormatCtx.nb_streams)); + {$ENDIF} - i := 0; - while ( i < aFormatCtx.nb_streams ) do -// while ( i < length(aFormatCtx.streams)-1 ) do + for i := 0 to aFormatCtx.nb_streams-1 do begin - debugwriteln( ' aFormatCtx.streams[i] : ' + inttostr( i ) ); st := aFormatCtx.streams[i]; - if(st.codec.codec_type = CODEC_TYPE_VIDEO ) AND - (aFirstVideoStream < 0) THEN + if (st.codec.codec_type = CODEC_TYPE_VIDEO) and + (aFirstVideoStream < 0) then begin aFirstVideoStream := i; end; - if ( st.codec.codec_type = CODEC_TYPE_AUDIO ) AND - ( aFirstAudioStream < 0) THEN + if (st.codec.codec_type = CODEC_TYPE_AUDIO) and + (aFirstAudioStream < 0) then begin aFirstAudioStream := i; end; + end; - inc( i ); - end; // while + // return true if either an audio- or video-stream was found + result := (aFirstAudioStream > -1) or + (aFirstVideoStream > -1) ; +end; - result := (aFirstAudioStream > -1) OR - (aFirstVideoStream > -1) ; // Didn't find a video stream +procedure TVideoPlayback_ffmpeg.SynchronizeVideo(pFrame: PAVFrame; var pts: double); +var + FrameDelay: double; +begin + if (pts <> 0) then + begin + // if we have pts, set video clock to it + VideoTime := pts; + end else + begin + // if we aren't given a pts, set it to the clock + pts := VideoTime; + end; + // update the video clock + FrameDelay := av_q2d(VideoCodecContext^.time_base); + // if we are repeating a frame, adjust clock accordingly + FrameDelay := FrameDelay + pFrame^.repeat_pict * (FrameDelay * 0.5); + VideoTime := VideoTime + FrameDelay; end; +function TVideoPlayback_ffmpeg.DecodeFrame(var AVPacket: TAVPacket; out pts: double): boolean; +var + FrameFinished: Integer; + VideoPktPts: int64; + pbIOCtx: PByteIOContext; + errnum: integer; +begin + Result := false; + FrameFinished := 0; + + if EOF then + Exit; + + // read packets until we have a finished frame (or there are no more packets) + while (FrameFinished = 0) do + begin + errnum := av_read_frame(VideoFormatContext, AVPacket); + if (errnum < 0) then + begin + // failed to read a frame, check reason + + {$IF (LIBAVFORMAT_VERSION_MAJOR >= 52)} + pbIOCtx := VideoFormatContext^.pb; + {$ELSE} + pbIOCtx := @VideoFormatContext^.pb; + {$IFEND} + + // check for end-of-file (eof is not an error) + if (url_feof(pbIOCtx) <> 0) then + begin + EOF := true; + Exit; + end; + + // check for errors + if (url_ferror(pbIOCtx) <> 0) then + Exit; + + // url_feof() does not detect an EOF for some mov-files (e.g. deluxe.mov) + // so we have to do it this way. + if ((VideoFormatContext^.file_size <> 0) and + (pbIOCtx^.pos >= VideoFormatContext^.file_size)) then + begin + EOF := true; + Exit; + end; + // no error -> wait for user input + SDL_Delay(100); + continue; + end; + // if we got a packet from the video stream, then decode it + if (AVPacket.stream_index = VideoStreamIndex) then + begin + // save pts to be stored in pFrame in first call of PtsGetBuffer() + VideoPktPts := AVPacket.pts; + VideoCodecContext^.opaque := @VideoPktPts; + + // decode packet + avcodec_decode_video(VideoCodecContext, AVFrame, + frameFinished, AVPacket.data, AVPacket.size); + + // reset opaque data + VideoCodecContext^.opaque := nil; + + // update pts + if (AVPacket.dts <> AV_NOPTS_VALUE) then + begin + pts := AVPacket.dts; + end + else if ((AVFrame^.opaque <> nil) and + (Pint64(AVFrame^.opaque)^ <> AV_NOPTS_VALUE)) then + begin + pts := Pint64(AVFrame^.opaque)^; + end + else + begin + pts := 0; + end; + pts := pts * av_q2d(VideoStream^.time_base); + + // synchronize on each complete frame + if (frameFinished <> 0) then + SynchronizeVideo(AVFrame, pts); + end; + + // free the packet from av_read_frame + av_free_packet( @AVPacket ); + end; + + Result := true; +end; procedure TVideoPlayback_ffmpeg.GetFrame(Time: Extended); var - FrameFinished: Integer; AVPacket: TAVPacket; -errnum, {*x, *}y: Integer; // Auto Removed, Unused Variable (x) -// FrameDataPtr: PByteArray; // Auto Removed, Unused Variable -// linesize: integer; // Auto Removed, Unused Variable + errnum: Integer; myTime: Extended; - DropFrame: Boolean; - droppedFrames: Integer; + TimeDifference: Extended; + DropFrameCount: Integer; + pts: double; + i: Integer; const - FRAMEDROPCOUNT=3; + FRAME_DROPCOUNT = 3; begin - if not fVideoOpened then Exit; + if not fVideoOpened then + Exit; - if fVideoPaused then Exit; + if fVideoPaused then + Exit; - myTime := ( Time - flooptime ) + fVideoSkipTime; + // current time, relative to last loop (if any) + myTime := Time - fLoopTime; + // time since the last frame was returned TimeDifference := myTime - VideoTime; - DropFrame := False; -{$IFDEF DebugDisplay} - showmessage('Time: '+inttostr(floor(Time*1000))+#13#10+ - 'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+ - 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ - 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); -{$endif} + {$IFDEF DebugDisplay} + DebugWriteln('Time: '+inttostr(floor(Time*1000)) + sLineBreak + + 'VideoTime: '+inttostr(floor(VideoTime*1000)) + sLineBreak + + 'TimeBase: '+inttostr(floor(VideoTimeBase*1000)) + sLineBreak + + 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); + {$endif} - if (VideoTime <> 0) and (TimeDifference+flooptime <= VideoTimeBase) then + // check if a new frame is needed + if (VideoTime <> 0) and (TimeDifference < VideoTimeBase) then begin -{$ifdef DebugFrames} + {$ifdef DebugFrames} // frame delay debug display GoldenRec.Spawn(200,15,1,16,0,-1,ColoredStar,$00ff00); -{$endif} + {$endif} -{$IFDEF DebugDisplay} - showmessage('not getting new frame'+#13#10+ - 'Time: '+inttostr(floor(Time*1000))+#13#10+ - 'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+ - 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ - 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); -{$endif} + {$IFDEF DebugDisplay} + DebugWriteln('not getting new frame' + sLineBreak + + 'Time: '+inttostr(floor(Time*1000)) + sLineBreak + + 'VideoTime: '+inttostr(floor(VideoTime*1000)) + sLineBreak + + 'TimeBase: '+inttostr(floor(VideoTimeBase*1000)) + sLineBreak + + 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); + {$endif} - Exit;// we don't need a new frame now + // we do not need a new frame now + Exit; end; - VideoTime:=VideoTime+VideoTimeBase; - TimeDifference:=myTime-VideoTime; - if TimeDifference >= (FRAMEDROPCOUNT-1)*VideoTimeBase then // skip frames + // update video-time to the next frame + VideoTime := VideoTime + VideoTimeBase; + TimeDifference := myTime - VideoTime; + + // check if we have to skip frames + if (TimeDifference >= FRAME_DROPCOUNT*VideoTimeBase) then begin -{$ifdef DebugFrames} + {$IFDEF DebugFrames} //frame drop debug display GoldenRec.Spawn(200,55,1,16,0,-1,ColoredStar,$ff0000); -{$endif} -{$IFDEF DebugDisplay} - showmessage('skipping frames'+#13#10+ - 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ - 'TimeDiff: '+inttostr(floor(TimeDifference*1000))+#13#10+ - 'Time2Skip: '+inttostr(floor((Time-LastFrameTime)*1000))); -{$endif} - VideoTime:=VideoTime+FRAMEDROPCOUNT*VideoTimeBase; - DropFrame:=True; - end; - - AVPacket.data := nil; - av_init_packet( AVPacket ); // JB-ffmpeg - - FrameFinished:=0; - // read packets until we have a finished frame (or there are no more packets) - while ( FrameFinished = 0 ) do - begin - if ( av_read_frame(VideoFormatContext, AVPacket) < 0 ) then - begin - // Record the Time we looped, this is used to keep the loops, in time. otherwise they speed - flooptime := time; - - // Dont use SetPosition() it dosnt let us go back to frame 0... can we / should we fix this ?? - fVideoSkipTime := 0; - VideoTime := 0; - - // Free the packet we just got from av_read_frame - av_free_packet( @AVPacket ); - - // Seek to frame 0 in the video stream - av_seek_frame(VideoFormatContext,VideoStreamIndex,0,AVSEEK_FLAG_ANY); - break; - end; - - - // if we got a packet from the video stream, then decode it - if (AVPacket.stream_index=VideoStreamIndex) then - begin - errnum := avcodec_decode_video(VideoCodecContext, AVFrame, frameFinished , AVPacket.data, AVPacket.size); // JB-ffmpeg - (* FIXME - {$ifdef UseFFMpegAudio} - end - else - if (AVPacket.stream_index = AudioStreamIndex ) then - begin - debugwriteln('Encue Audio packet'); - audioq.put(AVPacket); + {$ENDIF} + {$IFDEF DebugDisplay} + DebugWriteln('skipping frames' + sLineBreak + + 'TimeBase: '+inttostr(floor(VideoTimeBase*1000)) + sLineBreak + + 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); {$endif} - *) - end; - try -// if AVPacket.data <> nil then - av_free_packet( @AVPacket ); // JB-ffmpeg - except - // TODO : JB_FFMpeg ... why does this now AV sometimes ( or always !! ) - end; + // update video-time + DropFrameCount := Trunc(TimeDifference / VideoTimeBase); + VideoTime := VideoTime + DropFrameCount*VideoTimeBase; + // skip half of the frames, this is much smoother than to skip all at once + for i := 1 to DropFrameCount (*div 2*) do + DecodeFrame(AVPacket, pts); end; - if DropFrame then - for droppedFrames:=1 to FRAMEDROPCOUNT do begin - FrameFinished:=0; - // read packets until we have a finished frame (or there are no more packets) - while (FrameFinished=0) do - begin - if (av_read_frame(VideoFormatContext, AVPacket)<0) then - Break; - // if we got a packet from the video stream, then decode it - if (AVPacket.stream_index=VideoStreamIndex) then - errnum:=avcodec_decode_video(VideoCodecContext, AVFrame, frameFinished , AVPacket.data, AVPacket.size); // JB-ffmpeg - - // release internal packet structure created by av_read_frame - try -// if AVPacket.data <> nil then - av_free_packet( @AVPacket ); // JB-ffmpeg - except - // TODO : JB_FFMpeg ... why does this now AV sometimes ( or always !! ) - end; - end; - end; + {$IFDEF VideoBenchmark} + Log.BenchmarkStart(15); + {$ENDIF} - // if we did not get an new frame, there's nothing more to do - if Framefinished=0 then begin + if (not DecodeFrame(AVPacket, pts)) then + begin + if Loop then + begin + // Record the time we looped. This is used to keep the loops in time. otherwise they speed + SetPosition(0); + fLoopTime := Time; + end; Exit; end; // otherwise we convert the pixeldata from YUV to RGB {$IFDEF UseSWScale} - errnum:=sws_scale(SoftwareScaleContext,@(AVFrame.data),@(AVFrame.linesize), - 0,VideoCodecContext^.Height, - @(AVFrameRGB.data),@(AVFrameRGB.linesize)); + errnum := sws_scale(SoftwareScaleContext, @(AVFrame.data), @(AVFrame.linesize), + 0, VideoCodecContext^.Height, + @(AVFrameRGB.data), @(AVFrameRGB.linesize)); {$ELSE} - errnum:=img_convert(PAVPicture(AVFrameRGB), PIX_FMT_RGB24, + errnum := img_convert(PAVPicture(AVFrameRGB), PIXEL_FMT_FFMPEG, PAVPicture(AVFrame), VideoCodecContext^.pix_fmt, VideoCodecContext^.width, VideoCodecContext^.height); {$ENDIF} - - if errnum >=0 then + + if (errnum < 0) then begin - glBindTexture(GL_TEXTURE_2D, fVideoTex); - glTexImage2D(GL_TEXTURE_2D, 0, 3, dataX, dataY, 0, GL_RGB, GL_UNSIGNED_BYTE, AVFrameRGB^.data[0]); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); -{$ifdef DebugFrames} - //frame decode debug display - GoldenRec.Spawn(200,35,1,16,0,-1,ColoredStar,$ffff00); -{$endif} + Log.LogError('Image conversion failed', 'TVideoPlayback_ffmpeg.GetFrame'); + Exit; end; + + {$IFDEF VideoBenchmark} + Log.BenchmarkEnd(15); + Log.BenchmarkStart(16); + {$ENDIF} + + // TODO: data is not padded, so we will need to tell OpenGL. + // Or should we add padding with avpicture_fill? (check which one is faster) + //glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + glBindTexture(GL_TEXTURE_2D, fVideoTex); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, + VideoCodecContext^.width, VideoCodecContext^.height, + PIXEL_FMT_OPENGL, GL_UNSIGNED_BYTE, AVFrameRGB^.data[0]); + + {$ifdef DebugFrames} + //frame decode debug display + GoldenRec.Spawn(200, 35, 1, 16, 0, -1, ColoredStar, $ffff00); + {$endif} + + {$IFDEF VideoBenchmark} + Log.BenchmarkEnd(16); + Log.LogBenchmark('FFmpeg', 15); + Log.LogBenchmark('Texture', 16); + {$ENDIF} end; procedure TVideoPlayback_ffmpeg.DrawGL(Screen: integer); +var + TexVideoRightPos, TexVideoLowerPos: Single; + ScreenLeftPos, ScreenRightPos: Single; + ScreenUpperPos, ScreenLowerPos: Single; +const + ScreenMidPosX = 400.0; + ScreenMidPosY = 300.0; begin // have a nice black background to draw on (even if there were errors opening the vid) - if Screen=1 then + if (Screen = 1) then begin - glClearColor(0,0,0,0); + glClearColor(0, 0, 0, 0); glClear(GL_COLOR_BUFFER_BIT or GL_DEPTH_BUFFER_BIT); end; + // exit if there's nothing to draw - if not fVideoOpened then Exit; + if (not fVideoOpened) then + Exit; + + {$IFDEF VideoBenchmark} + Log.BenchmarkStart(15); + {$ENDIF} + + TexVideoRightPos := VideoCodecContext^.width / TexWidth; + TexVideoLowerPos := VideoCodecContext^.height / TexHeight; + ScreenLeftPos := ScreenMidPosX - ScaledVideoWidth/2; + ScreenRightPos := ScreenMidPosX + ScaledVideoWidth/2; + ScreenUpperPos := ScreenMidPosY - ScaledVideoHeight/2; + ScreenLowerPos := ScreenMidPosY + ScaledVideoHeight/2; + + // we could use blending for brightness control, but do we need this? + glDisable(GL_BLEND); + + // TODO: disable other stuff like lightning, etc. glEnable(GL_TEXTURE_2D); - glEnable(GL_BLEND); - glColor4f(1, 1, 1, 1); glBindTexture(GL_TEXTURE_2D, fVideoTex); - glbegin(gl_quads); - glTexCoord2f( 0, 0); glVertex2f(400-ScaledVideoWidth/2, 300-ScaledVideoHeight/2); - glTexCoord2f( 0, TexY/dataY); glVertex2f(400-ScaledVideoWidth/2, 300+ScaledVideoHeight/2); - glTexCoord2f(TexX/dataX, TexY/dataY); glVertex2f(400+ScaledVideoWidth/2, 300+ScaledVideoHeight/2); - glTexCoord2f(TexX/dataX, 0); glVertex2f(400+ScaledVideoWidth/2, 300-ScaledVideoHeight/2); + glColor3f(1, 1, 1); + glBegin(GL_QUADS); + // upper-left coord + glTexCoord2f(0, 0); + glVertex2f(ScreenLeftPos, ScreenUpperPos); + // lower-left coord + glTexCoord2f(0, TexVideoLowerPos); + glVertex2f(ScreenLeftPos, ScreenLowerPos); + // lower-right coord + glTexCoord2f(TexVideoRightPos, TexVideoLowerPos); + glVertex2f(ScreenRightPos, ScreenLowerPos); + // upper-right coord + glTexCoord2f(TexVideoRightPos, 0); + glVertex2f(ScreenRightPos, ScreenUpperPos); glEnd; glDisable(GL_TEXTURE_2D); - glDisable(GL_BLEND); -{$ifdef Info} - if fVideoSkipTime+VideoTime+VideoTimeBase < 0 then + {$IFDEF VideoBenchmark} + Log.BenchmarkEnd(15); + Log.LogBenchmark('DrawGL', 15); + {$ENDIF} + + {$IFDEF Info} + if (fVideoSkipTime+VideoTime+VideoTimeBase < 0) then begin glColor4f(0.7, 1, 0.3, 1); SetFontStyle (1); @@ -381,18 +534,18 @@ begin glPrint('Delay due to negative VideoGap'); glColor4f(1, 1, 1, 1); end; -{$endif} + {$ENDIF} -{$ifdef DebugFrames} + {$IFDEF DebugFrames} glColor4f(0, 0, 0, 0.2); - glbegin(gl_quads); + glbegin(GL_QUADS); glVertex2f(0, 0); glVertex2f(0, 70); glVertex2f(250, 70); glVertex2f(250, 0); glEnd; - glColor4f(1,1,1,1); + glColor4f(1, 1, 1, 1); SetFontStyle (1); SetFontItalic(False); SetFontSize(9); @@ -402,248 +555,223 @@ begin glPrint('fetching frame'); SetFontPos (5, 40); glPrint('dropping frame'); -{$endif} + {$ENDIF} end; -constructor TVideoPlayback_ffmpeg.create(); +constructor TVideoPlayback_ffmpeg.Create(); begin inherited; - - av_register_all; - - fVideoOpened := False; - fVideoPaused := False; + Reset(); + av_register_all(); end; -procedure TVideoPlayback_ffmpeg.init(); +procedure TVideoPlayback_ffmpeg.Init(); begin - glGenTextures(1, PglUint(@fVideoTex)); + glGenTextures(1, PGLuint(@fVideoTex)); end; +procedure TVideoPlayback_ffmpeg.Reset(); +begin + // close previously opened video + Close(); + + fVideoOpened := False; + fVideoPaused := False; + VideoTimeBase := 0; + VideoTime := 0; + VideoStream := nil; + VideoFormatContext := nil; + VideoCodecContext := nil; + VideoStreamIndex := -1; + + AVFrame := nil; + AVFrameRGB := nil; + FrameBuffer := nil; + + EOF := false; + + // TODO: do we really want this by default? + Loop := true; + fLoopTime := 0; +end; function TVideoPlayback_ffmpeg.Open(const aFileName : string): boolean; // true if succeed var -errnum {*i, x, y*}: Integer; // Auto Removed, Unused Variable (i) // Auto Removed, Unused Variable (x) // Auto Removed, Unused Variable (x) // Auto Removed, Unused Variable (x) // Auto Removed, Unused Variable (y) -// lStreamsCount : Integer; // Auto Removed, Unused Variable - - wanted_spec , -// spec : TSDL_AudioSpec; // Auto Removed, Unused Variable -// aCodec : pAVCodec; // Auto Removed, Unused Variable + errnum: Integer; + err: GLenum; + AudioStreamIndex: integer; -{*sws_dst_w, *}sws_dst_h: Integer; // Auto Removed, Unused Variable (sws_dst_w) + procedure CleanOnError(); + begin + if (VideoCodecContext <> nil) then + avcodec_close(VideoCodecContext); + if (VideoFormatContext <> nil) then + av_close_input_file(VideoFormatContext); + av_free(AVFrameRGB); + av_free(AVFrame); + av_free(FrameBuffer); + end; begin Result := false; - // close previously opened video - if (fVideoOpened) then - Close; + Reset(); - fVideoOpened := False; - fVideoPaused := False; - VideoTimeBase := 0; - VideoTime := 0; - LastFrameTime := 0; - TimeDifference := 0; - VideoFormatContext := nil; - -// debugwriteln( aFileName ); + errnum := av_open_input_file(VideoFormatContext, pchar( aFileName ), nil, 0, nil); + if (errnum <> 0) then + begin + Log.LogError('Failed to open file "'+aFileName+'" ('+FFMpegErrorString(errnum)+')'); + Exit; + end; - errnum := av_open_input_file(VideoFormatContext, pchar( aFileName ), Nil, 0, Nil); -// debugwriteln( 'Errnum : ' +inttostr( errnum )); - if(errnum <> 0) then + // update video info + if (av_find_stream_info(VideoFormatContext) < 0) then begin -{$ifdef DebugDisplay} - case errnum of - AVERROR_UNKNOWN: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_UNKNOWN'); - AVERROR_IO: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_IO'); - AVERROR_NUMEXPECTED: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_NUMEXPECTED'); - AVERROR_INVALIDDATA: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_INVALIDDATA'); - AVERROR_NOMEM: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_NOMEM'); - AVERROR_NOFMT: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_NOFMT'); - AVERROR_NOTSUPP: showmessage('failed to open file '+aFileName+#13#10+'AVERROR_NOTSUPP'); - else showmessage('failed to open file '+aFileName+#13#10+'Error number: '+inttostr(Errnum)); - end; -{$ENDIF} + Log.LogError('No stream info found', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); Exit; - end - else + end; + Log.LogInfo('VideoStreamIndex : ' + inttostr(VideoStreamIndex), 'TVideoPlayback_ffmpeg.Open'); + + // find video stream + FindStreamIDs(VideoFormatContext, VideoStreamIndex, AudioStreamIndex); + if (VideoStreamIndex < 0) then begin - VideoStreamIndex := -1; - AudioStreamIndex := -1; + Log.LogError('No video stream found', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; - // Find which stream contains the video - if( av_find_stream_info(VideoFormatContext) >= 0 ) then - begin - find_stream_ids( VideoFormatContext, VideoStreamIndex, AudioStreamIndex ); + VideoStream := VideoFormatContext^.streams[VideoStreamIndex]; + VideoCodecContext := VideoStream^.codec; - debugwriteln( 'VideoStreamIndex : ' + inttostr(VideoStreamIndex) ); - debugwriteln( 'AudioStreamIndex : ' + inttostr(AudioStreamIndex) ); - end; - // FIXME: AudioStreamIndex is -1 if video has no sound -> memory access error - // Just a temporary workaround for now - aCodecCtx := nil; - if( AudioStreamIndex >= 0) then - aCodecCtx := VideoFormatContext.streams[ AudioStreamIndex ].codec; - - (* FIXME - {$ifdef UseFFMpegAudio} - // This is the audio ffmpeg audio support Jay is working on. - if aCodecCtx <> nil then - begin - wanted_spec.freq := aCodecCtx.sample_rate; - wanted_spec.format := AUDIO_S16SYS; - wanted_spec.channels := aCodecCtx.channels; - wanted_spec.silence := 0; - wanted_spec.samples := SDL_AUDIO_BUFFER_SIZE; - wanted_spec.callback := UAudio_FFMpeg.audio_callback; - wanted_spec.userdata := aCodecCtx; + VideoCodec := avcodec_find_decoder(VideoCodecContext^.codec_id); + if (VideoCodec = nil) then + begin + Log.LogError('No matching codec found', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; + // set debug options + VideoCodecContext^.debug_mv := 0; + VideoCodecContext^.debug := 0; - if (SDL_OpenAudio(@wanted_spec, @spec) < 0) then - begin - debugwriteln('SDL_OpenAudio: '+SDL_GetError()); - exit; - end; + // detect bug-workarounds automatically + VideoCodecContext^.workaround_bugs := FF_BUG_AUTODETECT; + // error resilience strategy (careful/compliant/agressive/very_aggressive) + //VideoCodecContext^.error_resilience := FF_ER_CAREFUL; //FF_ER_COMPLIANT; + // allow non spec compliant speedup tricks. + //VideoCodecContext^.flags2 := VideoCodecContext^.flags2 or CODEC_FLAG2_FAST; - debugwriteln( 'SDL opened audio device' ); + errnum := avcodec_open(VideoCodecContext, VideoCodec); + if (errnum < 0) then + begin + Log.LogError('No matching codec found', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; - aCodec := avcodec_find_decoder(aCodecCtx.codec_id); - if (aCodec = nil) then - begin - debugwriteln('Unsupported codec!'); - exit; - end; + // register custom callbacks for pts-determination + VideoCodecContext^.get_buffer := PtsGetBuffer; + VideoCodecContext^.release_buffer := PtsReleaseBuffer; + + {$ifdef DebugDisplay} + DebugWriteln('Found a matching Codec: '+ VideoCodecContext^.Codec.Name + sLineBreak + + sLineBreak + + ' Width = '+inttostr(VideoCodecContext^.width) + + ', Height='+inttostr(VideoCodecContext^.height) + sLineBreak + + ' Aspect : '+inttostr(VideoCodecContext^.sample_aspect_ratio.num) + '/' + + inttostr(VideoCodecContext^.sample_aspect_ratio.den) + sLineBreak + + ' Framerate : '+inttostr(VideoCodecContext^.time_base.num) + '/' + + inttostr(VideoCodecContext^.time_base.den)); + {$endif} + + // allocate space for decoded frame and rgb frame + AVFrame := avcodec_alloc_frame(); + AVFrameRGB := avcodec_alloc_frame(); + FrameBuffer := av_malloc(avpicture_get_size(PIXEL_FMT_FFMPEG, + VideoCodecContext^.width, VideoCodecContext^.height)); + + if ((AVFrame = nil) or (AVFrameRGB = nil) or (FrameBuffer = nil)) then + begin + Log.LogError('Failed to allocate buffers', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; - avcodec_open(aCodecCtx, aCodec); + // TODO: pad data for OpenGL to GL_UNPACK_ALIGNMENT + // (otherwise video will be distorted if width/height is not a multiple of the alignment) + errnum := avpicture_fill(PAVPicture(AVFrameRGB), FrameBuffer, PIXEL_FMT_FFMPEG, + VideoCodecContext^.width, VideoCodecContext^.height); + if (errnum < 0) then + begin + Log.LogError('avpicture_fill failed: ' + FFMpegErrorString(errnum), 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; - debugwriteln( 'Opened the codec' ); + // calculate some information for video display + VideoAspect := av_q2d(VideoCodecContext^.sample_aspect_ratio); + if (VideoAspect = 0) then + VideoAspect := VideoCodecContext^.width / + VideoCodecContext^.height + else + VideoAspect := VideoAspect * VideoCodecContext^.width / + VideoCodecContext^.height; - packet_queue_init( audioq ); - SDL_PauseAudio(0); + ScaledVideoWidth := 800.0; + ScaledVideoHeight := 800.0 / VideoAspect; - debugwriteln( 'SDL_PauseAudio' ); + VideoTimeBase := 1/av_q2d(VideoStream^.r_frame_rate); + // hack to get reasonable timebase (for divx and others) + if (VideoTimeBase < 0.02) then // 0.02 <-> 50 fps + begin + VideoTimeBase := av_q2d(VideoStream^.r_frame_rate); + while (VideoTimeBase > 50) do + VideoTimeBase := VideoTimeBase/10; + VideoTimeBase := 1/VideoTimeBase; + end; - end; - {$endif} - *) + Log.LogInfo('VideoTimeBase: ' + floattostr(VideoTimeBase), 'TVideoPlayback_ffmpeg.Open'); + Log.LogInfo('Framerate: '+inttostr(floor(1/VideoTimeBase))+'fps', 'TVideoPlayback_ffmpeg.Open'); - if(VideoStreamIndex >= 0) then - begin - VideoCodecContext:=VideoFormatContext^.streams[VideoStreamIndex]^.codec; - VideoCodec:=avcodec_find_decoder(VideoCodecContext^.codec_id); - end - else - begin -{$ifdef DebugDisplay} - showmessage('found no video stream'); -{$ENDIF} - av_close_input_file(VideoFormatContext); - Exit; - end; + {$IFDEF UseSWScale} + // if available get a SWScale-context -> faster than the deprecated img_convert(). + // SWScale has accelerated support for PIX_FMT_RGB32/PIX_FMT_BGR24/PIX_FMT_BGR565/PIX_FMT_BGR555. + // Note: PIX_FMT_RGB32 is a BGR- and not an RGB-format (maybe a bug)!!! + // The BGR565-formats (GL_UNSIGNED_SHORT_5_6_5) is way too slow because of its + // bad OpenGL support. The BGR formats have MMX(2) implementations but no speed-up + // could be observed in comparison to the RGB versions. + SoftwareScaleContext := sws_getContext( + VideoCodecContext^.width, VideoCodecContext^.height, + integer(VideoCodecContext^.pix_fmt), + VideoCodecContext^.width, VideoCodecContext^.height, + integer(PIXEL_FMT_FFMPEG), + SWS_FAST_BILINEAR, nil, nil, nil); + if (SoftwareScaleContext = nil) then + begin + Log.LogError('Failed to get swscale context', 'TVideoPlayback_ffmpeg.Open'); + CleanOnError(); + Exit; + end; + {$ENDIF} - if(VideoCodec<>Nil) then - begin - errnum:=avcodec_open(VideoCodecContext, VideoCodec); - end else begin -{$ifdef DebugDisplay} - showmessage('no matching codec found'); -{$ENDIF} - avcodec_close(VideoCodecContext); - av_close_input_file(VideoFormatContext); - Exit; - end; - if(errnum >=0) then - begin - if (VideoCodecContext^.width >1024) or (VideoCodecContext^.height >1024) then - begin - ScreenPopupError.ShowPopup('Video dimensions\nmust not exceed\n1024 pixels\n\nvideo disabled'); //show error message - avcodec_close(VideoCodecContext); - av_close_input_file(VideoFormatContext); - Exit; - end; -{$ifdef DebugDisplay} - showmessage('Found a matching Codec: '+ VideoCodecContext^.Codec.Name +#13#10#13#10+ - ' Width = '+inttostr(VideoCodecContext^.width)+ ', Height='+inttostr(VideoCodecContext^.height)+#13#10+ - ' Aspect : '+inttostr(VideoCodecContext^.sample_aspect_ratio.num)+'/'+inttostr(VideoCodecContext^.sample_aspect_ratio.den)+#13#10+ - ' Framerate : '+inttostr(VideoCodecContext^.time_base.num)+'/'+inttostr(VideoCodecContext^.time_base.den)); -{$endif} - // allocate space for decoded frame and rgb frame - AVFrame:=avcodec_alloc_frame; - AVFrameRGB:=avcodec_alloc_frame; - end; + TexWidth := Round(Power(2, Ceil(Log2(VideoCodecContext^.width)))); + TexHeight := Round(Power(2, Ceil(Log2(VideoCodecContext^.height)))); - dataX := Round(Power(2, Ceil(Log2(VideoCodecContext^.width)))); - dataY := Round(Power(2, Ceil(Log2(VideoCodecContext^.height)))); - myBuffer:=Nil; - if(AVFrame <> Nil) and (AVFrameRGB <> Nil) then - begin - myBuffer:=av_malloc(avpicture_get_size(PIX_FMT_RGB24, dataX, dataY)); - end; - if myBuffer <> Nil then errnum:=avpicture_fill(PAVPicture(AVFrameRGB), myBuffer, PIX_FMT_RGB24, - dataX, dataY) - else begin - {$ifdef DebugDisplay} - showmessage('failed to allocate video buffer'); - {$endif} - av_free(AVFrameRGB); - av_free(AVFrame); - avcodec_close(VideoCodecContext); - av_close_input_file(VideoFormatContext); - Exit; - end; + // we retrieve a texture just once with glTexImage2D and update it with glTexSubImage2D later. + // Benefits: glTexSubImage2D is faster and supports non-power-of-two widths/height. + glBindTexture(GL_TEXTURE_2D, fVideoTex); + glTexEnvi(GL_TEXTURE_2D, GL_TEXTURE_ENV_MODE, GL_REPLACE); + glTexImage2D(GL_TEXTURE_2D, 0, 3, TexWidth, TexHeight, 0, + PIXEL_FMT_OPENGL, GL_UNSIGNED_BYTE, nil); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - // this is the errnum from avpicture_fill - if errnum >=0 then - begin - fVideoOpened:=True; - - TexX := VideoCodecContext^.width; - TexY := VideoCodecContext^.height; - dataX := Round(Power(2, Ceil(Log2(TexX)))); - dataY := Round(Power(2, Ceil(Log2(TexY)))); - // calculate some information for video display - VideoAspect:=VideoCodecContext^.sample_aspect_ratio.num/VideoCodecContext^.sample_aspect_ratio.den; - if (VideoAspect = 0) then - VideoAspect:=VideoCodecContext^.width/VideoCodecContext^.height - else - VideoAspect:=VideoAspect*VideoCodecContext^.width/VideoCodecContext^.height; - ScaledVideoWidth:=800.0; - ScaledVideoHeight:=800.0/VideoAspect; - VideoTimeBase:=VideoFormatContext^.streams[VideoStreamIndex]^.r_frame_rate.den/VideoFormatContext^.streams[VideoStreamIndex]^.r_frame_rate.num; - debugwriteln( floattostr(VideoTimeBase) ); -{$ifdef DebugDisplay} - showmessage('framerate: '+inttostr(floor(1/videotimebase))+'fps'); - {$endif} - - {$IFDEF UseSWScale} - - // what the hell it should do? - SoftwareScaleContext:=sws_getContext(VideoCodecContext^.width,VideoCodecContext^.height,integer(VideoCodecContext^.pix_fmt), - TexX, TexY, integer(PIX_FMT_RGB24), - SWS_FAST_BILINEAR, nil, nil, nil); - if SoftwareScaleContext <> Nil then - debugwriteln('got swscale context') - else begin - debugwriteln('ERROR: didn''t get swscale context'); - av_free(AVFrameRGB); - av_free(AVFrame); - avcodec_close(VideoCodecContext); - av_close_input_file(VideoFormatContext); - Exit; - end; - {$ENDIF} - // hack to get reasonable timebase (for divx and others) - if VideoTimeBase < 0.02 then // 0.02 <-> 50 fps - begin - VideoTimeBase:=VideoFormatContext^.streams[VideoStreamIndex]^.r_frame_rate.num/VideoFormatContext^.streams[VideoStreamIndex]^.r_frame_rate.den; - while VideoTimeBase > 50 do VideoTimeBase:=VideoTimeBase/10; - VideoTimeBase:=1/VideoTimeBase; - end; - end; - end; + fVideoOpened := True; Result := true; end; @@ -652,14 +780,14 @@ procedure TVideoPlayback_ffmpeg.Close; begin if fVideoOpened then begin - av_free(myBuffer); + av_free(FrameBuffer); av_free(AVFrameRGB); av_free(AVFrame); avcodec_close(VideoCodecContext); av_close_input_file(VideoFormatContext); - fVideoOpened:=False; + fVideoOpened := False; end; end; @@ -677,19 +805,33 @@ begin end; procedure TVideoPlayback_ffmpeg.SetPosition(Time: real); +var + SeekFlags: integer; begin - fVideoSkipTime := Time; + if (Time < 0) then + Time := 0; + + // TODO: handle loop-times + //Time := Time mod VideoDuration; + + // backward seeking might fail without AVSEEK_FLAG_BACKWARD + SeekFlags := AVSEEK_FLAG_ANY; + if (Time < VideoTime) then + SeekFlags := SeekFlags or AVSEEK_FLAG_BACKWARD; + + VideoTime := Time; + EOF := false; - if fVideoSkipTime > 0 then + if (av_seek_frame(VideoFormatContext, VideoStreamIndex, Floor(Time/VideoTimeBase), SeekFlags) < 0) then begin - av_seek_frame(VideoFormatContext,VideoStreamIndex,Floor(Time/VideoTimeBase),AVSEEK_FLAG_ANY); + Log.LogError('av_seek_frame() failed', 'TVideoPlayback_ffmpeg.SetPosition'); end; end; -// what is this supposed to do? return VideoTime? function TVideoPlayback_ffmpeg.GetPosition: real; begin - result := 0; + // TODO: return video-position in seconds + result := VideoTime; end; initialization -- cgit v1.2.3