From b7f377e62a7e941d7898c187a63dd6ed99b0798f Mon Sep 17 00:00:00 2001 From: b1indy Date: Sat, 28 Jul 2007 13:29:32 +0000 Subject: experimental ffmpeg videodecoding support, frameskipping doesn't work quite right yet git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/trunk@332 b956fd51-792f-4845-bead-9b4dfca2ff2c --- Game/Code/SMpeg/Uffmpeg.pas | 265 ++++++++++++++++++++++++++++++++++++++ Game/Code/Screens/UScreenSing.pas | 35 +++-- Game/Code/UltraStar.dpr | 28 +--- 3 files changed, 292 insertions(+), 36 deletions(-) create mode 100644 Game/Code/SMpeg/Uffmpeg.pas (limited to 'Game') diff --git a/Game/Code/SMpeg/Uffmpeg.pas b/Game/Code/SMpeg/Uffmpeg.pas new file mode 100644 index 00000000..3d21f33c --- /dev/null +++ b/Game/Code/SMpeg/Uffmpeg.pas @@ -0,0 +1,265 @@ +unit Uffmpeg; + +interface +uses SDL, UGraphicClasses, textgl, avcodec, avformat, avutil, math, OpenGL12, SysUtils, UIni, dialogs; + +procedure Init; +procedure FFmpegOpenFile(FileName: pAnsiChar); +procedure FFmpegClose; +procedure FFmpegGetFrame(Time: Extended); +procedure FFmpegDrawGL; +procedure FFmpegTogglePause; +procedure FFmpegSkip(Time: Single); + +var + VideoOpened, VideoPaused: Boolean; + VideoFormatContext: PAVFormatContext; + VideoStreamIndex: Integer; + VideoCodecContext: PAVCodecContext; + VideoCodec: PAVCodec; + AVFrame: PAVFrame; + AVFrameRGB: PAVFrame; + myBuffer: pByte; + VideoTex: glUint; + TexX, TexY, dataX, dataY: Cardinal; + TexData: array of Byte; + ScaledVideoWidth, ScaledVideoHeight: Real; + VideoAspect: Real; + VideoTextureU, VideoTextureV: Real; + VideoTimeBase, VideoTime, LastFrameTime, TimeDifference: Extended; + VideoSkipTime: Single; + +implementation + +const DebugDisplay=True; + +procedure Init; +begin + av_register_all; + VideoOpened:=False; + VideoPaused:=False; + glGenTextures(1, PglUint(&VideoTex)); + SetLength(TexData,0); +end; + +procedure FFmpegOpenFile(FileName: pAnsiChar); +var errnum, i, x,y: Integer; +begin + VideoOpened:=False; + VideoPaused:=False; + VideoTimeBase:=0; + VideoTime:=0; + LastFrameTime:=0; + TimeDifference:=0; + errnum:=av_open_input_file(VideoFormatContext, FileName, Nil, 0, Nil); + if(errnum <> 0) + then begin + case errnum of + AVERROR_UNKNOWN: showmessage('failed to open file '+Filename+#13#10+'AVERROR_UNKNOWN'); + AVERROR_IO: showmessage('failed to open file '+Filename+#13#10+'AVERROR_IO'); + AVERROR_NUMEXPECTED: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NUMEXPECTED'); + AVERROR_INVALIDDATA: showmessage('failed to open file '+Filename+#13#10+'AVERROR_INVALIDDATA'); + AVERROR_NOMEM: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOMEM'); + AVERROR_NOFMT: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOFMT'); + AVERROR_NOTSUPP: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOTSUPP'); + else showmessage('failed to open file '+Filename+#13#10+'Error number: '+inttostr(Errnum)); + end; + Exit; + end + else begin + VideoStreamIndex:=-1; + if(av_find_stream_info(VideoFormatContext)>=0) then + begin + for i:=0 to VideoFormatContext^.nb_streams-1 do + if(VideoFormatContext^.streams[i]^.codec^.codec_type=CODEC_TYPE_VIDEO) then begin + VideoStreamIndex:=i; + end else + end; + if(VideoStreamIndex >= 0) then + begin + VideoCodecContext:=VideoFormatContext^.streams[VideoStreamIndex]^.codec; + VideoCodec:=avcodec_find_decoder(VideoCodecContext^.codec_id); + end else showmessage('found no video stream'); + if(VideoCodec<>Nil) then + begin + errnum:=avcodec_open(VideoCodecContext, VideoCodec); + end else showmessage('no matching codec found'); + if(errnum >=0) then + begin + showmessage('Found a matching Codec:'+#13#10#13#10+ + 'Width='+inttostr(VideoCodecContext^.width)+ + ', Height='+inttostr(VideoCodecContext^.height)+#13#10+ + 'Aspect: '+inttostr(VideoCodecContext^.sample_aspect_ratio.num)+'/'+inttostr(VideoCodecContext^.sample_aspect_ratio.den)+#13#10+ + 'Framerate: '+inttostr(VideoCodecContext^.time_base.num)+'/'+inttostr(VideoCodecContext^.time_base.den)); + // allocate space for decoded frame and rgb frame + AVFrame:=avcodec_alloc_frame; + AVFrameRGB:=avcodec_alloc_frame; + end; + myBuffer:=Nil; + if(AVFrame <> Nil) and (AVFrameRGB <> Nil) then + begin + myBuffer:=av_malloc(avpicture_get_size(PIX_FMT_RGB24, VideoCodecContext^.width, + VideoCodecContext^.height)); + end; + if myBuffer <> Nil then errnum:=avpicture_fill(PAVPicture(AVFrameRGB), myBuffer, PIX_FMT_RGB24, + VideoCodecContext^.width, VideoCodecContext^.height) + else showmessage('failed to allocate video buffer'); + if errnum >=0 then + begin + VideoOpened:=True; + + TexX := VideoCodecContext^.width; + TexY := VideoCodecContext^.height; + dataX := Round(Power(2, Ceil(Log2(TexX)))); + dataY := Round(Power(2, Ceil(Log2(TexY)))); + SetLength(TexData,dataX*dataY*3); + // calculate some information for video display + VideoAspect:=VideoCodecContext^.sample_aspect_ratio.num/VideoCodecContext^.sample_aspect_ratio.den; + if (VideoAspect = 0) then + VideoAspect:=VideoCodecContext^.width/VideoCodecContext^.height + else + VideoAspect:=VideoAspect*VideoCodecContext^.width/VideoCodecContext^.height; + if VideoAspect >= 4/3 then + begin + ScaledVideoWidth:=800.0; + ScaledVideoHeight:=800.0/VideoAspect; + end else + begin + ScaledVideoHeight:=600.0; + ScaledVideoWidth:=600.0*VideoAspect; + end; + VideoTimeBase:=VideoCodecContext^.time_base.num/VideoCodecContext^.time_base.den; + if (VideoAspect*VideoCodecContext^.width*VideoCodecContext^.height)>200000 then + showmessage('you are trying to play a rather large video'+#13#10+ + 'be prepared to experience some timing problems'); + end; + end; +end; + +procedure FFmpegClose; +begin + if VideoOpened then begin + av_free(myBuffer); + av_free(AVFrameRGB); + av_free(AVFrame); + avcodec_close(VideoCodecContext); + av_close_input_file(VideoFormatContext); + SetLength(TexData,0); + VideoOpened:=False; + end; +end; + +procedure FFmpegTogglePause; +begin + if VideoPaused then VideoPaused:=False + else VideoPaused:=True; +end; + +procedure FFmpegSkip(Time: Single); +begin + VideoSkiptime:=Time; +end; + +procedure FFmpegGetFrame(Time: Extended); +var + FrameFinished: Integer; + AVPacket: TAVPacket; + errnum, x, y: Integer; + FrameDataPtr: PByteArray; + linesize: integer; + myTime: Extended; +begin + if not VideoOpened then Exit; + if VideoPaused then Exit; + myTime:=Time+VideoSkipTime; + TimeDifference:=myTime-VideoTime; +{ showmessage('Time: '+inttostr(floor(Time*1000))+#13#10+ + 'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+ + 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ + 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); +} + if (VideoTime <> 0) and (TimeDifference <= VideoTimeBase) then begin + if DebugDisplay then GoldenRec.Spawn(200,15,1,16,0,-1,ColoredStar,$00ff00); +{ showmessage('not getting new frame'+#13#10+ + 'Time: '+inttostr(floor(Time*1000))+#13#10+ + 'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+ + 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ + 'TimeDiff: '+inttostr(floor(TimeDifference*1000))); +} + Exit;// we don't need a new frame now + end; + VideoTime:=VideoTime+VideoTimeBase; + TimeDifference:=myTime-VideoTime; + if TimeDifference >= 3*VideoTimeBase then begin // skip frames + if DebugDisplay then GoldenRec.Spawn(200,35,1,16,0,-1,ColoredStar,$ff0000); +// showmessage('skipping frames'+#13#10+ +// 'TimeBase: '+inttostr(floor(VideoTimeBase*1000))+#13#10+ +// 'TimeDiff: '+inttostr(floor(TimeDifference*1000))+#13#10+ +// 'Time2Skip: '+inttostr(floor((Time-LastFrameTime)*1000))); +// av_seek_frame(VideoFormatContext,VideoStreamIndex,Floor(Time*VideoTimeBase),0); + av_seek_frame(VideoFormatContext,-1,Floor((myTime)*1100000),0); + VideoTime:=floor(myTime/VideoTimeBase)*VideoTimeBase; + end; + + FrameFinished:=0; + // read packets until we have a finished frame (or there are no more packets) + while (FrameFinished=0) and (av_read_frame(VideoFormatContext, @AVPacket)>=0) do + begin + // if we got a packet from the video stream, then decode it + if (AVPacket.stream_index=VideoStreamIndex) then + errnum:=avcodec_decode_video(VideoCodecContext, AVFrame, @frameFinished, + AVPacket.data, AVPacket.size); + // release internal packet structure created by av_read_frame + av_free_packet(PAVPacket(@AVPacket)); + end; + // if we did not get an new frame, there's nothing more to do + if Framefinished=0 then Exit; + // otherwise we convert the pixeldata from YUV to RGB + errnum:=img_convert(PAVPicture(AVFrameRGB), PIX_FMT_RGB24, + PAVPicture(AVFrame), VideoCodecContext^.pix_fmt, + VideoCodecContext^.width, VideoCodecContext^.height); + if errnum >=0 then begin + // copy RGB pixeldata to our TextureBuffer + // (line by line) + FrameDataPtr:=AVFrameRGB^.data[0]; + linesize:=AVFrameRGB^.linesize[0]; + for y:=0 to TexY-1 do begin + System.Move(FrameDataPtr[y*linesize],TexData[3*y*dataX],linesize); + end; + + // generate opengl texture out of whatever we got + glBindTexture(GL_TEXTURE_2D, VideoTex); + glTexImage2D(GL_TEXTURE_2D, 0, 3, dataX, dataY, 0, GL_RGB, GL_UNSIGNED_BYTE, TexData); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + end; +end; + +procedure FFmpegDrawGL; +begin + if not VideoOpened then Exit; + glEnable(GL_TEXTURE_2D); + glEnable(GL_BLEND); + glColor4f(1, 1, 1, 1); + glBindTexture(GL_TEXTURE_2D, VideoTex); + glbegin(gl_quads); + glTexCoord2f( 0, 0); glVertex2f(400-ScaledVideoWidth/2, 300-ScaledVideoHeight/2); + glTexCoord2f( 0, TexY/dataY); glVertex2f(400-ScaledVideoWidth/2, 300+ScaledVideoHeight/2); + glTexCoord2f(TexX/dataX, TexY/dataY); glVertex2f(400+ScaledVideoWidth/2, 300+ScaledVideoHeight/2); + glTexCoord2f(TexX/dataX, 0); glVertex2f(400+ScaledVideoWidth/2, 300-ScaledVideoHeight/2); + glEnd; + glDisable(GL_TEXTURE_2D); + glDisable(GL_BLEND); + + if DebugDisplay then begin + SetFontStyle (2); + SetFontItalic(False); + SetFontSize(9); + SetFontPos (5, 0); + glPrint('delaying frame'); + SetFontPos (5, 20); + glPrint('dropping frame'); + end; +end; + +end. diff --git a/Game/Code/Screens/UScreenSing.pas b/Game/Code/Screens/UScreenSing.pas index 2ba7522f..4bf405f6 100644 --- a/Game/Code/Screens/UScreenSing.pas +++ b/Game/Code/Screens/UScreenSing.pas @@ -3,7 +3,7 @@ unit UScreenSing; interface uses UMenu, UMusic, SDL, SysUtils, UFiles, UTime, USongs, UIni, ULog, USmpeg, UTexture, ULyrics, - TextGL, OpenGL12, BASS, UThemes, ULCD, UGraphicClasses; + TextGL, OpenGL12, BASS, UThemes, ULCD, UGraphicClasses, Uffmpeg; type TScreenSing = class(TMenu) @@ -145,7 +145,7 @@ begin //stop Music Music.Pause; if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then //Video - PauseSmpeg; //Video + FFmpegTogglePause;//PauseSmpeg; //Video end else //Pause ausschalten begin @@ -153,7 +153,7 @@ begin Music.MoveTo (PauseTime);//Position of Music Music.Play; //Play Music if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then //Video - PlaySmpeg; + FFmpegTogglePause;//PlaySmpeg; //SkipSmpeg(PauseTime); Paused := false; end; @@ -222,6 +222,7 @@ begin LyricMain := TLyric.Create; LyricSub := TLyric.Create; + Uffmpeg.Init; end; procedure TScreenSing.onShow; @@ -349,8 +350,11 @@ begin // set movie if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then begin - OpenSmpeg(AktSong.Path + AktSong.Video); - SkipSmpeg(AktSong.VideoGAP + AktSong.Start); +{ OpenSmpeg(AktSong.Path + AktSong.Video); + SkipSmpeg(AktSong.VideoGAP + AktSong.Start);} + // todo: VideoGap and Start time verwursten + FFmpegOpenFile(pAnsiChar(AktSong.Path + AktSong.Video)); + FFmpegSkip(AktSong.VideoGAP + AktSong.Start); AktSong.VideoLoaded := true; end; @@ -739,14 +743,17 @@ begin if AktSong.VideoLoaded then begin try - PlaySmpeg; + FFmpegGetFrame(Czas.Teraz); + FFmpegDrawGL; +// PlaySmpeg; except //If an Error occurs Reading Video: prevent Video from being Drawn again and Close Video AktSong.VideoLoaded := False; Log.LogError('Error drawing Video, Video has been disabled for this Song/Session.'); Log.LogError('Corrupted File: ' + AktSong.Video); try - CloseSmpeg; +// CloseSmpeg; + FFmpegClose; except end; @@ -1061,14 +1068,19 @@ begin // update and draw movie if ShowFinish and AktSong.VideoLoaded then begin try - UpdateSmpeg; // this only draws +// UpdateSmpeg; // this only draws + // todo: find a way to determine, when a new frame is needed + // toto: same for the need to skip frames + FFmpegGetFrame(Czas.Teraz); + FFmpegDrawGL; except //If an Error occurs drawing: prevent Video from being Drawn again and Close Video AktSong.VideoLoaded := False; log.LogError('Error drawing Video, Video has been disabled for this Song/Session.'); Log.LogError('Corrupted File: ' + AktSong.Video); try - CloseSmpeg; +// CloseSmpeg; + FFmpegClose; except end; @@ -1095,7 +1107,7 @@ begin end; end; end; - + // draw custom items SingDraw; // always draw @@ -1143,7 +1155,8 @@ begin end; if AktSong.VideoLoaded then begin - CloseSmpeg; +// CloseSmpeg; + FFmpegClose; AktSong.VideoLoaded := false; // to prevent drawing closed video end; diff --git a/Game/Code/UltraStar.dpr b/Game/Code/UltraStar.dpr index a49d43c2..e523e98e 100644 --- a/Game/Code/UltraStar.dpr +++ b/Game/Code/UltraStar.dpr @@ -5,9 +5,6 @@ program UltraStar; {$R 'UltraStar.res' 'UltraStar.rc'} uses - //------------------------------ - //Includes - Menu System - //------------------------------ UDisplay in 'Menu\UDisplay.pas', UMenu in 'Menu\UMenu.pas', UMenuStatic in 'Menu\UMenuStatic.pas', @@ -18,10 +15,6 @@ uses UMenuSelectSlide in 'Menu\UMenuSelectSlide.pas', UDrawTexture in 'Menu\UDrawTexture.pas', UMenuButtonCollection in 'Menu\UMenuButtonCollection.pas', - - //------------------------------ - //Includes - Classes - //------------------------------ UGraphic in 'Classes\UGraphic.pas', UTexture in 'Classes\UTexture.pas', UMusic in 'Classes\UMusic.pas', @@ -49,11 +42,7 @@ uses UDLLManager in 'Classes\UDLLManager.pas', UParty in 'Classes\UParty.pas', UPlaylist in 'Classes\UPlaylist.pas', - UCommandLine in 'Classes\UCommandLine.pas', - - //------------------------------ - //Includes - Screens - //------------------------------ + UCommandLine in 'Classes\UCommandLine.pas', UScreenLoading in 'Screens\UScreenLoading.pas', UScreenWelcome in 'Screens\UScreenWelcome.pas', UScreenMain in 'Screens\UScreenMain.pas', @@ -82,28 +71,17 @@ uses UScreenStatDetail in 'Screens\UScreenStatDetail.pas', UScreenCredits in 'Screens\UScreenCredits.pas', UScreenPopup in 'Screens\UScreenPopup.pas', - - //------------------------------ - //Includes - Screens PartyMode - //------------------------------ UScreenSingModi in 'Screens\UScreenSingModi.pas', UScreenPartyNewRound in 'Screens\UScreenPartyNewRound.pas', UScreenPartyScore in 'Screens\UScreenPartyScore.pas', UScreenPartyPlayer in 'Screens\UScreenPartyPlayer.pas', UScreenPartyOptions in 'Screens\UScreenPartyOptions.pas', UScreenPartyWin in 'Screens\UScreenPartyWin.pas', - - //------------------------------ - //Includes - Modi SDK - //------------------------------ ModiSDK in '..\..\Modis\SDK\ModiSDK.pas', - - //------------------------------ - //Includes - Delphi - //------------------------------ Windows, SDL, - SysUtils; + SysUtils, + Uffmpeg in 'SMpeg\Uffmpeg.pas'; const Version = 'UltraStar Deluxe V 1.00 RC1'; -- cgit v1.2.3