experimental ffmpeg videodecoding support, frameskipping doesn't work quite right yet

git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/trunk@332 b956fd51-792f-4845-bead-9b4dfca2ff2c

3 files changed, 292 insertions, 36 deletions

diff --git a/Game/Code/SMpeg/Uffmpeg.pas b/Game/Code/SMpeg/Uffmpeg.pas
new file mode 100644
index 00000000..3d21f33c
--- /dev/null
+++ b/Game/Code/SMpeg/Uffmpeg.pas
@@ -0,0 +1,265 @@
+unit Uffmpeg;

+

+interface

+uses SDL, UGraphicClasses, textgl, avcodec, avformat, avutil, math, OpenGL12, SysUtils, UIni, dialogs;

+

+procedure Init;

+procedure FFmpegOpenFile(FileName: pAnsiChar);

+procedure FFmpegClose;

+procedure FFmpegGetFrame(Time: Extended);

+procedure FFmpegDrawGL;

+procedure FFmpegTogglePause;

+procedure FFmpegSkip(Time: Single);

+

+var

+  VideoOpened, VideoPaused: Boolean;

+  VideoFormatContext: PAVFormatContext;

+  VideoStreamIndex: Integer;

+  VideoCodecContext: PAVCodecContext;

+  VideoCodec: PAVCodec;

+  AVFrame: PAVFrame;

+  AVFrameRGB: PAVFrame;

+  myBuffer: pByte;

+  VideoTex: glUint;

+  TexX, TexY, dataX, dataY: Cardinal;

+  TexData: array of Byte;

+  ScaledVideoWidth, ScaledVideoHeight: Real;

+  VideoAspect: Real;

+  VideoTextureU, VideoTextureV: Real;

+  VideoTimeBase, VideoTime, LastFrameTime, TimeDifference: Extended;

+  VideoSkipTime: Single;

+

+implementation

+

+const DebugDisplay=True;

+

+procedure Init;

+begin

+  av_register_all;

+  VideoOpened:=False;

+  VideoPaused:=False;

+  glGenTextures(1, PglUint(&VideoTex));

+  SetLength(TexData,0);

+end;

+

+procedure FFmpegOpenFile(FileName: pAnsiChar);

+var errnum, i, x,y: Integer;

+begin

+  VideoOpened:=False;

+  VideoPaused:=False;

+  VideoTimeBase:=0;

+  VideoTime:=0;

+  LastFrameTime:=0;

+  TimeDifference:=0;

+  errnum:=av_open_input_file(VideoFormatContext, FileName, Nil, 0, Nil);

+  if(errnum <> 0)

+  then begin

+    case errnum of

+      AVERROR_UNKNOWN: showmessage('failed to open file '+Filename+#13#10+'AVERROR_UNKNOWN');

+      AVERROR_IO: showmessage('failed to open file '+Filename+#13#10+'AVERROR_IO');

+      AVERROR_NUMEXPECTED: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NUMEXPECTED');

+      AVERROR_INVALIDDATA: showmessage('failed to open file '+Filename+#13#10+'AVERROR_INVALIDDATA');

+      AVERROR_NOMEM: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOMEM');

+      AVERROR_NOFMT: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOFMT');

+      AVERROR_NOTSUPP: showmessage('failed to open file '+Filename+#13#10+'AVERROR_NOTSUPP');

+    else showmessage('failed to open file '+Filename+#13#10+'Error number: '+inttostr(Errnum));

+    end;

+    Exit;

+  end

+  else begin

+    VideoStreamIndex:=-1;

+    if(av_find_stream_info(VideoFormatContext)>=0) then

+    begin

+      for i:=0 to VideoFormatContext^.nb_streams-1 do

+        if(VideoFormatContext^.streams[i]^.codec^.codec_type=CODEC_TYPE_VIDEO) then begin

+          VideoStreamIndex:=i;

+        end else

+    end;

+    if(VideoStreamIndex >= 0) then

+    begin

+      VideoCodecContext:=VideoFormatContext^.streams[VideoStreamIndex]^.codec;

+      VideoCodec:=avcodec_find_decoder(VideoCodecContext^.codec_id);

+    end else showmessage('found no video stream');

+    if(VideoCodec<>Nil) then

+    begin

+      errnum:=avcodec_open(VideoCodecContext, VideoCodec);

+    end else showmessage('no matching codec found');

+    if(errnum >=0) then

+    begin

+      showmessage('Found a matching Codec:'+#13#10#13#10+

+           'Width='+inttostr(VideoCodecContext^.width)+

+        ', Height='+inttostr(VideoCodecContext^.height)+#13#10+

+        'Aspect: '+inttostr(VideoCodecContext^.sample_aspect_ratio.num)+'/'+inttostr(VideoCodecContext^.sample_aspect_ratio.den)+#13#10+

+        'Framerate: '+inttostr(VideoCodecContext^.time_base.num)+'/'+inttostr(VideoCodecContext^.time_base.den));

+      // allocate space for decoded frame and rgb frame

+      AVFrame:=avcodec_alloc_frame;

+      AVFrameRGB:=avcodec_alloc_frame;

+    end;

+    myBuffer:=Nil;

+    if(AVFrame <> Nil) and (AVFrameRGB <> Nil) then

+    begin

+      myBuffer:=av_malloc(avpicture_get_size(PIX_FMT_RGB24, VideoCodecContext^.width,

+                            VideoCodecContext^.height));

+    end;

+    if myBuffer <> Nil then errnum:=avpicture_fill(PAVPicture(AVFrameRGB), myBuffer, PIX_FMT_RGB24,

+                VideoCodecContext^.width, VideoCodecContext^.height)

+    else showmessage('failed to allocate video buffer');

+    if errnum >=0 then

+    begin

+      VideoOpened:=True;

+

+      TexX := VideoCodecContext^.width;

+      TexY := VideoCodecContext^.height;

+      dataX := Round(Power(2, Ceil(Log2(TexX))));

+      dataY := Round(Power(2, Ceil(Log2(TexY))));

+      SetLength(TexData,dataX*dataY*3);

+      // calculate some information for video display

+      VideoAspect:=VideoCodecContext^.sample_aspect_ratio.num/VideoCodecContext^.sample_aspect_ratio.den;

+      if (VideoAspect = 0) then

+        VideoAspect:=VideoCodecContext^.width/VideoCodecContext^.height

+      else

+        VideoAspect:=VideoAspect*VideoCodecContext^.width/VideoCodecContext^.height;

+      if VideoAspect >= 4/3 then

+      begin

+        ScaledVideoWidth:=800.0;

+        ScaledVideoHeight:=800.0/VideoAspect;

+      end else

+      begin

+        ScaledVideoHeight:=600.0;

+        ScaledVideoWidth:=600.0*VideoAspect;

+      end;

+      VideoTimeBase:=VideoCodecContext^.time_base.num/VideoCodecContext^.time_base.den;

+      if (VideoAspect*VideoCodecContext^.width*VideoCodecContext^.height)>200000 then

+        showmessage('you are trying to play a rather large video'+#13#10+

+                    'be prepared to experience some timing problems');

+    end;

+  end;

+end;

+

+procedure FFmpegClose;

+begin

+  if VideoOpened then begin

+    av_free(myBuffer);

+    av_free(AVFrameRGB);

+    av_free(AVFrame);

+    avcodec_close(VideoCodecContext);

+    av_close_input_file(VideoFormatContext);

+    SetLength(TexData,0);

+    VideoOpened:=False;

+  end;

+end;

+

+procedure FFmpegTogglePause;

+begin

+  if VideoPaused then VideoPaused:=False

+  else VideoPaused:=True;

+end;

+

+procedure FFmpegSkip(Time: Single);

+begin

+  VideoSkiptime:=Time;

+end;

+

+procedure FFmpegGetFrame(Time: Extended);

+var

+  FrameFinished: Integer;

+  AVPacket: TAVPacket;

+  errnum, x, y: Integer;

+  FrameDataPtr: PByteArray;

+  linesize: integer;

+  myTime: Extended;

+begin

+  if not VideoOpened then Exit;

+  if VideoPaused then Exit;

+  myTime:=Time+VideoSkipTime;

+  TimeDifference:=myTime-VideoTime;

+{  showmessage('Time:      '+inttostr(floor(Time*1000))+#13#10+

+    'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+

+    'TimeBase:  '+inttostr(floor(VideoTimeBase*1000))+#13#10+

+    'TimeDiff:  '+inttostr(floor(TimeDifference*1000)));

+}

+  if (VideoTime <> 0) and (TimeDifference <= VideoTimeBase) then begin

+    if DebugDisplay then GoldenRec.Spawn(200,15,1,16,0,-1,ColoredStar,$00ff00);

+{    showmessage('not getting new frame'+#13#10+

+    'Time:      '+inttostr(floor(Time*1000))+#13#10+

+    'VideoTime: '+inttostr(floor(VideoTime*1000))+#13#10+

+    'TimeBase:  '+inttostr(floor(VideoTimeBase*1000))+#13#10+

+    'TimeDiff:  '+inttostr(floor(TimeDifference*1000)));

+}

+    Exit;// we don't need a new frame now

+  end;

+  VideoTime:=VideoTime+VideoTimeBase;

+  TimeDifference:=myTime-VideoTime;

+  if TimeDifference >= 3*VideoTimeBase then begin // skip frames

+    if DebugDisplay then GoldenRec.Spawn(200,35,1,16,0,-1,ColoredStar,$ff0000);

+//    showmessage('skipping frames'+#13#10+

+//    'TimeBase:  '+inttostr(floor(VideoTimeBase*1000))+#13#10+

+//    'TimeDiff:  '+inttostr(floor(TimeDifference*1000))+#13#10+

+//    'Time2Skip: '+inttostr(floor((Time-LastFrameTime)*1000)));

+//    av_seek_frame(VideoFormatContext,VideoStreamIndex,Floor(Time*VideoTimeBase),0);

+    av_seek_frame(VideoFormatContext,-1,Floor((myTime)*1100000),0);

+    VideoTime:=floor(myTime/VideoTimeBase)*VideoTimeBase;

+  end;

+

+  FrameFinished:=0;

+  // read packets until we have a finished frame (or there are no more packets)

+  while (FrameFinished=0) and (av_read_frame(VideoFormatContext, @AVPacket)>=0) do

+  begin

+    // if we got a packet from the video stream, then decode it

+    if (AVPacket.stream_index=VideoStreamIndex) then

+      errnum:=avcodec_decode_video(VideoCodecContext, AVFrame, @frameFinished,

+                                   AVPacket.data, AVPacket.size);

+    // release internal packet structure created by av_read_frame

+    av_free_packet(PAVPacket(@AVPacket));

+  end;

+  // if we did not get an new frame, there's nothing more to do

+  if Framefinished=0 then Exit;

+  // otherwise we convert the pixeldata from YUV to RGB

+  errnum:=img_convert(PAVPicture(AVFrameRGB), PIX_FMT_RGB24,

+            PAVPicture(AVFrame), VideoCodecContext^.pix_fmt,

+			      VideoCodecContext^.width, VideoCodecContext^.height);

+  if errnum >=0 then begin

+    // copy RGB pixeldata to our TextureBuffer

+    // (line by line)

+    FrameDataPtr:=AVFrameRGB^.data[0];

+    linesize:=AVFrameRGB^.linesize[0];

+    for y:=0 to TexY-1 do begin

+      System.Move(FrameDataPtr[y*linesize],TexData[3*y*dataX],linesize);

+    end;

+

+    // generate opengl texture out of whatever we got

+    glBindTexture(GL_TEXTURE_2D, VideoTex);

+    glTexImage2D(GL_TEXTURE_2D, 0, 3, dataX, dataY, 0, GL_RGB, GL_UNSIGNED_BYTE, TexData);

+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

+  end;

+end;

+

+procedure FFmpegDrawGL;

+begin

+  if not VideoOpened then Exit;

+  glEnable(GL_TEXTURE_2D);

+  glEnable(GL_BLEND);

+  glColor4f(1, 1, 1, 1);

+  glBindTexture(GL_TEXTURE_2D, VideoTex);

+  glbegin(gl_quads);

+    glTexCoord2f(         0,          0); glVertex2f(400-ScaledVideoWidth/2, 300-ScaledVideoHeight/2);

+    glTexCoord2f(         0, TexY/dataY); glVertex2f(400-ScaledVideoWidth/2, 300+ScaledVideoHeight/2);

+    glTexCoord2f(TexX/dataX, TexY/dataY); glVertex2f(400+ScaledVideoWidth/2, 300+ScaledVideoHeight/2);

+    glTexCoord2f(TexX/dataX,          0); glVertex2f(400+ScaledVideoWidth/2, 300-ScaledVideoHeight/2);

+  glEnd;

+  glDisable(GL_TEXTURE_2D);

+  glDisable(GL_BLEND);

+

+  if DebugDisplay then begin

+    SetFontStyle (2);

+    SetFontItalic(False);

+    SetFontSize(9);

+    SetFontPos (5, 0);

+    glPrint('delaying frame');

+    SetFontPos (5, 20);

+    glPrint('dropping frame');

+  end;

+end;

+

+end.

diff --git a/Game/Code/Screens/UScreenSing.pas b/Game/Code/Screens/UScreenSing.pas
index 2ba7522f..4bf405f6 100644
--- a/Game/Code/Screens/UScreenSing.pas
+++ b/Game/Code/Screens/UScreenSing.pas
@@ -3,7 +3,7 @@ unit UScreenSing;
 interface

 

 uses UMenu, UMusic, SDL, SysUtils, UFiles, UTime, USongs, UIni, ULog, USmpeg, UTexture, ULyrics,

-  TextGL, OpenGL12, BASS, UThemes, ULCD, UGraphicClasses;

+  TextGL, OpenGL12, BASS, UThemes, ULCD, UGraphicClasses, Uffmpeg;

 

 type

   TScreenSing = class(TMenu)

@@ -145,7 +145,7 @@ begin
     //stop Music

     Music.Pause;

     if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then //Video

-      PauseSmpeg; //Video

+      FFmpegTogglePause;//PauseSmpeg; //Video

   end

   else            //Pause ausschalten

   begin

@@ -153,7 +153,7 @@ begin
     Music.MoveTo (PauseTime);//Position of Music

     Music.Play; //Play Music

     if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then //Video

-      PlaySmpeg;

+      FFmpegTogglePause;//PlaySmpeg;

       //SkipSmpeg(PauseTime);

     Paused := false;

   end;

@@ -222,6 +222,7 @@ begin
 

   LyricMain := TLyric.Create;

   LyricSub :=  TLyric.Create;

+  Uffmpeg.Init;

 end;

 

 procedure TScreenSing.onShow;

@@ -349,8 +350,11 @@ begin
 

   // set movie

   if (AktSong.Video <> '') and FileExists(AktSong.Path + AktSong.Video) then begin

-    OpenSmpeg(AktSong.Path + AktSong.Video);

-    SkipSmpeg(AktSong.VideoGAP + AktSong.Start);

+{    OpenSmpeg(AktSong.Path + AktSong.Video);

+    SkipSmpeg(AktSong.VideoGAP + AktSong.Start);}

+    // todo: VideoGap and Start time verwursten

+    FFmpegOpenFile(pAnsiChar(AktSong.Path + AktSong.Video));

+    FFmpegSkip(AktSong.VideoGAP + AktSong.Start);

     AktSong.VideoLoaded := true;

   end;

 

@@ -739,14 +743,17 @@ begin
   if AktSong.VideoLoaded then

   begin

     try

-      PlaySmpeg;

+      FFmpegGetFrame(Czas.Teraz);

+      FFmpegDrawGL;

+//      PlaySmpeg;

     except

       //If an Error occurs Reading Video: prevent Video from being Drawn again and Close Video

       AktSong.VideoLoaded := False;

       Log.LogError('Error drawing Video, Video has been disabled for this Song/Session.');

       Log.LogError('Corrupted File: ' + AktSong.Video);

       try

-        CloseSmpeg;

+//        CloseSmpeg;

+        FFmpegClose;

       except

 

       end;

@@ -1061,14 +1068,19 @@ begin
   // update and draw movie

   if ShowFinish and AktSong.VideoLoaded then begin

     try

-      UpdateSmpeg; // this only draws

+//      UpdateSmpeg; // this only draws

+      // todo: find a way to determine, when a new frame is needed

+      // toto: same for the need to skip frames

+      FFmpegGetFrame(Czas.Teraz);

+      FFmpegDrawGL;

     except

       //If an Error occurs drawing: prevent Video from being Drawn again and Close Video

       AktSong.VideoLoaded := False;

       log.LogError('Error drawing Video, Video has been disabled for this Song/Session.');

       Log.LogError('Corrupted File: ' + AktSong.Video);

       try

-        CloseSmpeg;

+//        CloseSmpeg;

+        FFmpegClose;

       except

 

       end;

@@ -1095,7 +1107,7 @@ begin
     end;

   end;

   end;

-  

+

   // draw custom items

   SingDraw;  // always draw

 

@@ -1143,7 +1155,8 @@ begin
   end;

 

   if AktSong.VideoLoaded then begin

-    CloseSmpeg;

+//    CloseSmpeg;

+    FFmpegClose;

     AktSong.VideoLoaded := false; // to prevent drawing closed video

   end;

 

diff --git a/Game/Code/UltraStar.dpr b/Game/Code/UltraStar.dpr
index a49d43c2..e523e98e 100644
--- a/Game/Code/UltraStar.dpr
+++ b/Game/Code/UltraStar.dpr
@@ -5,9 +5,6 @@ program UltraStar;
 {$R 'UltraStar.res' 'UltraStar.rc'}

 

 uses

-  //------------------------------

-  //Includes - Menu System

-  //------------------------------

   UDisplay in 'Menu\UDisplay.pas',

   UMenu in 'Menu\UMenu.pas',

   UMenuStatic in 'Menu\UMenuStatic.pas',

@@ -18,10 +15,6 @@ uses
   UMenuSelectSlide in 'Menu\UMenuSelectSlide.pas',

   UDrawTexture in 'Menu\UDrawTexture.pas',

   UMenuButtonCollection in 'Menu\UMenuButtonCollection.pas',

-

-  //------------------------------

-  //Includes - Classes

-  //------------------------------

   UGraphic in 'Classes\UGraphic.pas',

   UTexture in 'Classes\UTexture.pas',

   UMusic in 'Classes\UMusic.pas',

@@ -49,11 +42,7 @@ uses
   UDLLManager in 'Classes\UDLLManager.pas',

   UParty in 'Classes\UParty.pas',

   UPlaylist in 'Classes\UPlaylist.pas',

-  UCommandLine  in 'Classes\UCommandLine.pas',

-

-  //------------------------------

-  //Includes - Screens

-  //------------------------------

+  UCommandLine in 'Classes\UCommandLine.pas',

   UScreenLoading in 'Screens\UScreenLoading.pas',

   UScreenWelcome in 'Screens\UScreenWelcome.pas',

   UScreenMain in 'Screens\UScreenMain.pas',

@@ -82,28 +71,17 @@ uses
   UScreenStatDetail in 'Screens\UScreenStatDetail.pas',

   UScreenCredits in 'Screens\UScreenCredits.pas',

   UScreenPopup in 'Screens\UScreenPopup.pas',

-

-  //------------------------------

-  //Includes - Screens PartyMode

-  //------------------------------

   UScreenSingModi in 'Screens\UScreenSingModi.pas',

   UScreenPartyNewRound in 'Screens\UScreenPartyNewRound.pas',

   UScreenPartyScore in 'Screens\UScreenPartyScore.pas',

   UScreenPartyPlayer in 'Screens\UScreenPartyPlayer.pas',

   UScreenPartyOptions in 'Screens\UScreenPartyOptions.pas',

   UScreenPartyWin in 'Screens\UScreenPartyWin.pas',

-

-  //------------------------------

-  //Includes - Modi SDK

-  //------------------------------

   ModiSDK in '..\..\Modis\SDK\ModiSDK.pas',

-

-  //------------------------------

-  //Includes - Delphi

-  //------------------------------

   Windows,

   SDL,

-  SysUtils;

+  SysUtils,

+  Uffmpeg in 'SMpeg\Uffmpeg.pas';

 

 const

   Version = 'UltraStar Deluxe V 1.00 RC1';