From 1822c266c552a76cdbe75288515b5abfa9a41d5d Mon Sep 17 00:00:00 2001 From: tobigun Date: Wed, 22 Dec 2010 18:21:41 +0000 Subject: support for video pixel format selection git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/branches/experimental@2767 b956fd51-792f-4845-bead-9b4dfca2ff2c --- mediaplugin/src/base/UMusic.pas | 15 +++++++-- mediaplugin/src/media/UMediaPlugin.pas | 13 +++++++- mediaplugin/src/media/UMedia_dummy.pas | 4 +-- mediaplugin/src/media/UVideo.pas | 19 ++++++----- mediaplugin/src/media/UVideoDecoderPlugin.pas | 39 ++++++++++++++++++---- .../src/mediaplugins/ffmpeg/ffmpeg_core.cpp | 20 +++++++++++ mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.h | 1 + .../mediaplugins/ffmpeg/ffmpeg_video_decode.cpp | 39 +++++++++++++++++----- .../src/mediaplugins/ffmpeg/ffmpeg_video_decode.h | 29 ++++------------ .../src/mediaplugins/include/core/plugin_core.h | 15 ++++++++- .../include/core/plugin_video_decode.h | 3 +- 11 files changed, 143 insertions(+), 54 deletions(-) diff --git a/mediaplugin/src/base/UMusic.pas b/mediaplugin/src/base/UMusic.pas index 1f7c8e8f..a4faa98a 100644 --- a/mediaplugin/src/base/UMusic.pas +++ b/mediaplugin/src/base/UMusic.pas @@ -417,9 +417,17 @@ type property Position: real read GetPosition write SetPosition; end; + TVideoFrameFormat = ( + vffUnknown, + vffRGB, // packed RGB 24bpp (R:8,G:8,B:8) + vffRGBA, // packed RGBA 32bpp (R:8,G:8,B:8,A:8) + vffBGR, // packed RGB 24bpp (B:8,G:8,R:8) + vffBGRA // packed BGRA 32bpp (B:8,G:8,R:8,A:8) + ); + TVideoDecodeStream = class public - function Open(const FileName: IPath): boolean; virtual; abstract; + function Open(const FileName: IPath; Format: TVideoFrameFormat): boolean; virtual; abstract; procedure Close; virtual; abstract; procedure SetLoop(Enable: boolean); virtual; abstract; @@ -428,11 +436,12 @@ type procedure SetPosition(Time: real); virtual; abstract; function GetPosition: real; virtual; abstract; - function GetFrameAspect(): real; virtual; abstract; function GetFrame(Time: Extended): PByteArray; virtual; abstract; function GetFrameWidth(): integer; virtual; abstract; function GetFrameHeight(): integer; virtual; abstract; + function GetFrameAspect(): real; virtual; abstract; + function GetFrameFormat(): TVideoFrameFormat; virtual; abstract; end; type @@ -525,7 +534,7 @@ type IVideoDecoder = interface(IGenericDecoder) ['{2F184B2B-FE69-44D5-9031-0A2462391DCA}'] - function Open(const FileName: IPath): TVideoDecodeStream; + function Open(const FileName: IPath; Format: TVideoFrameFormat): TVideoDecodeStream; end; IAudioDecoder = interface(IGenericDecoder) diff --git a/mediaplugin/src/media/UMediaPlugin.pas b/mediaplugin/src/media/UMediaPlugin.pas index cf25e033..8d4a8a7e 100644 --- a/mediaplugin/src/media/UMediaPlugin.pas +++ b/mediaplugin/src/media/UMediaPlugin.pas @@ -119,11 +119,22 @@ type getRatio: function(stream: PAudioConvertStream): double; cdecl; end; +type + TCVideoFrameFormat = cint; +const + FRAME_FORMAT_UNKNOWN = 0; + FRAME_FORMAT_RGB = 1; // packed RGB 24bpp (R:8,G:8,B:8) + FRAME_FORMAT_RGBA = 2; // packed RGBA 32bpp (R:8,G:8,B:8,A:8) + FRAME_FORMAT_BGR = 3; // packed RGB 24bpp (B:8,G:8,R:8) + FRAME_FORMAT_BGRA = 4; // packed BGRA 32bpp (B:8,G:8,R:8,A:8) + +type PVideoFrameInfo = ^TVideoFrameInfo; TVideoFrameInfo = record width: cint; height: cint; aspect: double; + format: TCVideoFrameFormat; end; PVideoDecoderInfo = ^TVideoDecoderInfo; @@ -131,7 +142,7 @@ type priority: cint; init: function(): cbool; cdecl; finalize: function(): cbool; cdecl; - open: function(filename: PAnsiChar): PVideoDecodeStream; cdecl; + open: function(filename: PAnsiChar; format: TCVideoFrameFormat): PVideoDecodeStream; cdecl; close: procedure(stream: PVideoDecodeStream); cdecl; setLoop: procedure(stream: PVideoDecodeStream; enable: cbool); cdecl; getLoop: function(stream: PVideoDecodeStream): cbool; cdecl; diff --git a/mediaplugin/src/media/UMedia_dummy.pas b/mediaplugin/src/media/UMedia_dummy.pas index d5e99aa3..fc560ce8 100644 --- a/mediaplugin/src/media/UMedia_dummy.pas +++ b/mediaplugin/src/media/UMedia_dummy.pas @@ -174,7 +174,7 @@ type function InitializeDecoder(): boolean; function FinalizeDecoder(): boolean; - function Open(const FileName: IPath): TVideoDecodeStream; + function Open(const FileName: IPath; Format: TVideoFrameFormat): TVideoDecodeStream; end; function TAudio_Dummy.GetName: string; @@ -529,7 +529,7 @@ begin Result := true; end; -function TVideoDecoder_Dummy.Open(const FileName: IPath): TVideoDecodeStream; +function TVideoDecoder_Dummy.Open(const FileName: IPath; Format: TVideoFrameFormat): TVideoDecodeStream; begin Result := nil; end; diff --git a/mediaplugin/src/media/UVideo.pas b/mediaplugin/src/media/UVideo.pas index 776d9f33..8361e2a8 100644 --- a/mediaplugin/src/media/UVideo.pas +++ b/mediaplugin/src/media/UVideo.pas @@ -60,16 +60,19 @@ uses const {$IFDEF PIXEL_FMT_BGR} - PIXEL_FMT_OPENGL = GL_BGR; - PIXEL_FMT_SIZE = 3; + PIXEL_FMT_DECODER = vffBGR; + PIXEL_FMT_OPENGL = GL_BGR; + PIXEL_FMT_SIZE = 3; // looks strange on linux: - //PIXEL_FMT_OPENGL = GL_RGBA; - //PIXEL_FMT_SIZE = 4; + //PIXEL_FMT_DECODER = vffRGBA; + //PIXEL_FMT_OPENGL = GL_RGBA; + //PIXEL_FMT_SIZE = 4; {$ELSE} // looks strange on linux: - PIXEL_FMT_OPENGL = GL_RGB; - PIXEL_FMT_SIZE = 3; + PIXEL_FMT_DECODER = vffRGB; + PIXEL_FMT_OPENGL = GL_RGB; + PIXEL_FMT_SIZE = 3; {$ENDIF} ReflectionH = 0.5; //reflection height (50%) @@ -202,14 +205,14 @@ begin Result := true; end; -function TVideoPlayback_FFmpeg.Open(const FileName : IPath): IVideo; +function TVideoPlayback_FFmpeg.Open(const FileName: IPath): IVideo; var Video: IVideo_FFmpeg; Decoder: TVideoDecodeStream; begin Result := nil; - Decoder := VideoDecoder.Open(FileName); + Decoder := VideoDecoder.Open(FileName, PIXEL_FMT_DECODER); if (Decoder = nil) then Exit; diff --git a/mediaplugin/src/media/UVideoDecoderPlugin.pas b/mediaplugin/src/media/UVideoDecoderPlugin.pas index ef411023..7ed18ded 100644 --- a/mediaplugin/src/media/UVideoDecoderPlugin.pas +++ b/mediaplugin/src/media/UVideoDecoderPlugin.pas @@ -51,7 +51,7 @@ type function InitializeDecoder(): boolean; function FinalizeDecoder: boolean; - function Open(const FileName: IPath): TVideoDecodeStream; + function Open(const FileName: IPath; Format: TVideoFrameFormat): TVideoDecodeStream; end; implementation @@ -75,7 +75,7 @@ type constructor Create(Info: PVideoDecoderInfo); destructor Destroy; override; - function Open(const FileName: IPath): boolean; override; + function Open(const FileName: IPath; Format: TVideoFrameFormat): boolean; override; procedure Close; override; procedure SetLoop(Enable: boolean); override; @@ -86,8 +86,9 @@ type function GetFrameWidth(): integer; override; function GetFrameHeight(): integer; override; - function GetFrameAspect(): real; override; + function GetFrameFormat(): TVideoFrameFormat; override; + function GetFrame(Time: Extended): PByteArray; override; end; @@ -121,14 +122,14 @@ begin Result := true; end; -function TVideoDecoderPlugin.Open(const FileName : IPath): TVideoDecodeStream; +function TVideoDecoderPlugin.Open(const FileName: IPath; Format: TVideoFrameFormat): TVideoDecodeStream; var Stream: TPluginVideoDecodeStream; begin Result := nil; Stream := TPluginVideoDecodeStream.Create(fPluginInfo.videoDecoder); - if (not Stream.Open(FileName)) then + if (not Stream.Open(FileName, Format)) then begin Stream.Free; Exit; @@ -153,13 +154,23 @@ begin inherited; end; -function TPluginVideoDecodeStream.Open(const FileName: IPath): boolean; +function TPluginVideoDecodeStream.Open(const FileName: IPath; Format: TVideoFrameFormat): boolean; +var + CFormat: TCVideoFrameFormat; begin Result := false; Close(); - fStream := fVideoDecoderInfo.open(PChar(Filename.ToUTF8())); + case Format of + vffRGB: CFormat := FRAME_FORMAT_RGB; + vffRGBA: CFormat := FRAME_FORMAT_RGBA; + vffBGR: CFormat := FRAME_FORMAT_BGR; + vffBGRA: CFormat := FRAME_FORMAT_BGRA; + else CFormat := FRAME_FORMAT_UNKNOWN; + end; + + fStream := fVideoDecoderInfo.open(PChar(Filename.ToUTF8()), CFormat); if (fStream = nil) then Exit; @@ -227,4 +238,18 @@ begin Result := FrameInfo.aspect; end; +function TPluginVideoDecodeStream.GetFrameFormat(): TVideoFrameFormat; +var + FrameInfo: TVideoFrameInfo; +begin + fVideoDecoderInfo.getFrameInfo(fStream, @FrameInfo); + case FrameInfo.format of + FRAME_FORMAT_RGB: Result := vffRGB; + FRAME_FORMAT_RGBA: Result := vffRGBA; + FRAME_FORMAT_BGR: Result := vffBGR; + FRAME_FORMAT_BGRA: Result := vffBGRA; + else Result := vffUnknown; + end; +end; + end. diff --git a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.cpp b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.cpp index d33af650..79f353e1 100644 --- a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.cpp +++ b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.cpp @@ -187,6 +187,26 @@ bool MediaCore_FFmpeg::convertFFmpegToAudioFormat(SampleFormat ffmpegFormat, aud return true; } +bool MediaCore_FFmpeg::convertVideoFrameFormatToFFmpeg(videoFrameFormat_t format, enum PixelFormat *ffmpegFormat) const { + switch (format) { + case FRAME_FORMAT_RGB: + *ffmpegFormat = PIX_FMT_RGB24; + break; + case FRAME_FORMAT_RGBA: + *ffmpegFormat = PIX_FMT_RGB32; //PIX_FMT_RGBA; + break; + case FRAME_FORMAT_BGR: + *ffmpegFormat = PIX_FMT_BGR24; + break; + case FRAME_FORMAT_BGRA: + *ffmpegFormat = PIX_FMT_BGR32; //PIX_FMT_BGRA; + break; + default: + return false; + } + return true; +} + /** * UTF-8 Filename wrapper based on: * http://www.mail-archive.com/libav-user@mplayerhq.hu/msg02460.html diff --git a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.h b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.h index 524e37b4..37b9067c 100644 --- a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.h +++ b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_core.h @@ -118,6 +118,7 @@ public: bool findStreamIDs(AVFormatContext *formatCtx, int *firstVideoStream, int *firstAudioStream) const; int findAudioStreamIndex(AVFormatContext *formatCtx) const; bool convertFFmpegToAudioFormat(SampleFormat ffmpegFormat, audioSampleFormat_t *format) const; + bool convertVideoFrameFormatToFFmpeg(videoFrameFormat_t format, enum PixelFormat *ffmpegFormat) const; public: class AVCodecLock { diff --git a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.cpp b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.cpp index 8107c1b2..ad34c7f3 100644 --- a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.cpp +++ b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.cpp @@ -67,6 +67,8 @@ FFmpegVideoDecodeStream::FFmpegVideoDecodeStream() : _codec(NULL), _avFrame(NULL), _avFrameRGB(NULL), + _frameFormat(FRAME_FORMAT_UNKNOWN), + _pixelFormat(PIX_FMT_NONE), _frameBuffer(NULL), _frameTexValid(false), #ifdef USE_SWSCALE @@ -77,16 +79,18 @@ FFmpegVideoDecodeStream::FFmpegVideoDecodeStream() : _frameTime(0), _loopTime(0) {} -FFmpegVideoDecodeStream* FFmpegVideoDecodeStream::open(const IPath &filename) { +FFmpegVideoDecodeStream* FFmpegVideoDecodeStream::open(const IPath &filename, + videoFrameFormat_t format) +{ FFmpegVideoDecodeStream *stream = new FFmpegVideoDecodeStream(); - if (!stream->_open(filename)) { + if (!stream->_open(filename, format)) { delete stream; return 0; } return stream; } -bool FFmpegVideoDecodeStream::_open(const IPath &filename) { +bool FFmpegVideoDecodeStream::_open(const IPath &filename, videoFrameFormat_t format) { std::stringstream ss; // use custom 'ufile' protocol for UTF-8 support @@ -169,10 +173,20 @@ bool FFmpegVideoDecodeStream::_open(const IPath &filename) { logger.status(ss.str(), ""); #endif + _frameFormat = format; + // choose default (use BGR-format for accelerated colorspace conversion with swscale) + if (_frameFormat == FRAME_FORMAT_UNKNOWN) + _frameFormat = FRAME_FORMAT_BGR; + if (!ffmpegCore->convertVideoFrameFormatToFFmpeg(_frameFormat, &_pixelFormat)) { + logger.error("Invalid pixel format", "VideoPlayback_ffmpeg.Open"); + close(); + return false; + } + // allocate space for decoded frame and rgb frame _avFrame = avcodec_alloc_frame(); _avFrameRGB = avcodec_alloc_frame(); - _frameBuffer = (uint8_t*) av_malloc(avpicture_get_size(PIXEL_FMT_FFMPEG, + _frameBuffer = (uint8_t*) av_malloc(avpicture_get_size(_pixelFormat, _codecContext->width, _codecContext->height)); if (!_avFrame || !_avFrameRGB || !_frameBuffer) { @@ -184,7 +198,7 @@ bool FFmpegVideoDecodeStream::_open(const IPath &filename) { // TODO: pad data for OpenGL to GL_UNPACK_ALIGNMENT // (otherwise video will be distorted if width/height is not a multiple of the alignment) errnum = avpicture_fill((AVPicture*)_avFrameRGB, _frameBuffer, - PIXEL_FMT_FFMPEG, _codecContext->width, _codecContext->height); + _pixelFormat, _codecContext->width, _codecContext->height); if (errnum < 0) { logger.error("avpicture_fill failed: " + ffmpegCore->getErrorString(errnum), "VideoPlayback_ffmpeg.Open"); @@ -223,7 +237,7 @@ bool FFmpegVideoDecodeStream::_open(const IPath &filename) { // could be observed in comparison to the RGB versions. _swScaleContext = sws_getCachedContext(NULL, _codecContext->width, _codecContext->height, _codecContext->pix_fmt, - _codecContext->width, _codecContext->height, PIXEL_FMT_FFMPEG, + _codecContext->width, _codecContext->height, _pixelFormat, SWS_FAST_BILINEAR, NULL, NULL, NULL); if (!_swScaleContext) { @@ -503,7 +517,7 @@ uint8_t* FFmpegVideoDecodeStream::getFrame(long double time) { // I think this should be removed, but am not sure whether there should // be some other replacement or a warning, Therefore, I leave it for now. // April 2009, mischi - errnum = img_convert((AVPicture*)_avFrameRGB, PIXEL_FMT_FFMPEG, + errnum = img_convert((AVPicture*)_avFrameRGB, _pixelFormat, (AVPicture*)_avFrame, _codecContext->pix_fmt, _codecContext->width, _codecContext->height); #endif @@ -587,6 +601,10 @@ double FFmpegVideoDecodeStream::getFrameAspect() { return _aspect; } +videoFrameFormat_t FFmpegVideoDecodeStream::getFrameFormat() { + return _frameFormat; +} + /************************************ * C Interface ************************************/ @@ -601,8 +619,10 @@ static BOOL PLUGIN_CALL ffmpegVideoDecoder_finalize() { return TRUE; } -static videoDecodeStream_t* PLUGIN_CALL ffmpegVideoDecoder_open(const char *filename) { - return (videoDecodeStream_t*)FFmpegVideoDecodeStream::open(filename); +static videoDecodeStream_t* PLUGIN_CALL ffmpegVideoDecoder_open(const char *filename, + videoFrameFormat_t format) +{ + return (videoDecodeStream_t*)FFmpegVideoDecodeStream::open(filename, format); } static void PLUGIN_CALL ffmpegVideoDecoder_close(videoDecodeStream_t *stream) { @@ -632,6 +652,7 @@ static void PLUGIN_CALL ffmpegVideoDecoder_getFrameInfo(videoDecodeStream_t *str info->width = s->getFrameWidth(); info->height = s->getFrameHeight(); info->aspect = s->getFrameAspect(); + info->format = s->getFrameFormat(); } static uint8_t* PLUGIN_CALL ffmpegVideoDecoder_getFrame(videoDecodeStream_t *stream, long double time) { diff --git a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.h b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.h index 9b0df889..addf2a82 100644 --- a/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.h +++ b/mediaplugin/src/mediaplugins/ffmpeg/ffmpeg_video_decode.h @@ -32,25 +32,6 @@ //#define DEBUG_DISPLAY //#define DEBUG_FRAMES -// use BGR-format for accelerated colorspace conversion with swscale -#ifdef USE_SWSCALE -# define PIXEL_FMT_BGR -#endif - -#define PIXEL_FMT_BGR - -#ifdef PIXEL_FMT_BGR -# define PIXEL_FMT_FFMPEG PIX_FMT_BGR24 -# define PIXEL_FMT_SIZE 3 -// looks strange on linux -//# define PIXEL_FMT_FFMPEG PIX_FMT_BGR32; -//# define PIXEL_FMT_SIZE 4; -#else -// looks strange on linux: -# define PIXEL_FMT_FFMPEG PIX_FMT_RGB24 -# define PIXEL_FMT_SIZE 3 -#endif - extern const videoDecoderInfo_t videoDecoderInfo; class FFmpegVideoDecodeStream : public VideoDecodeStream { @@ -68,6 +49,9 @@ private: AVFrame *_avFrame; AVFrame *_avFrameRGB; + + videoFrameFormat_t _frameFormat; + enum PixelFormat _pixelFormat; uint8_t *_frameBuffer; //**< stores a FFmpeg video frame bool _frameTexValid; //**< if true, fFrameTex contains the current frame @@ -87,7 +71,7 @@ private: bool decodeFrame(); void synchronizeTime(AVFrame *frame, double &pts); - bool _open(const IPath &filename); + bool _open(const IPath &filename, videoFrameFormat_t format); void close(); public: @@ -95,7 +79,7 @@ public: close(); } - static FFmpegVideoDecodeStream* open(const IPath &filename); + static FFmpegVideoDecodeStream* open(const IPath &filename, videoFrameFormat_t format); virtual void setLoop(bool enable); virtual bool getLoop(); @@ -105,8 +89,9 @@ public: virtual int getFrameWidth(); virtual int getFrameHeight(); - virtual double getFrameAspect(); + virtual videoFrameFormat_t getFrameFormat(); + virtual uint8_t* getFrame(long double time); }; diff --git a/mediaplugin/src/mediaplugins/include/core/plugin_core.h b/mediaplugin/src/mediaplugins/include/core/plugin_core.h index acbe0f4a..c7d078b0 100644 --- a/mediaplugin/src/mediaplugins/include/core/plugin_core.h +++ b/mediaplugin/src/mediaplugins/include/core/plugin_core.h @@ -203,17 +203,30 @@ typedef struct audioConverterInfo_t { double PLUGIN_CALL (*getRatio)(audioConvertStream_t *stream); } audioConverterInfo_t; +typedef enum videoFrameFormat_t { + FRAME_FORMAT_UNKNOWN, + FRAME_FORMAT_RGB, // packed RGB 24bpp (R:8,G:8,B:8) + FRAME_FORMAT_RGBA, // packed RGBA 32bpp (R:8,G:8,B:8,A:8) + FRAME_FORMAT_BGR, // packed RGB 24bpp (B:8,G:8,R:8) + FRAME_FORMAT_BGRA, // packed BGRA 32bpp (B:8,G:8,R:8,A:8) +} videoFrameFormat_t; + +const int videoFrameFormatSize[5] = { + -1, 3, 4, 3, 4 +}; + typedef struct videoFrameInfo_t { int width; int height; double aspect; + videoFrameFormat_t format; } videoFrameInfo_t; typedef struct videoDecoderInfo_t { int priority; BOOL PLUGIN_CALL (*init)(); BOOL PLUGIN_CALL (*finalize)(); - videoDecodeStream_t* PLUGIN_CALL (*open)(const char *filename); + videoDecodeStream_t* PLUGIN_CALL (*open)(const char *filename, videoFrameFormat_t format); void PLUGIN_CALL (*close)(videoDecodeStream_t *stream); void PLUGIN_CALL (*setLoop)(videoDecodeStream_t *stream, BOOL enable); BOOL PLUGIN_CALL (*getLoop)(videoDecodeStream_t *stream); diff --git a/mediaplugin/src/mediaplugins/include/core/plugin_video_decode.h b/mediaplugin/src/mediaplugins/include/core/plugin_video_decode.h index 6403852f..a96daf6a 100644 --- a/mediaplugin/src/mediaplugins/include/core/plugin_video_decode.h +++ b/mediaplugin/src/mediaplugins/include/core/plugin_video_decode.h @@ -43,8 +43,9 @@ public: virtual int getFrameWidth() = 0; virtual int getFrameHeight() = 0; - virtual double getFrameAspect() = 0; + virtual videoFrameFormat_t getFrameFormat() = 0; + virtual uint8_t* getFrame(long double time) = 0; }; -- cgit v1.2.3