diff options
author | tobigun <tobigun@b956fd51-792f-4845-bead-9b4dfca2ff2c> | 2010-04-23 21:08:06 +0000 |
---|---|---|
committer | tobigun <tobigun@b956fd51-792f-4845-bead-9b4dfca2ff2c> | 2010-04-23 21:08:06 +0000 |
commit | d5a50482317c90ef83bfa4787b392b34f59d0989 (patch) | |
tree | b9c7b2a5864a83a712e33091d776b0e46d943695 /src/encoding | |
parent | e207d2671c3db087f0ec1b9f6594f1746ca16fa7 (diff) | |
download | usdx-d5a50482317c90ef83bfa4787b392b34f59d0989.tar.gz usdx-d5a50482317c90ef83bfa4787b392b34f59d0989.tar.xz usdx-d5a50482317c90ef83bfa4787b392b34f59d0989.zip |
With FPC ConsoleWriteLn is not initialized. Fallback to AlexanderS original solution but make the warning more explicit.
git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/trunk@2287 b956fd51-792f-4845-bead-9b4dfca2ff2c
Diffstat (limited to 'src/encoding')
-rw-r--r-- | src/encoding/Auto.inc | 269 |
1 files changed, 137 insertions, 132 deletions
diff --git a/src/encoding/Auto.inc b/src/encoding/Auto.inc index 8c32b5d0..f404c2f6 100644 --- a/src/encoding/Auto.inc +++ b/src/encoding/Auto.inc @@ -1,132 +1,137 @@ -{* UltraStar Deluxe - Karaoke Game - * - * UltraStar Deluxe is the legal property of its developers, whose names - * are too numerous to list here. Please refer to the COPYRIGHT - * file distributed with this source distribution. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; see the file COPYING. If not, write to - * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - * - * $URL$ - * $Id$ - *} - -// Auto -// try to match the w3c regex and decode as unicode on match and as fallback if not match -// (copied from http://www.w3.org/International/questions/qa-forms-utf-8.en.php) -// -// m/\A( -// [\x09\x0A\x0D\x20-\x7E] # ASCII -// | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte -// | \xE0[\xA0-\xBF][\x80-\xBF] # excluding overlongs -// | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2} # straight 3-byte -// | \xED[\x80-\x9F][\x80-\xBF] # excluding surrogates -// | \xF0[\x90-\xBF][\x80-\xBF]{2} # planes 1-3 -// | [\xF1-\xF3][\x80-\xBF]{3} # planes 4-15 -// | \xF4[\x80-\x8F][\x80-\xBF]{2} # plane 16 -// )*\z/x - -type - TEncoderAuto = class(TEncoder) - public - function GetName(): AnsiString; override; - function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override; - function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override; - - constructor Create(const UTF8Encoder, FallbackEncoder: IEncoder); - - private - FallbackEncoder: IEncoder; - UTF8Encoder: IEncoder; - Regex: PPCRE; - RegexExtra: PPCREExtra; - end; - -function PCREGetMem(Size: SizeInt): Pointer; cdecl; -begin - GetMem(Result, Size); -end; - -procedure PCREFreeMem(P: Pointer); cdecl; -begin - FreeMem(P); -end; - -constructor TEncoderAuto.Create(const UTF8Encoder, FallbackEncoder: IEncoder); -var - Error: PChar; - ErrorOffset: Integer; -begin - inherited Create(); - self.FallbackEncoder := FallbackEncoder; - self.UTF8Encoder := UTF8Encoder; - - // Load and initialize PCRE Library - if LoadPCRE() then - begin - // compile regex - self.Regex := pcre_compile('\A([\x09\x0A\x0D\x20-\x7E]|[\xC2-\xDF][\x80-\xBF]|\xE0[\xA0-\xBF][\x80-\xBF]|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}|\xED[\x80-\x9F][\x80-\xBF]|\xF0[\x90-\xBF][\x80-\xBF]{2}|[\xF1-\xF3][\x80-\xBF]{3}|\xF4[\x80-\x8F][\x80-\xBF]{2})*\z', 0, @Error, @ErrorOffset, nil); - - if self.Regex = Nil then - begin - // NOTICE: Log.LogError() is not possible here because it isn't loaded - ConsoleWriteLn(Format('ERROR: UTF8 Regex compilation failed: %s at %d', [Error, ErrorOffset])); - end - else - begin - // if compiled successfull, try to get more informations the speed up the matching - self.RegexExtra := pcre_study(self.Regex, 0, @Error); - - if Error <> Nil then - begin - // NOTICE: Log.LogError() is not possible here because it isn't loaded - ConsoleWriteLn('ERROR: UTF8 Regex study failed: ' + Error); - end; - end; - end - else - begin - // NOTICE: Log.LogError() is not possible here because it isn't loaded - ConsoleWriteLn('ERROR: pcre not loaded. utf-8 autodetection will not work.'); - end; -end; - -function TEncoderAuto.GetName(): AnsiString; -begin - Result := 'Auto'; -end; - -function TEncoderAuto.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; -var - RegexResults: Integer; -begin - if (self.Regex <> Nil) then - begin - RegexResults := pcre_exec(Regex, RegexExtra, PChar(InStr), Length(InStr), 0, 0, Nil, 0); - - if RegexResults >= 0 then - begin - Result := UTF8Encoder.Decode(InStr, OutStr); - Exit; - end; - end; - - Result := FallbackEncoder.Decode(InStr, OutStr); -end; - -function TEncoderAuto.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; -begin - Result := UTF8Encoder.Encode(InStr, OutStr); -end; +{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+// Auto
+// try to match the w3c regex and decode as unicode on match and as fallback if not match
+// (copied from http://www.w3.org/International/questions/qa-forms-utf-8.en.php)
+//
+// m/\A(
+// [\x09\x0A\x0D\x20-\x7E] # ASCII
+// | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte
+// | \xE0[\xA0-\xBF][\x80-\xBF] # excluding overlongs
+// | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2} # straight 3-byte
+// | \xED[\x80-\x9F][\x80-\xBF] # excluding surrogates
+// | \xF0[\x90-\xBF][\x80-\xBF]{2} # planes 1-3
+// | [\xF1-\xF3][\x80-\xBF]{3} # planes 4-15
+// | \xF4[\x80-\x8F][\x80-\xBF]{2} # plane 16
+// )*\z/x
+
+type
+ TEncoderAuto = class(TEncoder)
+ public
+ function GetName(): AnsiString; override;
+ function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+ function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+
+ constructor Create(const UTF8Encoder, FallbackEncoder: IEncoder);
+
+ private
+ FallbackEncoder: IEncoder;
+ UTF8Encoder: IEncoder;
+ Regex: PPCRE;
+ RegexExtra: PPCREExtra;
+ end;
+
+function PCREGetMem(Size: SizeInt): Pointer; cdecl;
+begin
+ GetMem(Result, Size);
+end;
+
+procedure PCREFreeMem(P: Pointer); cdecl;
+begin
+ FreeMem(P);
+end;
+
+// NOTICE: Log.LogError/ConsoleWriteLn/DebugWriteLn are initialized yet
+procedure ShowError(const msg: string);
+begin
+ {$IFDEF CONSOLE}
+ WriteLn('ERROR: ', msg);
+ {$ENDIF}
+end;
+
+constructor TEncoderAuto.Create(const UTF8Encoder, FallbackEncoder: IEncoder);
+var
+ Error: PChar;
+ ErrorOffset: Integer;
+begin
+ inherited Create();
+ self.FallbackEncoder := FallbackEncoder;
+ self.UTF8Encoder := UTF8Encoder;
+
+ // Load and initialize PCRE Library
+ if LoadPCRE() then
+ begin
+ // compile regex
+ self.Regex := pcre_compile('\A([\x09\x0A\x0D\x20-\x7E]|[\xC2-\xDF][\x80-\xBF]|\xE0[\xA0-\xBF][\x80-\xBF]|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}|\xED[\x80-\x9F][\x80-\xBF]|\xF0[\x90-\xBF][\x80-\xBF]{2}|[\xF1-\xF3][\x80-\xBF]{3}|\xF4[\x80-\x8F][\x80-\xBF]{2})*\z', 0, @Error, @ErrorOffset, nil);
+
+ if self.Regex = Nil then
+ begin
+ ShowError(Format('UTF8 Regex compilation failed: %s at %d', [Error, ErrorOffset]));
+ end
+ else
+ begin
+ // if compiled successfull, try to get more informations the speed up the matching
+ self.RegexExtra := pcre_study(self.Regex, 0, @Error);
+
+ if Error <> Nil then
+ begin
+ ShowError('UTF8 Regex study failed: ' + Error);
+ end;
+ end;
+ end
+ else
+ begin
+ ShowError('pcre not loaded. utf-8 autodetection will not work.');
+ end;
+end;
+
+function TEncoderAuto.GetName(): AnsiString;
+begin
+ Result := 'Auto';
+end;
+
+function TEncoderAuto.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+var
+ RegexResults: Integer;
+begin
+ if (self.Regex <> Nil) then
+ begin
+ RegexResults := pcre_exec(Regex, RegexExtra, PChar(InStr), Length(InStr), 0, 0, Nil, 0);
+
+ if RegexResults >= 0 then
+ begin
+ Result := UTF8Encoder.Decode(InStr, OutStr);
+ Exit;
+ end;
+ end;
+
+ Result := FallbackEncoder.Decode(InStr, OutStr);
+end;
+
+function TEncoderAuto.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+ Result := UTF8Encoder.Encode(InStr, OutStr);
+end;
|