- encodings seperated to multiple files located in "encoding"

git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/branches/experimental@1857 b956fd51-792f-4845-bead-9b4dfca2ff2c

5 files changed, 1585 insertions, 137 deletions

diff --git a/unicode/src/base/UTextEncoding.pas b/unicode/src/base/UTextEncoding.pas
index 61cd0098..966064c0 100644
--- a/unicode/src/base/UTextEncoding.pas
+++ b/unicode/src/base/UTextEncoding.pas
@@ -34,181 +34,164 @@ interface
 {$I switches.inc}
 
 uses
-  SysUtils,
-  StrUtils;
+  SysUtils;
 
 type
   TEncoding = (
-    encCP1250,  // Windows-1250 Central/Eastern Europe (used by Ultrastar)
-    encCP1252,  // Windows-1252 Western Europe (used by UltraStar Deluxe < 1.1)
+    encLocale,  // current locale (needs cwstring on linux)
     encUTF8,    // UTF-8
-    encLocale   // current locale (needs cwstring on linux)
-  );
-
-const
-  EncodingNames: array[TEncoding] of AnsiString = (
-    'CP1250',
-    'CP1252',
-    'UTF8',
-    'LOCALE'
+    encCP1250,  // Windows-1250 Central/Eastern Europe (used by Ultrastar)
+    encCP1252   // Windows-1252 Western Europe (used by UltraStar Deluxe < 1.1)
   );
 
 const
   UTF8_BOM: UTF8String = #$EF#$BB#$BF;
 
 {**
- * Changes encoding of string Src with encoding SrcEncoding to UTF-16
- * If SrcEncoding is encUnknown the result is undefined.
+ * Decodes Src encoded in SrcEncoding to a UTF-16 or UTF-8 encoded Dst string.
+ * Returns true if the conversion was successful.
  *}
-function RecodeStringWide(const Src: string; SrcEncoding: TEncoding): WideString;
+function DecodeString(const Src: AnsiString; out Dst: WideString; SrcEncoding: TEncoding): boolean; overload;
+function DecodeString(const Src: AnsiString; SrcEncoding: TEncoding): WideString; overload;
+function DecodeStringUTF8(const Src: AnsiString; out Dst: UTF8String; SrcEncoding: TEncoding): boolean; overload;
+function DecodeStringUTF8(const Src: AnsiString; SrcEncoding: TEncoding): UTF8String; overload;
 
 {**
- * Changes encoding of string Src with encoding SrcEncoding to UTF-8.
- * If SrcEncoding is encUnknown the result is undefined.
+ * Encodes the UTF-16 or UTF-8 encoded Src string to Dst using DstEncoding
+ * Returns true if the conversion was successful.
  *}
-function RecodeStringUTF8(const Src: string; SrcEncoding: TEncoding): UTF8String;
+function EncodeString(const Src: WideString; out Dst: AnsiString; DstEncoding: TEncoding): boolean; overload;
+function EncodeString(const Src: WideString; DstEncoding: TEncoding): AnsiString; overload;
+function EncodeStringUTF8(const Src: UTF8String; out Dst: AnsiString; DstEncoding: TEncoding): boolean; overload;
+function EncodeStringUTF8(const Src: UTF8String; DstEncoding: TEncoding): AnsiString; overload;
 
 {**
  * If Text starts with an UTF-8 BOM, the BOM is removed and true will
  * be returned.
  *}
-function CheckReplaceUTF8BOM(var Text: string): boolean;
+function CheckReplaceUTF8BOM(var Text: AnsiString): boolean;
 
 {**
  * Parses an encoding string to its TEncoding equivalent.
  * Surrounding whitespace and dashes ('-') are removed, the upper-cased
  * resulting value is then compared with TEncodingNames.
- * If the encoding was not found, the result is set to the Default encoding. 
+ * If the encoding was not found, the result is set to the Default encoding.
  *}
 function ParseEncoding(const EncodingStr: AnsiString; Default: TEncoding): TEncoding;
 
+{**
+ * Returns the name of an encoding.
+ *}
+function EncodingName(Encoding: TEncoding): AnsiString;
+
 implementation
 
+uses
+  StrUtils,
+  UUnicodeUtils;
+
 type
-  TConversionTable = array[0..127] of WideChar;
+  IEncoder = interface
+    function GetName(): AnsiString;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+  end;
+
+  TEncoder = class(TInterfacedObject, IEncoder)
+  public
+    function GetName(): AnsiString; virtual; abstract;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; virtual; abstract;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; virtual; abstract;
+  end;
+
+  TSingleByteEncoder = class(TEncoder)
+  public
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; virtual; abstract;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; virtual; abstract;
+  end;
 
 const
-  // Windows-1250 Central/Eastern Europe (used by Ultrastar)
-  CP1250Table: TConversionTable = (
-    { $80 }
-    #$20AC,     #0, #$201A,     #0, #$201E, #$2026, #$2020, #$2021,
-        #0, #$2030, #$0160, #$2039, #$015A, #$0164, #$017D, #$0179,
-    { $90 }
-        #0, #$2018, #$2019, #$201C, #$201D, #$2022, #$2013, #$2014,
-        #0, #$2122, #$0161, #$203A, #$015B, #$0165, #$017E, #$017A,
-    { $A0 }
-    #$00A0, #$02C7, #$02D8, #$0141, #$00A4, #$0104, #$00A6, #$00A7,
-    #$00A8, #$00A9, #$015E, #$00AB, #$00AC, #$00AD, #$00AE, #$017B,
-    { $B0 }
-    #$00B0, #$00B1, #$02DB, #$0142, #$00B4, #$00B5, #$00B6, #$00B7,
-    #$00B8, #$0105, #$015F, #$00BB, #$013D, #$02DD, #$013E, #$017C,
-    { $C0 }
-    #$0154, #$00C1, #$00C2, #$0102, #$00C4, #$0139, #$0106, #$00C7,
-    #$010C, #$00C9, #$0118, #$00CB, #$011A, #$00CD, #$00CE, #$010E,
-    { $D0 }
-    #$0110, #$0143, #$0147, #$00D3, #$00D4, #$0150, #$00D6, #$00D7,
-    #$0158, #$016E, #$00DA, #$0170, #$00DC, #$00DD, #$0162, #$00DF,
-    { $E0 }
-    #$0155, #$00E1, #$00E2, #$0103, #$00E4, #$013A, #$0107, #$00E7,
-    #$010D, #$00E9, #$0119, #$00EB, #$011B, #$00ED, #$00EE, #$010F,
-    { $F0 }
-    #$0111, #$0144, #$0148, #$00F3, #$00F4, #$0151, #$00F6, #$00F7,
-    #$0159, #$016F, #$00FA, #$0171, #$00FC, #$00FD, #$0163, #$02D9
-  );
+  ERROR_CHAR = '?';
 
-  // Windows-1252 Western Europe (used by UltraStar Deluxe < 1.1)
-  CP1252Table: TConversionTable = (
-    { $80 }
-    #$20AC,     #0, #$201A, #$0192, #$201E, #$2026, #$2020, #$2021,
-    #$02C6, #$2030, #$0160, #$2039, #$0152,     #0, #$017D,     #0,
-    { $90 }
-        #0, #$2018, #$2019, #$201C, #$201D, #$2022, #$2013, #$2014,
-    #$02DC, #$2122, #$0161, #$203A, #$0153,     #0, #$017E, #$0178,
-    { $A0 }
-    #$00A0, #$00A1, #$00A2, #$00A3, #$00A4, #$00A5, #$00A6, #$00A7,
-    #$00A8, #$00A9, #$00AA, #$00AB, #$00AC, #$00AD, #$00AE, #$00AF,
-    { $B0 }
-    #$00B0, #$00B1, #$00B2, #$00B3, #$00B4, #$00B5, #$00B6, #$00B7,
-    #$00B8, #$00B9, #$00BA, #$00BB, #$00BC, #$00BD, #$00BE, #$00BF,
-    { $C0 }
-    #$00C0, #$00C1, #$00C2, #$00C3, #$00C4, #$00C5, #$00C6, #$00C7,
-    #$00C8, #$00C9, #$00CA, #$00CB, #$00CC, #$00CD, #$00CE, #$00CF,
-    { $D0 }
-    #$00D0, #$00D1, #$00D2, #$00D3, #$00D4, #$00D5, #$00D6, #$00D7,
-    #$00D8, #$00D9, #$00DA, #$00DB, #$00DC, #$00DD, #$00DE, #$00DF,
-    { $E0 }
-    #$00E0, #$00E1, #$00E2, #$00E3, #$00E4, #$00E5, #$00E6, #$00E7,
-    #$00E8, #$00E9, #$00EA, #$00EB, #$00EC, #$00ED, #$00EE, #$00EF,
-    { $F0 }
-    #$00F0, #$00F1, #$00F2, #$00F3, #$00F4, #$00F5, #$00F6, #$00F7,
-    #$00F8, #$00F9, #$00FA, #$00FB, #$00FC, #$00FD, #$00FE, #$00FF
-  );
+var
+  Encoders: array[TEncoding] of IEncoder;
 
-{**
- * Internal conversion function
- *}
-function Convert(const Src: string; const Table: TConversionTable): WideString;
+function TSingleByteEncoder.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
 var
-  SrcPos, DstPos: integer;
+  I: integer;
 begin
-  SetLength(Result, Length(Src));
-  DstPos := 1;
-  for SrcPos := 1 to Length(Src) do
+  SetLength(OutStr, LengthUCS4(InStr));
+  Result := true;
+  for I := 1 to Length(OutStr) do
   begin
-    if (Src[SrcPos] < #128) then
-    begin
-      // copy ASCII char
-      // Important: the Ord() is necessary to prevent FPC from an automatic
-      // encoding conversion (using the local codepage). Delphi does not perform
-      // such a conversion.
-      Result[DstPos] := WideChar(Ord(Src[SrcPos]));
-      Inc(DstPos);
-    end
-    else
-    begin
-      // look-up char
-      Result[DstPos] := Table[Ord(Src[SrcPos]) - 128];
-      // ignore invalid characters
-      if (Result[DstPos] <> #0) then
-        Inc(DstPos);
-    end;
+    if (not EncodeChar(InStr[I-1], OutStr[I])) then
+      Result := false;
   end;
-  SetLength(Result, DstPos-1);
 end;
 
-function RecodeStringWide(const Src: string; SrcEncoding: TEncoding): WideString;
+function TSingleByteEncoder.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+var
+  I: integer;
 begin
-  case SrcEncoding of
-    encCP1250:
-      Result := Convert(Src, CP1250Table);
-    encCP1252:
-      Result := Convert(Src, CP1252Table);
-    encUTF8:
-      Result := UTF8Decode(Src);
-    encLocale:
-      Result := UTF8Decode(AnsiToUtf8(Src));
-    else
-      Result := '';
+  SetLength(OutStr, Length(InStr)+1);
+  Result := true;
+  for I := 1 to Length(InStr) do
+  begin
+    if (not DecodeChar(InStr[I], OutStr[I-1])) then
+      Result := false;
   end;
+  OutStr[High(OutStr)] := 0;
 end;
 
-function RecodeStringUTF8(const Src: string; SrcEncoding: TEncoding): UTF8String;
+function DecodeString(const Src: AnsiString; out Dst: WideString; SrcEncoding: TEncoding): boolean;
+var
+  DstUCS4: UCS4String;
 begin
-  case SrcEncoding of
-    encCP1250:
-      Result := UTF8Encode(Convert(Src, CP1250Table));
-    encCP1252:
-      Result := UTF8Encode(Convert(Src, CP1252Table));
-    encUTF8:
-      Result := Src;
-    encLocale:
-      Result := AnsiToUtf8(Src);
-    else
-      Result := '';
-  end;
+  Result := Encoders[SrcEncoding].Decode(Src, DstUCS4);
+  Dst := UCS4StringToWideString(DstUCS4);
 end;
 
-function CheckReplaceUTF8BOM(var Text: string): boolean;
+function DecodeString(const Src: AnsiString; SrcEncoding: TEncoding): WideString;
+begin
+  DecodeString(Src, Result, SrcEncoding);
+end;
+
+function DecodeStringUTF8(const Src: AnsiString; out Dst: UTF8String; SrcEncoding: TEncoding): boolean;
+var
+  DstUCS4: UCS4String;
+begin
+  Result := Encoders[SrcEncoding].Decode(Src, DstUCS4);
+  Dst := UCS4ToUTF8String(DstUCS4);
+end;
+
+function DecodeStringUTF8(const Src: AnsiString; SrcEncoding: TEncoding): UTF8String;
+begin
+  DecodeStringUTF8(Src, Result, SrcEncoding);
+end;
+
+function EncodeString(const Src: WideString; out Dst: AnsiString; DstEncoding: TEncoding): boolean;
+begin
+  Result := Encoders[DstEncoding].Encode(WideStringToUCS4String(Src), Dst);
+end;
+
+function EncodeString(const Src: WideString; DstEncoding: TEncoding): AnsiString;
+begin
+  EncodeString(Src, Result, DstEncoding);
+end;
+
+function EncodeStringUTF8(const Src: UTF8String; out Dst: AnsiString; DstEncoding: TEncoding): boolean;
+begin
+  Result := Encoders[DstEncoding].Encode(UTF8ToUCS4String(Src), Dst);
+end;
+
+function EncodeStringUTF8(const Src: UTF8String; DstEncoding: TEncoding): AnsiString;
+begin
+  EncodeStringUTF8(Src, Result, DstEncoding);
+end;
+
+function CheckReplaceUTF8BOM(var Text: AnsiString): boolean;
 begin
   if AnsiStartsStr(UTF8_BOM, Text) then
   begin
@@ -221,20 +204,36 @@ end;
 
 function ParseEncoding(const EncodingStr: AnsiString; Default: TEncoding): TEncoding;
 var
-  PrepStr: string; // prepared encoding string
+  PrepStr: AnsiString; // prepared encoding string
   Encoding: TEncoding;
 begin
   // remove surrounding whitespace, replace dashes, to upper case
-  PrepStr := UpperCase(AnsiReplaceStr(Trim(EncodingStr), '-', ''));

-  for Encoding := Low(EncodingNames) to High(EncodingNames) do

-  begin

-    if (EncodingNames[Encoding] = PrepStr) then

-    begin

-      Result := Encoding;

-      Exit;

+  PrepStr := UpperCase(AnsiReplaceStr(Trim(EncodingStr), '-', ''));
+  for Encoding := Low(TEncoding) to High(TEncoding) do
+  begin
+    if (Encoders[Encoding].GetName() = PrepStr) then
+    begin
+      Result := Encoding;
+      Exit;
     end;

   end;

-  Result := Default;

+  Result := Default;
+end;
+
+function EncodingName(Encoding: TEncoding): AnsiString;
+begin
+  Result := Encoders[Encoding].GetName();
 end;
 
+{$I ../encoding/Locale.inc}
+{$I ../encoding/UTF8.inc}
+{$I ../encoding/CP1250.inc}
+{$I ../encoding/CP1252.inc}
+
+initialization
+  Encoders[encLocale] := TEncoderLocale.Create;
+  Encoders[encUTF8]   := TEncoderUTF8.Create;
+  Encoders[encCP1250] := TEncoderCP1250.Create;
+  Encoders[encCP1252] := TEncoderCP1252.Create;
+
 end.
diff --git a/unicode/src/encoding/CP1250.inc b/unicode/src/encoding/CP1250.inc
new file mode 100644
index 00000000..904e9dca
--- /dev/null
+++ b/unicode/src/encoding/CP1250.inc
@@ -0,0 +1,708 @@
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1250 Central/Eastern Europe
+ * (used by Ultrastar)
+ *}
+
+type
+  TEncoderCP1250 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1250.GetName(): AnsiString;
+begin
+  Result := 'CP1250';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1250Table: array[128..255] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A,     0, $201E, $2026, $2020, $2021,
+        0, $2030, $0160, $2039, $015A, $0164, $017D, $0179,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+        0, $2122, $0161, $203A, $015B, $0165, $017E, $017A,
+    { $A0 }
+    $00A0, $02C7, $02D8, $0141, $00A4, $0104, $00A6, $00A7,
+    $00A8, $00A9, $015E, $00AB, $00AC, $00AD, $00AE, $017B,
+    { $B0 }
+    $00B0, $00B1, $02DB, $0142, $00B4, $00B5, $00B6, $00B7,
+    $00B8, $0105, $015F, $00BB, $013D, $02DD, $013E, $017C,
+    { $C0 }
+    $0154, $00C1, $00C2, $0102, $00C4, $0139, $0106, $00C7,
+    $010C, $00C9, $0118, $00CB, $011A, $00CD, $00CE, $010E,
+    { $D0 }
+    $0110, $0143, $0147, $00D3, $00D4, $0150, $00D6, $00D7,
+    $0158, $016E, $00DA, $0170, $00DC, $00DD, $0162, $00DF,
+    { $E0 }
+    $0155, $00E1, $00E2, $0103, $00E4, $013A, $0107, $00E7,
+    $010D, $00E9, $0119, $00EB, $011B, $00ED, $00EE, $010F,
+    { $F0 }
+    $0111, $0144, $0148, $00F3, $00F4, $0151, $00F6, $00F7,
+    $0159, $016F, $00FA, $0171, $00FC, $00FD, $0163, $02D9
+  );
+
+function TEncoderCP1250.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1250Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1250.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+        $20AC: OutChr := #128;
+        // invalid: #129
+        $201A: OutChr := #130;
+        // invalid: #131
+        $201E: OutChr := #132;
+        $2026: OutChr := #133;
+        $2020: OutChr := #134;
+        $2021: OutChr := #135;
+        // invalid: #136
+        $2030: OutChr := #137;
+        $0160: OutChr := #138;
+        $2039: OutChr := #139;
+        $015A: OutChr := #140;
+        $0164: OutChr := #141;
+        $017D: OutChr := #142;
+        $0179: OutChr := #143;
+        // invalid: #144
+        $2018: OutChr := #145;
+        $2019: OutChr := #146;
+        $201C: OutChr := #147;
+        $201D: OutChr := #148;
+        $2022: OutChr := #149;
+        $2013: OutChr := #150;
+        $2014: OutChr := #151;
+        // invalid: #152
+        $2122: OutChr := #153;
+        $0161: OutChr := #154;
+        $203A: OutChr := #155;
+        $015B: OutChr := #156;
+        $0165: OutChr := #157;
+        $017E: OutChr := #158;
+        $017A: OutChr := #159;
+        $00A0: OutChr := #160;
+        $02C7: OutChr := #161;
+        $02D8: OutChr := #162;
+        $0141: OutChr := #163;
+        $00A4: OutChr := #164;
+        $0104: OutChr := #165;
+        $00A6: OutChr := #166;
+        $00A7: OutChr := #167;
+        $00A8: OutChr := #168;
+        $00A9: OutChr := #169;
+        $015E: OutChr := #170;
+        $00AB: OutChr := #171;
+        $00AC: OutChr := #172;
+        $00AD: OutChr := #173;
+        $00AE: OutChr := #174;
+        $017B: OutChr := #175;
+        $00B0: OutChr := #176;
+        $00B1: OutChr := #177;
+        $02DB: OutChr := #178;
+        $0142: OutChr := #179;
+        $00B4: OutChr := #180;
+        $00B5: OutChr := #181;
+        $00B6: OutChr := #182;
+        $00B7: OutChr := #183;
+        $00B8: OutChr := #184;
+        $0105: OutChr := #185;
+        $015F: OutChr := #186;
+        $00BB: OutChr := #187;
+        $013D: OutChr := #188;
+        $02DD: OutChr := #189;
+        $013E: OutChr := #190;
+        $017C: OutChr := #191;
+        $0154: OutChr := #192;
+        $00C1: OutChr := #193;
+        $00C2: OutChr := #194;
+        $0102: OutChr := #195;
+        $00C4: OutChr := #196;
+        $0139: OutChr := #197;
+        $0106: OutChr := #198;
+        $00C7: OutChr := #199;
+        $010C: OutChr := #200;
+        $00C9: OutChr := #201;
+        $0118: OutChr := #202;
+        $00CB: OutChr := #203;
+        $011A: OutChr := #204;
+        $00CD: OutChr := #205;
+        $00CE: OutChr := #206;
+        $010E: OutChr := #207;
+        $0110: OutChr := #208;
+        $0143: OutChr := #209;
+        $0147: OutChr := #210;
+        $00D3: OutChr := #211;
+        $00D4: OutChr := #212;
+        $0150: OutChr := #213;
+        $00D6: OutChr := #214;
+        $00D7: OutChr := #215;
+        $0158: OutChr := #216;
+        $016E: OutChr := #217;
+        $00DA: OutChr := #218;
+        $0170: OutChr := #219;
+        $00DC: OutChr := #220;
+        $00DD: OutChr := #221;
+        $0162: OutChr := #222;
+        $00DF: OutChr := #223;
+        $0155: OutChr := #224;
+        $00E1: OutChr := #225;
+        $00E2: OutChr := #226;
+        $0103: OutChr := #227;
+        $00E4: OutChr := #228;
+        $013A: OutChr := #229;
+        $0107: OutChr := #230;
+        $00E7: OutChr := #231;
+        $010D: OutChr := #232;
+        $00E9: OutChr := #233;
+        $0119: OutChr := #234;
+        $00EB: OutChr := #235;
+        $011B: OutChr := #236;
+        $00ED: OutChr := #237;
+        $00EE: OutChr := #238;
+        $010F: OutChr := #239;
+        $0111: OutChr := #240;
+        $0144: OutChr := #241;
+        $0148: OutChr := #242;
+        $00F3: OutChr := #243;
+        $00F4: OutChr := #244;
+        $0151: OutChr := #245;
+        $00F6: OutChr := #246;
+        $00F7: OutChr := #247;
+        $0159: OutChr := #248;
+        $016F: OutChr := #249;
+        $00FA: OutChr := #250;
+        $0171: OutChr := #251;
+        $00FC: OutChr := #252;
+        $00FD: OutChr := #253;
+        $0163: OutChr := #254;
+        $02D9: OutChr := #255;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1250 Central/Eastern Europe
+ * (used by Ultrastar)
+ *}
+
+type
+  TEncoderCP1250 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1250.GetName(): AnsiString;
+begin
+  Result := 'CP1250';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1250Table: array[128..255] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A,     0, $201E, $2026, $2020, $2021,
+        0, $2030, $0160, $2039, $015A, $0164, $017D, $0179,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+        0, $2122, $0161, $203A, $015B, $0165, $017E, $017A,
+    { $A0 }
+    $00A0, $02C7, $02D8, $0141, $00A4, $0104, $00A6, $00A7,
+    $00A8, $00A9, $015E, $00AB, $00AC, $00AD, $00AE, $017B,
+    { $B0 }
+    $00B0, $00B1, $02DB, $0142, $00B4, $00B5, $00B6, $00B7,
+    $00B8, $0105, $015F, $00BB, $013D, $02DD, $013E, $017C,
+    { $C0 }
+    $0154, $00C1, $00C2, $0102, $00C4, $0139, $0106, $00C7,
+    $010C, $00C9, $0118, $00CB, $011A, $00CD, $00CE, $010E,
+    { $D0 }
+    $0110, $0143, $0147, $00D3, $00D4, $0150, $00D6, $00D7,
+    $0158, $016E, $00DA, $0170, $00DC, $00DD, $0162, $00DF,
+    { $E0 }
+    $0155, $00E1, $00E2, $0103, $00E4, $013A, $0107, $00E7,
+    $010D, $00E9, $0119, $00EB, $011B, $00ED, $00EE, $010F,
+    { $F0 }
+    $0111, $0144, $0148, $00F3, $00F4, $0151, $00F6, $00F7,
+    $0159, $016F, $00FA, $0171, $00FC, $00FD, $0163, $02D9
+  );
+
+function TEncoderCP1250.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1250Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1250.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+        $20AC: OutChr := #128;
+        // invalid: #129
+        $201A: OutChr := #130;
+        // invalid: #131
+        $201E: OutChr := #132;
+        $2026: OutChr := #133;
+        $2020: OutChr := #134;
+        $2021: OutChr := #135;
+        // invalid: #136
+        $2030: OutChr := #137;
+        $0160: OutChr := #138;
+        $2039: OutChr := #139;
+        $015A: OutChr := #140;
+        $0164: OutChr := #141;
+        $017D: OutChr := #142;
+        $0179: OutChr := #143;
+        // invalid: #144
+        $2018: OutChr := #145;
+        $2019: OutChr := #146;
+        $201C: OutChr := #147;
+        $201D: OutChr := #148;
+        $2022: OutChr := #149;
+        $2013: OutChr := #150;
+        $2014: OutChr := #151;
+        // invalid: #152
+        $2122: OutChr := #153;
+        $0161: OutChr := #154;
+        $203A: OutChr := #155;
+        $015B: OutChr := #156;
+        $0165: OutChr := #157;
+        $017E: OutChr := #158;
+        $017A: OutChr := #159;
+        $00A0: OutChr := #160;
+        $02C7: OutChr := #161;
+        $02D8: OutChr := #162;
+        $0141: OutChr := #163;
+        $00A4: OutChr := #164;
+        $0104: OutChr := #165;
+        $00A6: OutChr := #166;
+        $00A7: OutChr := #167;
+        $00A8: OutChr := #168;
+        $00A9: OutChr := #169;
+        $015E: OutChr := #170;
+        $00AB: OutChr := #171;
+        $00AC: OutChr := #172;
+        $00AD: OutChr := #173;
+        $00AE: OutChr := #174;
+        $017B: OutChr := #175;
+        $00B0: OutChr := #176;
+        $00B1: OutChr := #177;
+        $02DB: OutChr := #178;
+        $0142: OutChr := #179;
+        $00B4: OutChr := #180;
+        $00B5: OutChr := #181;
+        $00B6: OutChr := #182;
+        $00B7: OutChr := #183;
+        $00B8: OutChr := #184;
+        $0105: OutChr := #185;
+        $015F: OutChr := #186;
+        $00BB: OutChr := #187;
+        $013D: OutChr := #188;
+        $02DD: OutChr := #189;
+        $013E: OutChr := #190;
+        $017C: OutChr := #191;
+        $0154: OutChr := #192;
+        $00C1: OutChr := #193;
+        $00C2: OutChr := #194;
+        $0102: OutChr := #195;
+        $00C4: OutChr := #196;
+        $0139: OutChr := #197;
+        $0106: OutChr := #198;
+        $00C7: OutChr := #199;
+        $010C: OutChr := #200;
+        $00C9: OutChr := #201;
+        $0118: OutChr := #202;
+        $00CB: OutChr := #203;
+        $011A: OutChr := #204;
+        $00CD: OutChr := #205;
+        $00CE: OutChr := #206;
+        $010E: OutChr := #207;
+        $0110: OutChr := #208;
+        $0143: OutChr := #209;
+        $0147: OutChr := #210;
+        $00D3: OutChr := #211;
+        $00D4: OutChr := #212;
+        $0150: OutChr := #213;
+        $00D6: OutChr := #214;
+        $00D7: OutChr := #215;
+        $0158: OutChr := #216;
+        $016E: OutChr := #217;
+        $00DA: OutChr := #218;
+        $0170: OutChr := #219;
+        $00DC: OutChr := #220;
+        $00DD: OutChr := #221;
+        $0162: OutChr := #222;
+        $00DF: OutChr := #223;
+        $0155: OutChr := #224;
+        $00E1: OutChr := #225;
+        $00E2: OutChr := #226;
+        $0103: OutChr := #227;
+        $00E4: OutChr := #228;
+        $013A: OutChr := #229;
+        $0107: OutChr := #230;
+        $00E7: OutChr := #231;
+        $010D: OutChr := #232;
+        $00E9: OutChr := #233;
+        $0119: OutChr := #234;
+        $00EB: OutChr := #235;
+        $011B: OutChr := #236;
+        $00ED: OutChr := #237;
+        $00EE: OutChr := #238;
+        $010F: OutChr := #239;
+        $0111: OutChr := #240;
+        $0144: OutChr := #241;
+        $0148: OutChr := #242;
+        $00F3: OutChr := #243;
+        $00F4: OutChr := #244;
+        $0151: OutChr := #245;
+        $00F6: OutChr := #246;
+        $00F7: OutChr := #247;
+        $0159: OutChr := #248;
+        $016F: OutChr := #249;
+        $00FA: OutChr := #250;
+        $0171: OutChr := #251;
+        $00FC: OutChr := #252;
+        $00FD: OutChr := #253;
+        $0163: OutChr := #254;
+        $02D9: OutChr := #255;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1250 Central/Eastern Europe
+ * (used by Ultrastar)
+ *}
+
+type
+  TEncoderCP1250 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1250.GetName(): AnsiString;
+begin
+  Result := 'CP1250';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1250Table: array[128..255] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A,     0, $201E, $2026, $2020, $2021,
+        0, $2030, $0160, $2039, $015A, $0164, $017D, $0179,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+        0, $2122, $0161, $203A, $015B, $0165, $017E, $017A,
+    { $A0 }
+    $00A0, $02C7, $02D8, $0141, $00A4, $0104, $00A6, $00A7,
+    $00A8, $00A9, $015E, $00AB, $00AC, $00AD, $00AE, $017B,
+    { $B0 }
+    $00B0, $00B1, $02DB, $0142, $00B4, $00B5, $00B6, $00B7,
+    $00B8, $0105, $015F, $00BB, $013D, $02DD, $013E, $017C,
+    { $C0 }
+    $0154, $00C1, $00C2, $0102, $00C4, $0139, $0106, $00C7,
+    $010C, $00C9, $0118, $00CB, $011A, $00CD, $00CE, $010E,
+    { $D0 }
+    $0110, $0143, $0147, $00D3, $00D4, $0150, $00D6, $00D7,
+    $0158, $016E, $00DA, $0170, $00DC, $00DD, $0162, $00DF,
+    { $E0 }
+    $0155, $00E1, $00E2, $0103, $00E4, $013A, $0107, $00E7,
+    $010D, $00E9, $0119, $00EB, $011B, $00ED, $00EE, $010F,
+    { $F0 }
+    $0111, $0144, $0148, $00F3, $00F4, $0151, $00F6, $00F7,
+    $0159, $016F, $00FA, $0171, $00FC, $00FD, $0163, $02D9
+  );
+
+function TEncoderCP1250.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1250Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1250.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+        $20AC: OutChr := #128;
+        // invalid: #129
+        $201A: OutChr := #130;
+        // invalid: #131
+        $201E: OutChr := #132;
+        $2026: OutChr := #133;
+        $2020: OutChr := #134;
+        $2021: OutChr := #135;
+        // invalid: #136
+        $2030: OutChr := #137;
+        $0160: OutChr := #138;
+        $2039: OutChr := #139;
+        $015A: OutChr := #140;
+        $0164: OutChr := #141;
+        $017D: OutChr := #142;
+        $0179: OutChr := #143;
+        // invalid: #144
+        $2018: OutChr := #145;
+        $2019: OutChr := #146;
+        $201C: OutChr := #147;
+        $201D: OutChr := #148;
+        $2022: OutChr := #149;
+        $2013: OutChr := #150;
+        $2014: OutChr := #151;
+        // invalid: #152
+        $2122: OutChr := #153;
+        $0161: OutChr := #154;
+        $203A: OutChr := #155;
+        $015B: OutChr := #156;
+        $0165: OutChr := #157;
+        $017E: OutChr := #158;
+        $017A: OutChr := #159;
+        $00A0: OutChr := #160;
+        $02C7: OutChr := #161;
+        $02D8: OutChr := #162;
+        $0141: OutChr := #163;
+        $00A4: OutChr := #164;
+        $0104: OutChr := #165;
+        $00A6: OutChr := #166;
+        $00A7: OutChr := #167;
+        $00A8: OutChr := #168;
+        $00A9: OutChr := #169;
+        $015E: OutChr := #170;
+        $00AB: OutChr := #171;
+        $00AC: OutChr := #172;
+        $00AD: OutChr := #173;
+        $00AE: OutChr := #174;
+        $017B: OutChr := #175;
+        $00B0: OutChr := #176;
+        $00B1: OutChr := #177;
+        $02DB: OutChr := #178;
+        $0142: OutChr := #179;
+        $00B4: OutChr := #180;
+        $00B5: OutChr := #181;
+        $00B6: OutChr := #182;
+        $00B7: OutChr := #183;
+        $00B8: OutChr := #184;
+        $0105: OutChr := #185;
+        $015F: OutChr := #186;
+        $00BB: OutChr := #187;
+        $013D: OutChr := #188;
+        $02DD: OutChr := #189;
+        $013E: OutChr := #190;
+        $017C: OutChr := #191;
+        $0154: OutChr := #192;
+        $00C1: OutChr := #193;
+        $00C2: OutChr := #194;
+        $0102: OutChr := #195;
+        $00C4: OutChr := #196;
+        $0139: OutChr := #197;
+        $0106: OutChr := #198;
+        $00C7: OutChr := #199;
+        $010C: OutChr := #200;
+        $00C9: OutChr := #201;
+        $0118: OutChr := #202;
+        $00CB: OutChr := #203;
+        $011A: OutChr := #204;
+        $00CD: OutChr := #205;
+        $00CE: OutChr := #206;
+        $010E: OutChr := #207;
+        $0110: OutChr := #208;
+        $0143: OutChr := #209;
+        $0147: OutChr := #210;
+        $00D3: OutChr := #211;
+        $00D4: OutChr := #212;
+        $0150: OutChr := #213;
+        $00D6: OutChr := #214;
+        $00D7: OutChr := #215;
+        $0158: OutChr := #216;
+        $016E: OutChr := #217;
+        $00DA: OutChr := #218;
+        $0170: OutChr := #219;
+        $00DC: OutChr := #220;
+        $00DD: OutChr := #221;
+        $0162: OutChr := #222;
+        $00DF: OutChr := #223;
+        $0155: OutChr := #224;
+        $00E1: OutChr := #225;
+        $00E2: OutChr := #226;
+        $0103: OutChr := #227;
+        $00E4: OutChr := #228;
+        $013A: OutChr := #229;
+        $0107: OutChr := #230;
+        $00E7: OutChr := #231;
+        $010D: OutChr := #232;
+        $00E9: OutChr := #233;
+        $0119: OutChr := #234;
+        $00EB: OutChr := #235;
+        $011B: OutChr := #236;
+        $00ED: OutChr := #237;
+        $00EE: OutChr := #238;
+        $010F: OutChr := #239;
+        $0111: OutChr := #240;
+        $0144: OutChr := #241;
+        $0148: OutChr := #242;
+        $00F3: OutChr := #243;
+        $00F4: OutChr := #244;
+        $0151: OutChr := #245;
+        $00F6: OutChr := #246;
+        $00F7: OutChr := #247;
+        $0159: OutChr := #248;
+        $016F: OutChr := #249;
+        $00FA: OutChr := #250;
+        $0171: OutChr := #251;
+        $00FC: OutChr := #252;
+        $00FD: OutChr := #253;
+        $0163: OutChr := #254;
+        $02D9: OutChr := #255;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
diff --git a/unicode/src/encoding/CP1252.inc b/unicode/src/encoding/CP1252.inc
new file mode 100644
index 00000000..0ea15819
--- /dev/null
+++ b/unicode/src/encoding/CP1252.inc
@@ -0,0 +1,366 @@
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1252 Western Europe
+ * (used by UltraStar Deluxe < 1.1)
+ *}
+
+type
+  TEncoderCP1252 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1252.GetName(): AnsiString;
+begin
+  Result := 'CP1252';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1252Table: array[128..159] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A, $0192, $201E, $2026, $2020, $2021,
+    $02C6, $2030, $0160, $2039, $0152,     0, $017D,     0,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+    $02DC, $2122, $0161, $203A, $0153,     0, $017E, $0178
+  );
+
+function TEncoderCP1252.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) or (InChr >= #160) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1252Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1252.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) or ((InChr >= 160) and (InChr <= 255)) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+      $20AC: OutChr := #128;
+      // invalid: #129
+      $201A: OutChr := #130;
+      $0192: OutChr := #131;
+      $201E: OutChr := #132;
+      $2026: OutChr := #133;
+      $2020: OutChr := #134;
+      $2021: OutChr := #135;
+      $02C6: OutChr := #136;
+      $2030: OutChr := #137;
+      $0160: OutChr := #138;
+      $2039: OutChr := #139;
+      $0152: OutChr := #140;
+      // invalid: #141
+      $017D: OutChr := #142;
+      // invalid: #143
+      // invalid: #144
+      $2018: OutChr := #145;
+      $2019: OutChr := #146;
+      $201C: OutChr := #147;
+      $201D: OutChr := #148;
+      $2022: OutChr := #149;
+      $2013: OutChr := #150;
+      $2014: OutChr := #151;
+      $02DC: OutChr := #152;
+      $2122: OutChr := #153;
+      $0161: OutChr := #154;
+      $203A: OutChr := #155;
+      $0153: OutChr := #156;
+      // invalid: #157
+      $017E: OutChr := #158;
+      $0178: OutChr := #159;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1252 Western Europe
+ * (used by UltraStar Deluxe < 1.1)
+ *}
+
+type
+  TEncoderCP1252 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1252.GetName(): AnsiString;
+begin
+  Result := 'CP1252';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1252Table: array[128..159] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A, $0192, $201E, $2026, $2020, $2021,
+    $02C6, $2030, $0160, $2039, $0152,     0, $017D,     0,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+    $02DC, $2122, $0161, $203A, $0153,     0, $017E, $0178
+  );
+
+function TEncoderCP1252.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) or (InChr >= #160) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1252Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1252.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) or ((InChr >= 160) and (InChr <= 255)) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+      $20AC: OutChr := #128;
+      // invalid: #129
+      $201A: OutChr := #130;
+      $0192: OutChr := #131;
+      $201E: OutChr := #132;
+      $2026: OutChr := #133;
+      $2020: OutChr := #134;
+      $2021: OutChr := #135;
+      $02C6: OutChr := #136;
+      $2030: OutChr := #137;
+      $0160: OutChr := #138;
+      $2039: OutChr := #139;
+      $0152: OutChr := #140;
+      // invalid: #141
+      $017D: OutChr := #142;
+      // invalid: #143
+      // invalid: #144
+      $2018: OutChr := #145;
+      $2019: OutChr := #146;
+      $201C: OutChr := #147;
+      $201D: OutChr := #148;
+      $2022: OutChr := #149;
+      $2013: OutChr := #150;
+      $2014: OutChr := #151;
+      $02DC: OutChr := #152;
+      $2122: OutChr := #153;
+      $0161: OutChr := #154;
+      $203A: OutChr := #155;
+      $0153: OutChr := #156;
+      // invalid: #157
+      $017E: OutChr := #158;
+      $0178: OutChr := #159;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Windows-1252 Western Europe
+ * (used by UltraStar Deluxe < 1.1)
+ *}
+
+type
+  TEncoderCP1252 = class(TSingleByteEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean; override;
+    function EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean; override;
+  end;
+
+function TEncoderCP1252.GetName(): AnsiString;
+begin
+  Result := 'CP1252';
+end;
+
+const
+  // Positions marked as #0 are invalid.
+  CP1252Table: array[128..159] of UCS4Char = (
+    { $80 }
+    $20AC,     0, $201A, $0192, $201E, $2026, $2020, $2021,
+    $02C6, $2030, $0160, $2039, $0152,     0, $017D,     0,
+    { $90 }
+        0, $2018, $2019, $201C, $201D, $2022, $2013, $2014,
+    $02DC, $2122, $0161, $203A, $0153,     0, $017E, $0178
+  );
+
+function TEncoderCP1252.DecodeChar(InChr: AnsiChar; out OutChr: UCS4Char): boolean;
+begin
+  Result := true;
+  if (InChr < #128) or (InChr >= #160) then
+    OutChr := UCS4Char(Ord(InChr)) // use Ord() to avoid automatic conversion
+  else
+  begin
+    OutChr := CP1252Table[Ord(InChr)];
+    if (OutChr = 0) then
+    begin
+      Result := false;
+      OutChr := Ord(ERROR_CHAR);
+    end;
+  end;
+end;
+
+function TEncoderCP1252.EncodeChar(InChr: UCS4Char; out OutChr: AnsiChar): boolean;
+begin
+  if (InChr < 128) or ((InChr >= 160) and (InChr <= 255)) then
+  begin
+    OutChr := AnsiChar(Ord(InChr));
+    Result := true;
+  end
+  else
+  begin
+    case InChr of
+      $20AC: OutChr := #128;
+      // invalid: #129
+      $201A: OutChr := #130;
+      $0192: OutChr := #131;
+      $201E: OutChr := #132;
+      $2026: OutChr := #133;
+      $2020: OutChr := #134;
+      $2021: OutChr := #135;
+      $02C6: OutChr := #136;
+      $2030: OutChr := #137;
+      $0160: OutChr := #138;
+      $2039: OutChr := #139;
+      $0152: OutChr := #140;
+      // invalid: #141
+      $017D: OutChr := #142;
+      // invalid: #143
+      // invalid: #144
+      $2018: OutChr := #145;
+      $2019: OutChr := #146;
+      $201C: OutChr := #147;
+      $201D: OutChr := #148;
+      $2022: OutChr := #149;
+      $2013: OutChr := #150;
+      $2014: OutChr := #151;
+      $02DC: OutChr := #152;
+      $2122: OutChr := #153;
+      $0161: OutChr := #154;
+      $203A: OutChr := #155;
+      $0153: OutChr := #156;
+      // invalid: #157
+      $017E: OutChr := #158;
+      $0178: OutChr := #159;
+      else begin
+        OutChr := ERROR_CHAR;
+        Result := false;
+        Exit;
+      end;
+    end;
+    Result := true;
+  end;
+end;
+
diff --git a/unicode/src/encoding/Locale.inc b/unicode/src/encoding/Locale.inc
new file mode 100644
index 00000000..b2ad5ee3
--- /dev/null
+++ b/unicode/src/encoding/Locale.inc
@@ -0,0 +1,165 @@
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Locale
+ *}
+
+type
+  TEncoderLocale = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderLocale.GetName(): AnsiString;
+begin
+  Result := 'LOCALE';
+end;
+
+function TEncoderLocale.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+begin
+  OutStr := WideStringToUCS4String(InStr); // use implicit conversion
+  Result := true;
+end;
+
+function TEncoderLocale.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4StringToWideString(InStr); // use implicit conversion
+  // any way to check for errors?
+  Result := true;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Locale
+ *}
+
+type
+  TEncoderLocale = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderLocale.GetName(): AnsiString;
+begin
+  Result := 'LOCALE';
+end;
+
+function TEncoderLocale.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+begin
+  OutStr := WideStringToUCS4String(InStr); // use implicit conversion
+  Result := true;
+end;
+
+function TEncoderLocale.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4StringToWideString(InStr); // use implicit conversion
+  // any way to check for errors?
+  Result := true;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * Locale
+ *}
+
+type
+  TEncoderLocale = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderLocale.GetName(): AnsiString;
+begin
+  Result := 'LOCALE';
+end;
+
+function TEncoderLocale.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+begin
+  OutStr := WideStringToUCS4String(InStr); // use implicit conversion
+  Result := true;
+end;
+
+function TEncoderLocale.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4StringToWideString(InStr); // use implicit conversion
+  // any way to check for errors?
+  Result := true;
+end;
+
diff --git a/unicode/src/encoding/UTF8.inc b/unicode/src/encoding/UTF8.inc
new file mode 100644
index 00000000..86feb8bf
--- /dev/null
+++ b/unicode/src/encoding/UTF8.inc
@@ -0,0 +1,210 @@
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * UTF-8
+ *}
+
+type
+  TEncoderUTF8 = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderUTF8.GetName(): AnsiString;
+begin
+  Result := 'UTF8';
+end;
+
+function TEncoderUTF8.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+var
+  I: integer;
+  StrPtr: PAnsiChar;
+begin
+  // UTF8Decode() may crash with FPC < 2.2.2 if the input string is not UTF-8
+  // encoded. Newer versions do not crash but do not signal errors either.
+  // So let's implement this stuff again.
+  Result := true;
+  SetLength(OutStr, Length(InStr)+1);
+  I := 0;
+  StrPtr := PChar(InStr);
+  while (StrPtr^ <> #0) do
+  begin
+    if (not NextCharUTF8(StrPtr, OutStr[I])) then
+      Result := false;;
+    Inc(I);
+  end;
+  SetLength(OutStr, I+1);
+  OutStr[High(OutStr)] := 0;
+end;
+
+function TEncoderUTF8.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4ToUTF8String(InStr);
+  Result := true;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * UTF-8
+ *}
+
+type
+  TEncoderUTF8 = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderUTF8.GetName(): AnsiString;
+begin
+  Result := 'UTF8';
+end;
+
+function TEncoderUTF8.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+var
+  I: integer;
+  StrPtr: PAnsiChar;
+begin
+  // UTF8Decode() may crash with FPC < 2.2.2 if the input string is not UTF-8
+  // encoded. Newer versions do not crash but do not signal errors either.
+  // So let's implement this stuff again.
+  Result := true;
+  SetLength(OutStr, Length(InStr)+1);
+  I := 0;
+  StrPtr := PChar(InStr);
+  while (StrPtr^ <> #0) do
+  begin
+    if (not NextCharUTF8(StrPtr, OutStr[I])) then
+      Result := false;;
+    Inc(I);
+  end;
+  SetLength(OutStr, I+1);
+  OutStr[High(OutStr)] := 0;
+end;
+
+function TEncoderUTF8.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4ToUTF8String(InStr);
+  Result := true;
+end;
+
+{* UltraStar Deluxe - Karaoke Game
+ *
+ * UltraStar Deluxe is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING. If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ *
+ * $URL$
+ * $Id$
+ *}
+
+{*
+ * UTF-8
+ *}
+
+type
+  TEncoderUTF8 = class(TEncoder)
+  public
+    function GetName(): AnsiString; override;
+    function Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean; override;
+    function Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean; override;
+  end;
+
+function TEncoderUTF8.GetName(): AnsiString;
+begin
+  Result := 'UTF8';
+end;
+
+function TEncoderUTF8.Decode(const InStr: AnsiString; out OutStr: UCS4String): boolean;
+var
+  I: integer;
+  StrPtr: PAnsiChar;
+begin
+  // UTF8Decode() may crash with FPC < 2.2.2 if the input string is not UTF-8
+  // encoded. Newer versions do not crash but do not signal errors either.
+  // So let's implement this stuff again.
+  Result := true;
+  SetLength(OutStr, Length(InStr)+1);
+  I := 0;
+  StrPtr := PChar(InStr);
+  while (StrPtr^ <> #0) do
+  begin
+    if (not NextCharUTF8(StrPtr, OutStr[I])) then
+      Result := false;;
+    Inc(I);
+  end;
+  SetLength(OutStr, I+1);
+  OutStr[High(OutStr)] := 0;
+end;
+
+function TEncoderUTF8.Encode(const InStr: UCS4String; out OutStr: AnsiString): boolean;
+begin
+  OutStr := UCS4ToUTF8String(InStr);
+  Result := true;
+end;
+