added autodetection of utf8

used w3c regex to match all song lines whether they are utf8 lines and
decode it on match as utf8 and as latin1 otherwise

git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/trunk@1964 b956fd51-792f-4845-bead-9b4dfca2ff2c

2 files changed, 17 insertions, 10 deletions

diff --git a/src/base/USong.pas b/src/base/USong.pas
index f8d465c7..f8698d43 100644
--- a/src/base/USong.pas
+++ b/src/base/USong.pas
@@ -114,7 +114,7 @@ type
     Mp3:        IPath;
     Background: IPath;
     Video:      IPath;
-    
+
     // sorting methods
     Genre:      UTF8String;
     Edition:    UTF8String;
@@ -127,7 +127,7 @@ type
     Creator:    UTF8String;
 
     CoverTex:   TTexture;
-    
+
     VideoGAP:   real;
     NotesGAP:   integer;
     Start:      real; // in seconds
@@ -180,8 +180,7 @@ uses
   UNote;   //needed for Player
 
 const
-  // use USDX < 1.1 encoding for backward compatibility
-  DEFAULT_ENCODING = encCP1252;
+  DEFAULT_ENCODING = encAuto;
 
 constructor TSong.Create();
 begin
@@ -381,7 +380,7 @@ begin
     LinePos := OldLinePos;
     raise EUSDXParseException.Create('Character expected');
   end;
-  Result := Str[1];  
+  Result := Str[1];
 end;
 
 {**
@@ -926,7 +925,7 @@ begin
     end
     else
     begin
-    
+
       //-----------
       //Required Attributes
       //-----------
@@ -1083,7 +1082,7 @@ begin
       begin
         AddCustomTag(Identifier, Value);
       end;
-      
+
     end; // End check for non-empty Value
 
     // read next line
@@ -1265,7 +1264,7 @@ begin
   //Sortings:
   Genre    := 'Unknown';
   Edition  := 'Unknown';
-  Language := 'Unknown'; 
+  Language := 'Unknown';
   Year := 0;
 
   // set to default encoding
diff --git a/src/base/UTextEncoding.pas b/src/base/UTextEncoding.pas
index bb3d0f1a..79e5a297 100644
--- a/src/base/UTextEncoding.pas
+++ b/src/base/UTextEncoding.pas
@@ -42,7 +42,9 @@ type
     encLocale,  // current locale (needs cwstring on linux)
     encUTF8,    // UTF-8
     encCP1250,  // Windows-1250 Central/Eastern Europe (used by Ultrastar)
-    encCP1252   // Windows-1252 Western Europe (used by UltraStar Deluxe < 1.1)
+    encCP1252,  // Windows-1252 Western Europe (used by UltraStar Deluxe < 1.1)
+    encAuto     // try to match the w3c regex and decode as unicode on match
+                // and as fallback if not match
   );
 
 const
@@ -88,7 +90,9 @@ function EncodingName(Encoding: TEncoding): AnsiString;
 implementation
 
 uses
-  StrUtils;
+  StrUtils,
+  pcre,
+  ULog;
 
 type
   IEncoder = interface
@@ -229,6 +233,7 @@ end;
 {$I ../encoding/UTF8.inc}
 {$I ../encoding/CP1250.inc}
 {$I ../encoding/CP1252.inc}
+{$I ../encoding/Auto.inc}
 
 initialization
   Encoders[encLocale] := TEncoderLocale.Create;
@@ -236,4 +241,7 @@ initialization
   Encoders[encCP1250] := TEncoderCP1250.Create;
   Encoders[encCP1252] := TEncoderCP1252.Create;
 
+  // use USDX < 1.1 encoding for backward compatibility (encCP1252)
+  Encoders[encAuto] := TEncoderAuto.Create(Encoders[encUTF8], Encoders[encCP1252]);
+
 end.