Unicode Text-file support

- lyrics, song-title, artist, etc. are now UTF8 strings
- filenames need some work (UTF-8 on Mac, otherwise still native encoding. Should always be UTF-8 instead)

File encoding:
- if the file starts with a UTF-8 BOM, UTF-8 is assumed
- if the file contains an encoding header (#ENCODING: UTF8/CP1252/CP1250), the given encoding is used
- if neither of the previous applies the default encoding (stored in the config-file, default CP1252 = USDX 1.01a encoding) is used

git-svn-id: svn://svn.code.sf.net/p/ultrastardx/svn/branches/experimental@1510 b956fd51-792f-4845-bead-9b4dfca2ff2c

4 files changed, 390 insertions, 305 deletions

diff --git a/unicode/src/base/UDataBase.pas b/unicode/src/base/UDataBase.pas
index 0f9d88a7..87e9519c 100644
--- a/unicode/src/base/UDataBase.pas
+++ b/unicode/src/base/UDataBase.pas
@@ -58,29 +58,29 @@ type
 
   TStatResultBestScores = class(TStatResult)
     public
-      Singer:       WideString;
+      Singer:       UTF8String;
       Score:        Word;
       Difficulty:   Byte;
-      SongArtist:   WideString;
-      SongTitle:    WideString;
+      SongArtist:   UTF8String;
+      SongTitle:    UTF8String;
   end;
 
   TStatResultBestSingers = class(TStatResult)
     public
-      Player:       WideString;
+      Player:       UTF8String;
       AverageScore: Word;
   end;
 
   TStatResultMostSungSong = class(TStatResult)
     public
-      Artist:       WideString;
-      Title:        WideString;
+      Artist:       UTF8String;
+      Title:        UTF8String;
       TimesSung:    Word;
   end;
 
   TStatResultMostPopBand = class(TStatResult)
     public
-      ArtistName:   WideString;
+      ArtistName:   UTF8String;
       TimesSungTot: Word;
   end;
 
@@ -99,7 +99,7 @@ type
 
       procedure Init(const Filename: string);
       procedure ReadScore(Song: TSong);
-      procedure AddScore(Song: TSong; Level: integer; const Name: WideString; Score: integer);
+      procedure AddScore(Song: TSong; Level: integer; const Name: UTF8String; Score: integer);
       procedure WriteScore(Song: TSong);
 
       function GetStats(Typ: TStatType; Count: Byte; Page: Cardinal; Reversed: Boolean): TList;
@@ -237,7 +237,7 @@ begin
         'WHERE [Artist] = ? AND [Title] = ? ' +
         'LIMIT 1) ' +
       'ORDER BY [Score] DESC  LIMIT 15',
-      [UTF8Encode(Song.Artist), UTF8Encode(Song.Title)]);
+      [Song.Artist, Song.Title]);
 
     // Empty Old Scores
     SetLength(Song.Score[0], 0);
@@ -255,7 +255,7 @@ begin
         SetLength(Song.Score[Difficulty], Length(Song.Score[Difficulty]) + 1);
 
         Song.Score[Difficulty, High(Song.Score[Difficulty])].Name  :=
-            UTF8Decode(TableData.FieldByName['Player']);
+            TableData.FieldByName['Player'];
         Song.Score[Difficulty, High(Song.Score[Difficulty])].Score :=
             TableData.FieldAsInteger(TableData.FieldIndex['Score']);
       end;
@@ -277,7 +277,7 @@ end;
 (**
  * Adds one new score to DB
  *)
-procedure TDataBaseSystem.AddScore(Song: TSong; Level: integer; const Name: WideString; Score: integer);
+procedure TDataBaseSystem.AddScore(Song: TSong; Level: integer; const Name: UTF8String; Score: integer);
 var
   ID: Integer;
   TableData: TSQLiteTable;
@@ -296,7 +296,7 @@ begin
     ID := ScoreDB.GetTableValue(
         'SELECT [ID] FROM ['+cUS_Songs+'] ' +
         'WHERE [Artist] = ? AND [Title] = ?',
-        [UTF8Encode(Song.Artist), UTF8Encode(Song.Title)]);
+        [Song.Artist, Song.Title]);
     if (ID = 0) then
     begin
       // Create song if it does not exist
@@ -304,7 +304,7 @@ begin
           'INSERT INTO ['+cUS_Songs+'] ' +
           '([ID], [Artist], [Title], [TimesPlayed]) VALUES ' +
           '(NULL, ?, ?, 0);',
-          [UTF8Encode(Song.Artist), UTF8Encode(Song.Title)]);
+          [Song.Artist, Song.Title]);
       // Get song-ID
       ID := ScoreDB.GetLastInsertRowID();
     end;
@@ -313,7 +313,7 @@ begin
         'INSERT INTO ['+cUS_Scores+'] ' +
         '([SongID] ,[Difficulty], [Player], [Score]) VALUES ' +
         '(?, ?, ?, ?);',
-        [ID, Level, UTF8Encode(Name), Score]);
+        [ID, Level, Name, Score]);
 
     // Delete last position when there are more than 5 entrys.
     // Fixes crash when there are > 5 ScoreEntrys
@@ -364,7 +364,7 @@ begin
         'UPDATE ['+cUS_Songs+'] ' +
         'SET [TimesPlayed] = [TimesPlayed] + 1 ' +
         'WHERE [Title] = ? AND [Artist] = ?;',
-        [UTF8Encode(Song.Title), UTF8Encode(Song.Artist)]);
+        [Song.Title, Song.Artist]);
   except on E: Exception do
     Log.LogError(E.Message, 'TDataBaseSystem.WriteScore');
   end;
@@ -437,18 +437,18 @@ begin
         Stat := TStatResultBestScores.Create;
         with TStatResultBestScores(Stat) do
         begin
-          Singer := UTF8Decode(TableData.Fields[0]);
+          Singer := TableData.Fields[0];
           Difficulty := TableData.FieldAsInteger(1);
           Score := TableData.FieldAsInteger(2);
-          SongArtist := UTF8Decode(TableData.Fields[3]);
-          SongTitle := UTF8Decode(TableData.Fields[4]);
+          SongArtist := TableData.Fields[3];
+          SongTitle := TableData.Fields[4];
         end;
       end;
       stBestSingers: begin
         Stat := TStatResultBestSingers.Create;
         with TStatResultBestSingers(Stat) do
         begin
-          Player := UTF8Decode(TableData.Fields[0]);
+          Player := TableData.Fields[0];
           AverageScore := TableData.FieldAsInteger(1);
         end;
       end;
@@ -456,8 +456,8 @@ begin
         Stat := TStatResultMostSungSong.Create;
         with TStatResultMostSungSong(Stat) do
         begin
-          Artist := UTF8Decode(TableData.Fields[0]);
-          Title  := UTF8Decode(TableData.Fields[1]);
+          Artist := TableData.Fields[0];
+          Title  := TableData.Fields[1];
           TimesSung  := TableData.FieldAsInteger(2);
         end;
       end;
@@ -465,7 +465,7 @@ begin
         Stat := TStatResultMostPopBand.Create;
         with TStatResultMostPopBand(Stat) do
         begin
-          ArtistName := UTF8Decode(TableData.Fields[0]);
+          ArtistName := TableData.Fields[0];
           TimesSungTot := TableData.FieldAsInteger(1);
         end;
       end
diff --git a/unicode/src/base/UIni.pas b/unicode/src/base/UIni.pas
index f50f95ce..2bf49d9d 100644
--- a/unicode/src/base/UIni.pas
+++ b/unicode/src/base/UIni.pas
@@ -37,6 +37,7 @@ uses
   Classes,
   IniFiles,
   ULog,
+  UTextEncoding,
   SysUtils;
 
 type
@@ -84,11 +85,11 @@ type
       procedure LoadScreenModes(IniFile: TCustomIniFile);
 
     public
-      Name:           array[0..11] of string;
+      Name:           array[0..11] of UTF8String;
 
       // Templates for Names Mod
-      NameTeam:       array[0..2] of string;
-      NameTemplate:   array[0..11] of string;
+      NameTeam:       array[0..2] of UTF8String;
+      NameTemplate:   array[0..11] of UTF8String;
 
       //Filename of the opened iniFile
       Filename:       string;
@@ -155,6 +156,9 @@ type
       // Controller
       Joypad:         integer;
 
+      // default encoding for texts (lyrics, song-name, ...)
+      EncodingDefault: TEncoding;
+
       procedure Load();
       procedure Save();
       procedure SaveNames;
@@ -724,6 +728,10 @@ begin
   // NoteLines
   NoteLines := GetArrayIndex(INoteLines, IniFile.ReadString('Lyrics', 'NoteLines', INoteLines[1]));
 
+  //Encoding default
+  // CP1252 is the USDX <1.1 default encoding
+  EncodingDefault := ParseEncoding(IniFile.ReadString('Lyrics', 'Encoding', ''), encCP1252);
+
   LoadThemes(IniFile);
 
   // Color
@@ -872,6 +880,9 @@ begin
   // NoteLines
   IniFile.WriteString('Lyrics', 'NoteLines', INoteLines[NoteLines]);
 
+  //Encoding default
+  IniFile.WriteString('Lyrics', 'Encoding', EncodingNames[EncodingDefault]);
+
   // Theme
   IniFile.WriteString('Themes', 'Theme', ITheme[Theme]);
 
diff --git a/unicode/src/base/USong.pas b/unicode/src/base/USong.pas
index cff56c1d..ac134350 100644
--- a/unicode/src/base/USong.pas
+++ b/unicode/src/base/USong.pas
@@ -56,7 +56,8 @@ uses
     PseudoThread,
   {$ENDIF}
   UCatCovers,
-  UXMLSong;
+  UXMLSong,
+  UTextEncoding;
 
 type
 
@@ -68,15 +69,16 @@ type
   end;
 
   TScore = record
-    Name:       WideString;
+    Name:       UTF8String;
     Score:      integer;
-    Length:     string;
   end;
 
   TSong = class
     FileLineNo  : integer;  //Line which is readed at Last, for error reporting
 
-    procedure ParseNote(LineNumber: integer; TypeP: char; StartP, DurationP, NoteP: integer; LyricS: string);
+    function EncodeFilename(Filename: string): string;
+    function Solmizate(Note: integer; Type_: integer): string;
+    procedure ParseNote(LineNumber: integer; TypeP: char; StartP, DurationP, NoteP: integer; LyricS: UTF8String);
     procedure NewSentence(LineNumberP: integer; Param1, Param2: integer);
 
     function ReadTXTHeader( const aFileName : WideString ): boolean;
@@ -87,23 +89,24 @@ type
     fFileName,
     FileName:   WideString;
 
+    // filenames
+    Cover:      WideString;
+    Mp3:        WideString;
+    Background: WideString;
+    Video:      WideString;
+    
     // sorting methods
-    Category:   array of WideString; // TODO: do we need this?
-    Genre:      WideString;
-    Edition:    WideString;
-    Language:   WideString;
+    Genre:      UTF8String;
+    Edition:    UTF8String;
+    Language:   UTF8String;
 
-    Title:      WideString;
-    Artist:     WideString;
+    Title:      UTF8String;
+    Artist:     UTF8String;
 
-    Text:       WideString;
-    Creator:    WideString;
+    Creator:    UTF8String;
 
-    Cover:      WideString;
     CoverTex:   TTexture;
-    Mp3:        WideString;
-    Background: WideString;
-    Video:      WideString;
+    
     VideoGAP:   real;
     NotesGAP:   integer;
     Start:      real; // in seconds
@@ -113,6 +116,8 @@ type
     BPM:        array of TBPM;
     GAP:        real; // in miliseconds
 
+    Encoding:   TEncoding;
+
     Score:      array[0..2] of array of TScore;
 
     // these are used when sorting is enabled
@@ -129,13 +134,13 @@ type
     Mult    : integer;
     MultBPM : integer;
 
-    LastError: String;
+    LastError: AnsiString;
     Function  GetErrorLineNo: Integer;
     Property  ErrorLineNo: Integer read GetErrorLineNo;
 
 
-    constructor Create  (); overload;
-    constructor Create  ( const aFileName : WideString ); overload;
+    constructor Create(); overload;
+    constructor Create( const aFileName : WideString ); overload;
     function    LoadSong: boolean;
     function    LoadXMLSong: boolean;
     function    Analyse(): boolean;
@@ -185,21 +190,32 @@ begin
   end;
 end;
 
+function TSong.EncodeFilename(Filename: string): string;
+begin
+  {$IFDEF UTF8_FILENAMES}
+  Result := RecodeStringUTF8(Filename, Encoding);
+  {$ELSE}
+  // FIXME: just for compatibility, should be UTF-8 in general
+  if (Encoding = encUTF8) then
+    Result := UTF8ToAnsi(Filename)
+  else
+    Result := Filename;
+  {$ENDIF}
+end;
+
 //Load TXT Song
 function TSong.LoadSong(): boolean;
-
 var
   TempC:    char;
-  Text:     string;
+  Text:     UTF8String;
   CP:       integer; // Current Player (0 or 1)
   Count:    integer;
   Both:     boolean;
   Param1:   integer;
   Param2:   integer;
   Param3:   integer;
-  ParamS:   string;
+  ParamS:   UTF8String;
   I:        integer;
-
 begin
   Result := false;
   LastError := '';
@@ -242,7 +258,7 @@ begin
       ReadLn(SongFile, Text);
       Inc(FileLineNo);
 
-      if (EoF(SongFile)) then
+      if (Eof(SongFile)) then
       begin //Song File Corrupted - No Notes
         CloseFile(SongFile);
         Log.LogError('Could not load txt File, no Notes found: ' + FileName);
@@ -342,7 +358,7 @@ begin
       else
       begin
         for Count := 0 to High(Lines) do
-	      begin
+        begin
           Lines[Count].Line[Lines[Count].High].BaseNote := Base[Count];
           Lines[Count].Line[Lines[Count].High].LyricWidth := glTextWidth(Lines[Count].Line[Lines[Count].High].Lyric);
           //Total Notes Patch
@@ -361,7 +377,7 @@ begin
 
       Read(SongFile, TempC);
       Inc(FileLineNo);
-    end; // while}
+    end; // while
 
     CloseFile(SongFile);
 
@@ -408,7 +424,6 @@ end;
 
 //Load XML Song
 function TSong.LoadXMLSong(): boolean;
-
 var
   //TempC:     char;
   Text:      string;
@@ -425,7 +440,6 @@ var
   NoteType:  char;
   SentenceEnd, Rest, Time: integer;
   Parser: TParser;
-  
 begin
   Result := false;
   LastError := '';
@@ -581,13 +595,9 @@ begin
 end;
 
 function TSong.ReadXMLHeader(const aFileName : WideString): boolean;
-
 var
-  //Line, Identifier, Value: string;
-  //Temp        : word;
   Done        : byte;
   Parser      : TParser;
-
 begin
   Result := true;
   Done   := 0;
@@ -647,7 +657,7 @@ begin
     //    self.Video := Value
 
     // Video Gap
-    //  self.VideoGAP := song_StrtoFloat( Value )
+    //  self.VideoGAP := StrtoFloatI18n( Value )
 
     //Genre Sorting
     self.Genre := Parser.SongInfo.Header.Genre;
@@ -686,25 +696,26 @@ end;
 
 function TSong.ReadTXTHeader(const aFileName : WideString): boolean;
 
-  function song_StrtoFloat( aValue : string ) : Extended;
-
+  {**
+   * "International" StrToFloat variant. Uses either ',' or '.' as decimal
+   * separator. 
+   *}
+  function StrToFloatI18n(const Value: string): Extended;
   var
-    lValue : string;
-
+    TempValue : string;
   begin
-    lValue := aValue;
-
-    if (Pos(',', lValue) <> 0) then
-      lValue[Pos(',', lValue)] := '.';
-
-    Result := StrToFloatDef(lValue, 0);
+    TempValue := Value;
+    if (Pos(',', TempValue) <> 0) then
+      TempValue[Pos(',', TempValue)] := '.';
+    Result := StrToFloatDef(TempValue, 0);
   end;
 
 var
-  Line, Identifier, Value: string;
-  Temp        : word;
-  Done        : byte;
-
+  Line, Identifier: string;
+  Value: string;
+  SepPos: integer; // separator position
+  Done: byte;      // bit-vector of mandatory fields
+  EncFile: string; // encoded filename
 begin
   Result := true;
   Done   := 0;
@@ -719,153 +730,188 @@ begin
     Exit;
   end;
 
+  // check if file begins with a UTF-8 BOM, if so set encoding to UTF-8
+  if (CheckReplaceUTF8BOM(Line)) then
+    Encoding := encUTF8;
+
   //Read Lines while Line starts with # or its empty
   while (Length(Line) = 0) or
         (Line[1] = '#') do
   begin
     //Increase Line Number
     Inc (FileLineNo);
-    Temp := Pos(':', Line);
+    SepPos := Pos(':', Line);
 
-    //Line has a Seperator-> Headerline
-    if (Temp <> 0) then
-    begin
-      //Read Identifier and Value
-      Identifier  := Uppercase(Trim(Copy(Line, 2, Temp - 2))); //Uppercase is for Case Insensitive Checks
-      Value       := Trim(Copy(Line, Temp + 1,Length(Line) - Temp));
+    //Line has no Seperator, ignore non header field
+    if (SepPos = 0) then
+      Continue;
 
-      //Check the Identifier (If Value is given)
-      if (Length(Value) <> 0) then
-      begin
-        //-----------
-        //Required Attributes
-        //-----------
+    //Read Identifier and Value
+    Identifier  := UpperCase(Trim(Copy(Line, 2, SepPos - 2))); //Uppercase is for Case Insensitive Checks
+    Value       := Trim(Copy(Line, SepPos + 1, Length(Line) - SepPos));
 
-        {$IFDEF UTF8_FILENAMES}
-        if ((Identifier = 'MP3') or (Identifier = 'BACKGROUND') or (Identifier = 'COVER') or (Identifier = 'VIDEO')) then
-          Value := Utf8Encode(Value);
-        {$ENDIF}
+    //Check the Identifier (If Value is given)
+    if (Length(Value) = 0) then
+      Continue;
 
-        //Title
-        if (Identifier = 'TITLE') then
-        begin
-          self.Title := Value;
+    //-----------
+    //Required Attributes
+    //-----------
 
-          //Add Title Flag to Done
-          Done := Done or 1;
-        end
+    if (Identifier = 'TITLE') then
+    begin
+      self.Title := RecodeStringUTF8(Value, Encoding);
 
-        //Artist
-        else if (Identifier = 'ARTIST') then
-        begin
-          self.Artist := Value;
+      //Add Title Flag to Done
+      Done := Done or 1;
+    end
 
-          //Add Artist Flag to Done
-          Done := Done or 2;
-        end
+    else if (Identifier = 'ARTIST') then
+    begin
+      self.Artist := RecodeStringUTF8(Value, Encoding);
 
-        //MP3 File //Test if Exists
-        else if (Identifier = 'MP3') AND
-            (FileExists(self.Path + Value)) then
-        begin
-          self.Mp3 := Value;
+      //Add Artist Flag to Done
+      Done := Done or 2;
+    end
 
-          //Add Mp3 Flag to Done
-          Done := Done or 4;
-        end
+    //MP3 File
+    else if (Identifier = 'MP3') then
+    begin
+      EncFile := EncodeFilename(Value);
+      if (FileExists(self.Path + EncFile)) then
+      begin
+        self.Mp3 := EncFile;
 
-        //Beats per Minute
-        else if (Identifier = 'BPM') then
-        begin
-          SetLength(self.BPM, 1);
-          self.BPM[0].StartBeat := 0;
+        //Add Mp3 Flag to Done
+        Done := Done or 4;
+      end;
+    end
 
-          self.BPM[0].BPM := song_StrtoFloat( Value ) * Mult * MultBPM;
+    //Beats per Minute
+    else if (Identifier = 'BPM') then
+    begin
+      SetLength(self.BPM, 1);
+      self.BPM[0].StartBeat := 0;
 
-          if self.BPM[0].BPM <> 0 then
-          begin
-            //Add BPM Flag to Done
-            Done := Done or 8;
-          end;
-        end
+      self.BPM[0].BPM := StrToFloatI18n( Value ) * Mult * MultBPM;
+
+      if self.BPM[0].BPM <> 0 then
+      begin
+        //Add BPM Flag to Done
+        Done := Done or 8;
+      end;
+    end
 
-        //---------
-        //Additional Header Information
-        //---------
+    //---------
+    //Additional Header Information
+    //---------
 
-        // Gap
-        else if (Identifier = 'GAP') then
-          self.GAP := song_StrtoFloat( Value )
+    // Gap
+    else if (Identifier = 'GAP') then
+    begin
+      self.GAP := StrToFloatI18n(Value);
+    end
 
-        //Cover Picture
-        else if (Identifier = 'COVER') then
-          self.Cover := Value
+    //Cover Picture
+    else if (Identifier = 'COVER') then
+    begin
+      self.Cover := EncodeFilename(Value);
+    end
 
-        //Background Picture
-        else if (Identifier = 'BACKGROUND') then
-          self.Background := Value
+    //Background Picture
+    else if (Identifier = 'BACKGROUND') then
+    begin
+      self.Background := EncodeFilename(Value);
+    end
 
-        // Video File
-        else if (Identifier = 'VIDEO') then
-        begin
-          if (FileExists(self.Path + Value)) then
-            self.Video := Value
-          else
-            Log.LogError('Can''t find Video File in Song: ' + aFileName);
-        end
+    // Video File
+    else if (Identifier = 'VIDEO') then
+    begin
+      EncFile := EncodeFilename(Value);
+      if (FileExists(self.Path + EncFile)) then
+        self.Video := EncFile
+      else
+        Log.LogError('Can''t find Video File in Song: ' + aFileName);
+    end
 
-        // Video Gap
-        else if (Identifier = 'VIDEOGAP') then
-          self.VideoGAP := song_StrtoFloat( Value )
+    // Video Gap
+    else if (Identifier = 'VIDEOGAP') then
+    begin
+      self.VideoGAP := StrToFloatI18n( Value )
+    end
 
-        //Genre Sorting
-        else if (Identifier = 'GENRE') then
-          self.Genre := Value
+    //Genre Sorting
+    else if (Identifier = 'GENRE') then
+    begin
+      self.Genre := RecodeStringUTF8(Value, Encoding)
+    end
 
-        //Edition Sorting
-        else if (Identifier = 'EDITION') then
-          self.Edition := Value
+    //Edition Sorting
+    else if (Identifier = 'EDITION') then
+    begin
+      self.Edition := RecodeStringUTF8(Value, Encoding)
+    end
+
+    //Creator Tag
+    else if (Identifier = 'CREATOR') then
+    begin
+      self.Creator := RecodeStringUTF8(Value, Encoding)
+    end
 
-        //Creator Tag
-        else if (Identifier = 'CREATOR') then
-          self.Creator := Value
+    //Language Sorting
+    else if (Identifier = 'LANGUAGE') then
+    begin
+      self.Language := RecodeStringUTF8(Value, Encoding)
+    end
 
-        //Language Sorting
-        else if (Identifier = 'LANGUAGE') then
-          self.Language := Value
+    // Song Start
+    else if (Identifier = 'START') then
+    begin
+      self.Start := StrToFloatI18n( Value )
+    end
 
-        // Song Start
-        else if (Identifier = 'START') then
-          self.Start := song_StrtoFloat( Value )
+    // Song Ending
+    else if (Identifier = 'END') then
+    begin
+      TryStrtoInt(Value, self.Finish)
+    end
 
-        // Song Ending
-        else if (Identifier = 'END') then
-          TryStrtoInt(Value, self.Finish)
+    // Resolution
+    else if (Identifier = 'RESOLUTION') then
+    begin
+      TryStrtoInt(Value, self.Resolution)
+    end
 
-        // Resolution
-        else if (Identifier = 'RESOLUTION') then
-          TryStrtoInt(Value, self.Resolution)
+    // Notes Gap
+    else if (Identifier = 'NOTESGAP') then
+    begin
+      TryStrtoInt(Value, self.NotesGAP)
+    end
 
-        // Notes Gap
-        else if (Identifier = 'NOTESGAP') then
-          TryStrtoInt(Value, self.NotesGAP)
-        // Relative Notes
-        else if (Identifier = 'RELATIVE') AND (uppercase(Value) = 'YES') then
-          self.Relative := True;
+    // Relative Notes
+    else if (Identifier = 'RELATIVE') then
+    begin
+      if (UpperCase(Value) = 'YES') then
+        self.Relative := True;
+    end
 
-      end;
+    // File encoding
+    else if (Identifier = 'ENCODING') then
+    begin
+      self.Encoding := ParseEncoding(Value, Ini.EncodingDefault);
     end;
 
-    if not EOf(SongFile) then
-      ReadLn (SongFile, Line)
-    else
+    // check for end of file
+    if Eof(SongFile) then
     begin
       Result := False;
       Log.LogError('File Incomplete or not Ultrastar TxT (A): ' + aFileName);
-      break;
+      Break;
     end;
 
-  end;
+    // read next line
+    ReadLn(SongFile, Line)
+  end; // while
 
   if self.Cover = '' then
     self.Cover := platform.FindSongFile(Path, '*[CO].jpg');
@@ -896,47 +942,52 @@ begin
     Result := -1;
 end;
 
-procedure TSong.ParseNote(LineNumber: integer; TypeP: char; StartP, DurationP, NoteP: integer; LyricS: string);
-
+function TSong.Solmizate(Note: integer; Type_: integer): string;
 begin
-  case Ini.Solmization of
+  case (Type_) of
     1:  // european
       begin
-        case (NoteP mod 12) of
-          0..1:   LyricS := ' do ';
-          2..3:   LyricS := ' re ';
-          4:      LyricS := ' mi ';
-          5..6:   LyricS := ' fa ';
-          7..8:   LyricS := ' sol ';
-          9..10:  LyricS := ' la ';
-          11:     LyricS := ' si ';
+        case (Note mod 12) of
+          0..1:   Result := ' do ';
+          2..3:   Result := ' re ';
+          4:      Result := ' mi ';
+          5..6:   Result := ' fa ';
+          7..8:   Result := ' sol ';
+          9..10:  Result := ' la ';
+          11:     Result := ' si ';
         end;
       end;
     2:  // japanese
       begin
-        case (NoteP mod 12) of
-          0..1:   LyricS := ' do ';
-          2..3:   LyricS := ' re ';
-          4:      LyricS := ' mi ';
-          5..6:   LyricS := ' fa ';
-          7..8:   LyricS := ' so ';
-          9..10:  LyricS := ' la ';
-          11:     LyricS := ' shi ';
+        case (Note mod 12) of
+          0..1:   Result := ' do ';
+          2..3:   Result := ' re ';
+          4:      Result := ' mi ';
+          5..6:   Result := ' fa ';
+          7..8:   Result := ' so ';
+          9..10:  Result := ' la ';
+          11:     Result := ' shi ';
         end;
       end;
     3:  // american
       begin
-        case (NoteP mod 12) of
-          0..1:   LyricS := ' do ';
-          2..3:   LyricS := ' re ';
-          4:      LyricS := ' mi ';
-          5..6:   LyricS := ' fa ';
-          7..8:   LyricS := ' sol ';
-          9..10:  LyricS := ' la ';
-          11:     LyricS := ' ti ';
+        case (Note mod 12) of
+          0..1:   Result := ' do ';
+          2..3:   Result := ' re ';
+          4:      Result := ' mi ';
+          5..6:   Result := ' fa ';
+          7..8:   Result := ' sol ';
+          9..10:  Result := ' la ';
+          11:     Result := ' ti ';
         end;
       end;
   end; // case
+end;
+
+procedure TSong.ParseNote(LineNumber: integer; TypeP: char; StartP, DurationP, NoteP: integer; LyricS: UTF8String);
+begin
+  if (Ini.Solmization <> 0) then
+    LyricS := Solmizate(NoteP, Ini.Solmization);
 
   with Lines[LineNumber].Line[Lines[LineNumber].High] do
   begin
@@ -969,7 +1020,9 @@ begin
     Note[HighNote].Tone := NoteP;
     if Note[HighNote].Tone < Base[LineNumber] then Base[LineNumber] := Note[HighNote].Tone;
 
-    Note[HighNote].Text := Copy(LyricS, 2, 100);
+    Note[HighNote].Text := RecodeStringUTF8(
+      Copy(LyricS, 2, Length(LyricS)-1),
+      Encoding);
     Lyric := Lyric + Note[HighNote].Text;
 
     End_ := Note[HighNote].Start + Note[HighNote].Length;
@@ -977,10 +1030,8 @@ begin
 end;
 
 procedure TSong.NewSentence(LineNumberP: integer; Param1, Param2: integer);
-
 var
   I: integer;
-
 begin
 
   If (Lines[LineNumberP].Line[Lines[LineNumberP].High].HighNote  <> -1) then
@@ -1039,6 +1090,9 @@ begin
   Edition  := 'Unknown';
   Language := 'Unknown'; //Language Patch
 
+  // set to default encoding
+  Encoding := Ini.EncodingDefault;
+
   //Required Information
   Mp3    := '';
   {$IFDEF FPC}
diff --git a/unicode/src/base/USongs.pas b/unicode/src/base/USongs.pas
index 6f66bf3f..be880433 100644
--- a/unicode/src/base/USongs.pas
+++ b/unicode/src/base/USongs.pas
@@ -156,7 +156,8 @@ uses StrUtils,
      UCovers,
      UFiles,
      UMain,
-     UIni;
+     UIni,
+     UUnicodeUtils;
 
 constructor TSongs.Create();
 begin
@@ -459,14 +460,14 @@ procedure TCatSongs.Refresh;
 var
   SongIndex:   integer;
   CurSong:     TSong;
-  CatIndex:    integer; // index of current song in Song
-  Letter:      char;    // current letter for sorting using letter
-  CurCategory: string;  // current edition for sorting using edition, genre etc.
-  Order:       integer; // number used for ordernum
-  LetterTmp:   char;
-  CatNumber:   integer; // Number of Song in Category
-
-  procedure AddCategoryButton(const CategoryName: string);
+  CatIndex:    integer;    // index of current song in Song
+  Letter:      UCS4Char;   // current letter for sorting using letter
+  CurCategory: UTF8String; // current edition for sorting using edition, genre etc.
+  Order:       integer;    // number used for ordernum
+  LetterTmp:   UCS4Char;
+  CatNumber:   integer;    // Number of Song in Category
+
+  procedure AddCategoryButton(const CategoryName: UTF8String);
   var
     PrevCatBtnIndex: integer;
   begin
@@ -501,7 +502,7 @@ begin
   // Note: do NOT set Letter to ' ', otherwise no category-button will be
   // created for songs beginning with ' ' if songs of this category exist.
   // TODO: trim song-properties so ' ' will not occur as first chararcter. 
-  Letter := #0;
+  Letter := 0;
 
   // clear song-list
   for SongIndex := 0 to Songs.SongList.Count-1 do
@@ -520,101 +521,120 @@ begin
     // if tabs are on, add section buttons for each new section
     if (Ini.Tabs = 1) then
     begin
-      if (Ini.Sorting = sEdition) and
-         (CompareText(CurCategory, CurSong.Edition) <> 0) then
-      begin
-        CurCategory := CurSong.Edition;
-
-        // TODO: remove this block if it is not needed anymore
-        {
-        if CurSection = 'Singstar Part 2' then CoverName := 'Singstar';
-        if CurSection = 'Singstar German' then CoverName := 'Singstar';
-        if CurSection = 'Singstar Spanish' then CoverName := 'Singstar';
-        if CurSection = 'Singstar Italian' then CoverName := 'Singstar';
-        if CurSection = 'Singstar French' then CoverName := 'Singstar';
-        if CurSection = 'Singstar 80s Polish' then CoverName := 'Singstar 80s';
-        }
-
-        // add Category Button
-        AddCategoryButton(CurCategory);
-      end
-
-      else if (Ini.Sorting = sGenre) and
-              (CompareText(CurCategory, CurSong.Genre) <> 0) then
-      begin
-        CurCategory := CurSong.Genre;
-        // add Genre Button
-        AddCategoryButton(CurCategory);
-      end
-
-      else if (Ini.Sorting = sLanguage) and
-              (CompareText(CurCategory, CurSong.Language) <> 0) then
-      begin
-        CurCategory := CurSong.Language;
-        // add Language Button
-        AddCategoryButton(CurCategory);
-      end
+      case (Ini.Sorting) of
+        sEdition: begin
+          if (CompareText(CurCategory, CurSong.Edition) <> 0) then
+          begin
+            CurCategory := CurSong.Edition;
+
+            // TODO: remove this block if it is not needed anymore
+            {
+            if CurSection = 'Singstar Part 2' then CoverName := 'Singstar';
+            if CurSection = 'Singstar German' then CoverName := 'Singstar';
+            if CurSection = 'Singstar Spanish' then CoverName := 'Singstar';
+            if CurSection = 'Singstar Italian' then CoverName := 'Singstar';
+            if CurSection = 'Singstar French' then CoverName := 'Singstar';
+            if CurSection = 'Singstar 80s Polish' then CoverName := 'Singstar 80s';
+            }
+
+            // add Category Button
+            AddCategoryButton(CurCategory);
+          end;
+        end;
 
-      else if (Ini.Sorting = sTitle) and
-              (Length(CurSong.Title) >= 1) and
-              (Letter <> UpperCase(CurSong.Title)[1]) then
-      begin
-        Letter := Uppercase(CurSong.Title)[1];
-        // add a letter Category Button
-        AddCategoryButton(Letter);
-      end
+        sGenre: begin
+          if (CompareText(CurCategory, CurSong.Genre) <> 0) then
+          begin
+            CurCategory := CurSong.Genre;
+            // add Genre Button
+            AddCategoryButton(CurCategory);
+          end;
+        end;
 
-      else if (Ini.Sorting = sArtist) and
-              (Length(CurSong.Artist) >= 1) and
-              (Letter <> UpperCase(CurSong.Artist)[1]) then
-      begin
-        Letter := UpperCase(CurSong.Artist)[1];
-        // add a letter Category Button
-        AddCategoryButton(Letter);
-      end
+        sLanguage: begin
+          if (CompareText(CurCategory, CurSong.Language) <> 0) then
+          begin
+            CurCategory := CurSong.Language;
+            // add Language Button
+            AddCategoryButton(CurCategory);
+          end
+        end;
 
-      else if (Ini.Sorting = sFolder) and
-              (CompareText(CurCategory, CurSong.Folder) <> 0) then
-      begin
-        CurCategory := CurSong.Folder;
-        // add folder tab
-        AddCategoryButton(CurCategory);
-      end
+        sTitle: begin
+          if (Length(CurSong.Title) >= 1) then
+          begin
+            LetterTmp := UCS4UpperCase(UTF8ToUCS4String(CurSong.Title)[0]);
+            if (Letter <> LetterTmp) then
+            begin
+              Letter := LetterTmp;
+              // add a letter Category Button
+              AddCategoryButton(UCS4ToUTF8String(Letter));
+            end;
+          end;
+        end;
 
-      else if (Ini.Sorting = sTitle2) and
-              (Length(CurSong.Title) >= 1) then
-      begin
-        // pack all numbers into a category named '#'
-        if (CurSong.Title[1] >= '0') and (CurSong.Title[1] <= '9') then
-          LetterTmp := '#'
-        else
-          LetterTmp := UpperCase(CurSong.Title)[1];
+        sArtist: begin
+          if (Length(CurSong.Artist) >= 1) then
+          begin
+            LetterTmp := UCS4UpperCase(UTF8ToUCS4String(CurSong.Artist)[0]);
+            if (Letter <> LetterTmp) then
+            begin
+              Letter := LetterTmp;
+              // add a letter Category Button
+              AddCategoryButton(UCS4ToUTF8String(Letter));
+            end;
+          end;
+        end;
 
-        if (Letter <> LetterTmp) then
-        begin
-          Letter := LetterTmp;
-          // add a letter Category Button
-          AddCategoryButton(Letter);
+        sFolder: begin
+          if (CompareText(CurCategory, CurSong.Folder) <> 0) then
+          begin
+            CurCategory := CurSong.Folder;
+            // add folder tab
+            AddCategoryButton(CurCategory);
+          end;
         end;
-      end
 
-      else if (Ini.Sorting = sArtist2) and
-              (Length(CurSong.Artist)>=1) then
-      begin
-        // pack all numbers into a category named '#'
-        if (CurSong.Artist[1] >= '0') and (CurSong.Artist[1] <= '9') then
-          LetterTmp := '#'
-        else
-          LetterTmp := UpperCase(CurSong.Artist)[1];
+        sTitle2: begin
+          if (Length(CurSong.Title) >= 1) then
+          begin
+            LetterTmp := UTF8ToUCS4String(CurSong.Title)[0];
+            // pack all numbers into a category named '#'
+            if (LetterTmp in [UCS4Char('0') .. UCS4Char('9')]) then
+              LetterTmp := UCS4Char('#')
+            else
+              LetterTmp := UCS4UpperCase(LetterTmp);
+
+            if (Letter <> LetterTmp) then
+            begin
+              Letter := LetterTmp;
+              // add a letter Category Button
+              AddCategoryButton(UCS4ToUTF8String(Letter));
+            end;
+          end;
+        end;
 
-        if (Letter <> LetterTmp) then
-        begin
-          Letter := LetterTmp;
-          // add a letter Category Button
-          AddCategoryButton(Letter);
+        sArtist2: begin
+          if (Length(CurSong.Artist) >= 1) then
+          begin
+            LetterTmp := UTF8ToUCS4String(CurSong.Artist)[0];
+            // pack all numbers into a category named '#'
+            if (LetterTmp in [UCS4Char('0') .. UCS4Char('9')]) then
+              LetterTmp := UCS4Char('#')
+            else
+              LetterTmp := UCS4UpperCase(LetterTmp);
+
+            if (Letter <> LetterTmp) then
+            begin
+              Letter := LetterTmp;
+              // add a letter Category Button
+              AddCategoryButton(UCS4ToUTF8String(Letter));
+            end;
+          end;
         end;
-      end;
-    end;
+        
+      end; // case (Ini.Sorting)
+    end; // if (Ini.Tabs = 1)
 
     CatIndex := Length(Song);
     SetLength(Song, CatIndex+1);