added converting certain text files from UTF-8 to internal text format

When reading text files with title screen messages, credits, level set
information or element descriptions, text in UTF-8 character encoding
will be converted on-the-fly to the internally used ISO-Latin-1 format.

diff --git a/src/libgame/misc.c b/src/libgame/misc.c
index 239a7d8ce8b637ab26764b9dcc2fb9ff7db3585f..05599b777ebe98a8784a0adf39067f841a6ac89b 100644 (file)
--- a/src/libgame/misc.c
+++ b/src/libgame/misc.c
@@ -1946,6 +1946,64 @@ char *getLatin1FromUTF8(char *utf8)
   return latin1;
 }
 
+int getTextEncoding(char *text)
+{
+  unsigned char *src = (unsigned char *)text;
+  int encoding = TEXT_ENCODING_ASCII;  // default: assume encoding is ASCII
+
+  while (*src)
+  {
+    if (*src >= 128)
+      encoding = TEXT_ENCODING_UTF_8;  // non-ASCII character: assume UTF-8
+
+    if (*src < 128)
+    {
+      src++;
+    }
+    else if (src[0] >= 192 && src[0] < 224 &&
+            src[1] >= 128 && src[1] < 192)
+    {
+      src += 2;
+    }
+    else if (src[0] >= 224 && src[0] < 240 &&
+            src[1] >= 128 && src[1] < 192 &&
+            src[2] >= 128 && src[2] < 192)
+    {
+      src += 3;
+    }
+    else if (src[0] >= 240 && src[0] < 248 &&
+            src[1] >= 128 && src[1] < 192 &&
+            src[2] >= 128 && src[2] < 192 &&
+            src[3] >= 128 && src[3] < 192)
+    {
+      src += 4;
+    }
+    else if (src[0] >= 248 && src[0] < 252 &&
+            src[1] >= 128 && src[1] < 192 &&
+            src[2] >= 128 && src[2] < 192 &&
+            src[3] >= 128 && src[3] < 192 &&
+            src[4] >= 128 && src[4] < 192)
+    {
+      src += 5;
+    }
+    else if (src[0] >= 252 && src[0] < 254 &&
+            src[1] >= 128 && src[1] < 192 &&
+            src[2] >= 128 && src[2] < 192 &&
+            src[3] >= 128 && src[3] < 192 &&
+            src[4] >= 128 && src[4] < 192 &&
+            src[5] >= 128 && src[5] < 192)
+    {
+      src += 6;
+    }
+    else
+    {
+      return TEXT_ENCODING_UNKNOWN;    // non-UTF-8 character: unknown encoding
+    }
+  }
+
+  return encoding;
+}
+
 
 // ----------------------------------------------------------------------------
 // functions for JSON handling

diff --git a/src/libgame/misc.h b/src/libgame/misc.h
index 19ac4be4deb0ecd8eb5db1c7ee9fa92a77b5f284..6c53d3181b65fc8571d546cf02222eccd3072225 100644 (file)
--- a/src/libgame/misc.h
+++ b/src/libgame/misc.h
@@ -45,6 +45,11 @@
 #define BIT_ORDER_MSB                  0
 #define BIT_ORDER_LSB                  1
 
+// values for character encoding
+#define TEXT_ENCODING_UNKNOWN          0
+#define TEXT_ENCODING_ASCII            1
+#define TEXT_ENCODING_UTF_8            2
+
 // values for createDirectory()
 #define PERMS_PRIVATE                  0
 #define PERMS_PUBLIC                   1
@@ -233,6 +238,8 @@ void WriteUnusedBytesToFile(FILE *, unsigned int);
 
 char *getUTF8FromLatin1(char *);
 char *getLatin1FromUTF8(char *);
+int getTextEncoding(char *);
+
 char *getEscapedJSON(char *);
 
 char *getKeyNameFromKey(Key);

diff --git a/src/libgame/text.c b/src/libgame/text.c
index 36096240203880ba31d6744f0c9bab2d9cc05eb0..eb36ab2285a47c42e0f8394e49a4d7850d419cf2 100644 (file)
--- a/src/libgame/text.c
+++ b/src/libgame/text.c
@@ -371,6 +371,15 @@ char *GetTextBufferFromFile(char *filename, int max_lines)
 
   closeFile(file);
 
+  if (getTextEncoding(buffer) == TEXT_ENCODING_UTF_8)
+  {
+    char *body_latin1 = getLatin1FromUTF8(buffer);
+
+    checked_free(buffer);
+
+    buffer = body_latin1;
+  }
+
   return buffer;
 }