Added UTF-8 <-> UTF-16 <-> UTF-32 <-> UCS-2 <-> UCS-4 conversion capability

--HG--
extra : convert_revision : svn%3Ac70aab31-4412-0410-b14c-859654838e24/trunk%401505

configure.in

@@ -94,7 +94,7 @@ if test x$enable_libc = xyes; then
 
     dnl Check for C library headers
     AC_HEADER_STDC
-    AC_CHECK_HEADERS(sys/types.h stdio.h stdlib.h stddef.h stdarg.h malloc.h memory.h string.h strings.h inttypes.h stdint.h ctype.h math.h signal.h)
+    AC_CHECK_HEADERS(sys/types.h stdio.h stdlib.h stddef.h stdarg.h malloc.h memory.h string.h strings.h inttypes.h stdint.h ctype.h math.h iconv.h signal.h)
 
     dnl Check for typedefs, structures, etc.
     AC_TYPE_SIZE_T
@@ -116,7 +116,7 @@ if test x$enable_libc = xyes; then
     if test x$ac_cv_func_strtod = xyes; then
         AC_DEFINE(HAVE_STRTOD)
     fi
-    AC_CHECK_FUNCS(malloc calloc realloc free getenv putenv unsetenv qsort abs bcopy memset memcpy memmove strlen strlcpy strlcat strdup _strrev _strupr _strlwr strchr strrchr strstr itoa _ltoa _uitoa _ultoa strtol strtoul _i64toa _ui64toa strtoll strtoull atoi atof strcmp strncmp stricmp strcasecmp sscanf snprintf vsnprintf sigaction setjmp nanosleep)
+    AC_CHECK_FUNCS(malloc calloc realloc free getenv putenv unsetenv qsort abs bcopy memset memcpy memmove strlen strlcpy strlcat strdup _strrev _strupr _strlwr strchr strrchr strstr itoa _ltoa _uitoa _ultoa strtol strtoul _i64toa _ui64toa strtoll strtoull atoi atof strcmp strncmp stricmp strcasecmp strncasecmp sscanf snprintf vsnprintf iconv sigaction setjmp nanosleep)
 
     AC_CHECK_LIB(m, pow, [BUILD_LDFLAGS="$BUILD_LDFLAGS -lm"])
 fi

include/SDL_config.h.in

@@ -68,6 +68,7 @@
 #undef HAVE_STDINT_H
 #undef HAVE_CTYPE_H
 #undef HAVE_MATH_H
+#undef HAVE_ICONV_H
 #undef HAVE_SIGNAL_H
 #undef HAVE_ALTIVEC_H
 
@@ -118,9 +119,11 @@
 #undef HAVE_STRNCMP
 #undef HAVE_STRICMP
 #undef HAVE_STRCASECMP
+#undef HAVE_STRNCASECMP
 #undef HAVE_SSCANF
 #undef HAVE_SNPRINTF
 #undef HAVE_VSNPRINTF
+#undef HAVE_ICONV
 #undef HAVE_SIGACTION
 #undef HAVE_SETJMP
 #undef HAVE_NANOSLEEP

include/SDL_stdinc.h

@@ -70,6 +70,9 @@
 #if HAVE_CTYPE_H
 # include <ctype.h>
 #endif
+#if HAVE_ICONV_H
+# include <iconv.h>
+#endif
 
 /* The number of elements in an array */
 #define SDL_arraysize(array)	(sizeof(array)/sizeof(array[0]))
@@ -518,6 +521,12 @@ extern DECLSPEC int SDLCALL SDL_strncmp(const char *str1, const char *str2, size
 extern DECLSPEC int SDLCALL SDL_strcasecmp(const char *str1, const char *str2);
 #endif
 
+#if HAVE_STRNCASECMP
+#define SDL_strncasecmp strncasecmp
+#else
+extern DECLSPEC int SDLCALL SDL_strncasecmp(const char *str1, const char *str2, size_t maxlen);
+#endif
+
 #if HAVE_SSCANF
 #define SDL_sscanf      sscanf
 #else
@@ -536,6 +545,32 @@ extern DECLSPEC int SDLCALL SDL_snprintf(char *text, size_t maxlen, const char *
 extern DECLSPEC int SDLCALL SDL_vsnprintf(char *text, size_t maxlen, const char *fmt, va_list ap);
 #endif
 
+/* The SDL implementation of iconv() returns these error codes */
+#define SDL_ICONV_ERROR		(size_t)-1
+#define SDL_ICONV_E2BIG		(size_t)-2
+#define SDL_ICONV_EILSEQ	(size_t)-3
+#define SDL_ICONV_EINVAL	(size_t)-4
+
+#if HAVE_ICONV
+#define SDL_iconv_t     iconv_t
+#define SDL_iconv_open  iconv_open
+#define SDL_iconv_close iconv_close
+extern DECLSPEC size_t SDLCALL SDL_iconv(SDL_iconv_t cd, char **inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft);
+#else
+typedef struct _SDL_iconv_t *SDL_iconv_t;
+extern DECLSPEC SDL_iconv_t SDLCALL SDL_iconv_open(const char *tocode, const char *fromcode);
+extern DECLSPEC int SDLCALL SDL_iconv_close(SDL_iconv_t cd);
+extern DECLSPEC size_t SDLCALL SDL_iconv(SDL_iconv_t cd, char **inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft);
+#endif
+/* This function converts a string between encodings in one pass, returning a
+   string that must be freed with SDL_free() or NULL on error.
+*/
+extern DECLSPEC char * SDLCALL SDL_iconv_string(const char *tocode, const char *fromcode, char *inbuf, size_t inbytesleft);
+#define SDL_iconv_utf8_ascii(S)		SDL_iconv_string("ASCII", "UTF-8", S, SDL_strlen(S)+1)
+#define SDL_iconv_utf8_latin1(S)	SDL_iconv_string("LATIN1", "UTF-8", S, SDL_strlen(S)+1)
+#define SDL_iconv_utf8_ucs2(S)		(Uint16 *)SDL_iconv_string("UCS-2", "UTF-8", S, SDL_strlen(S)+1)
+#define SDL_iconv_utf8_ucs4(S)		(Uint32 *)SDL_iconv_string("UCS-4", "UTF-8", S, SDL_strlen(S)+1)
+
 /* Ends C function definitions when using C++ */
 #ifdef __cplusplus
 }

src/stdlib/SDL_string.c

@@ -661,12 +661,12 @@ int SDL_strncmp(const char *str1, const char *str2, size_t maxlen)
 }
 #endif
 
-#ifndef HAVE_STRCASECMP
+#if !defined(HAVE_STRCASECMP) && !defined(HAVE_STRICMP)
 int SDL_strcasecmp(const char *str1, const char *str2)
 {
     char a = 0;
     char b = 0;
-    while (*str1 && *str2) {
+    while ( *str1 && *str2 ) {
         a = SDL_tolower(*str1);
         b = SDL_tolower(*str2);
         if ( a != b )
@@ -678,6 +678,24 @@ int SDL_strcasecmp(const char *str1, const char *str2)
 }
 #endif
 
+#ifndef HAVE_STRNCASECMP
+int SDL_strncasecmp(const char *str1, const char *str2, size_t maxlen)
+{
+    char a = 0;
+    char b = 0;
+    while ( *str1 && *str2 && maxlen ) {
+        a = SDL_tolower(*str1);
+        b = SDL_tolower(*str2);
+        if ( a != b )
+            break;
+        ++str1;
+        ++str2;
+        --maxlen;
+    }
+    return (int)((unsigned char)a - (unsigned char)b);
+}
+#endif
+
 #ifndef HAVE_SSCANF
 int SDL_sscanf(const char *text, const char *fmt, ...)
 {

test/Makefile.in

@@ -7,7 +7,7 @@ EXE	= @EXE@
 CFLAGS  = @CFLAGS@
 LIBS	= @LIBS@
 
-TARGETS = checkkeys$(EXE) graywin$(EXE) loopwave$(EXE) testalpha$(EXE) testbitmap$(EXE) testblitspeed$(EXE) testcdrom$(EXE) testdyngl$(EXE) testerror$(EXE) testfile$(EXE) testgamma$(EXE) testgl$(EXE) testhread$(EXE) testjoystick$(EXE) testkeys$(EXE) testlock$(EXE) testoverlay2$(EXE) testoverlay$(EXE) testpalette$(EXE) testplatform$(EXE) testsem$(EXE) testsprite$(EXE) testtimer$(EXE) testver$(EXE) testvidinfo$(EXE) testwin$(EXE) testwm$(EXE) threadwin$(EXE) torturethread$(EXE)
+TARGETS = checkkeys$(EXE) graywin$(EXE) loopwave$(EXE) testalpha$(EXE) testbitmap$(EXE) testblitspeed$(EXE) testcdrom$(EXE) testdyngl$(EXE) testerror$(EXE) testfile$(EXE) testgamma$(EXE) testgl$(EXE) testhread$(EXE) testiconv$(EXE) testjoystick$(EXE) testkeys$(EXE) testlock$(EXE) testoverlay2$(EXE) testoverlay$(EXE) testpalette$(EXE) testplatform$(EXE) testsem$(EXE) testsprite$(EXE) testtimer$(EXE) testver$(EXE) testvidinfo$(EXE) testwin$(EXE) testwm$(EXE) threadwin$(EXE) torturethread$(EXE)
 
 all: $(TARGETS)
 
@@ -50,6 +50,9 @@ testgl$(EXE): $(srcdir)/testgl.c
 testhread$(EXE): $(srcdir)/testhread.c
 	$(CC) -o $@ $? $(CFLAGS) $(LIBS)
 
+testiconv$(EXE): $(srcdir)/testiconv.c
+	$(CC) -o $@ $? $(CFLAGS) $(LIBS)
+
 testjoystick$(EXE): $(srcdir)/testjoystick.c
 	$(CC) -o $@ $? $(CFLAGS) $(LIBS)
 

test/testiconv.c

+
+#include <stdio.h>
+
+#include "SDL.h"
+
+static SDL_bool testutf16(char *data)
+{
+	Uint32 *p = (Uint32 *)data;
+	while(*p) {
+		if ( *p > 0x10FFFF ) {
+			return SDL_FALSE;
+		}
+		++p;
+	}
+	return SDL_TRUE;
+}
+
+static size_t widelen(char *data)
+{
+	size_t len = 0;
+	Uint32 *p = (Uint32 *)data;
+	while(*p++) {
+		++len;
+	}
+	return len;
+}
+
+int main(int argc, char *argv[])
+{
+	const char * formats[] = {
+		"UTF8",
+		"UTF-8",
+		"UTF16BE",
+		"UTF-16BE",
+		"UTF16LE",
+		"UTF-16LE",
+		"UTF32BE",
+		"UTF-32BE",
+		"UTF32LE",
+		"UTF-32LE",
+		"UCS4",
+		"UCS-4",
+	};
+	char buffer[BUFSIZ];
+	char *ucs4;
+	char *test[2];
+	int i, j, index = 0;
+	FILE *file;
+	int errors = 0;
+
+	if ( !argv[1] ) {
+		argv[1] = "utf8.txt";
+	}
+	file = fopen(argv[1], "rb");
+	if ( !file ) {
+		fprintf(stderr, "Unable to open %s\n", argv[1]);
+		return (1);
+	}
+
+	while ( fgets(buffer, sizeof(buffer), file) ) {
+		/* Convert to UCS-4 */
+		ucs4 = SDL_iconv_string("UCS-4", "UTF-8", buffer, SDL_strlen(buffer)+1);
+		size_t len = (widelen(ucs4)+1)*4;
+		for ( i = 0; i < SDL_arraysize(formats); ++i ) {
+			if ( (SDL_strncasecmp(formats[i], "UTF16", 5) == 0 ||
+			      SDL_strncasecmp(formats[i], "UTF-16", 6) == 0) &&
+			      !testutf16(ucs4) ) {
+				continue;
+			}
+			test[0] = SDL_iconv_string(formats[i], "UCS-4", ucs4, len);
+			test[1] = SDL_iconv_string("UCS-4", formats[i], test[0], len);
+			if ( SDL_memcmp(test[1], ucs4, len) != 0 ) {
+				fprintf(stderr, "FAIL: %s\n", formats[i]);
+				++errors;
+			}
+			SDL_free(test[0]);
+			SDL_free(test[1]);
+		}
+		test[0] = SDL_iconv_string("UTF-8", "UCS-4", ucs4, len);
+		SDL_free(ucs4);
+		fputs(test[0], stdout);
+		SDL_free(test[0]);
+	}
+	return (errors ? errors + 1 : 0);
+}