[Scummvm-git-logs] scummvm master -> 64f727ca3d32e15aa126857424fdb0f55acdc4c4
criezy
criezy at scummvm.org
Sat Oct 3 22:04:36 UTC 2020
This automated email contains information about 3 new commits which have been
pushed to the 'scummvm' repo located at https://github.com/scummvm/scummvm .
Summary:
5c76050760 TESTBED: Add some more test for encoding conversion
8555463758 COMMON: Fix Encoding switchEndian function
64f727ca3d COMMON: Fix performance issues in Encoding when checking endianness
Commit: 5c7605076084aff86f8b96c071299bdfeba11d06
https://github.com/scummvm/scummvm/commit/5c7605076084aff86f8b96c071299bdfeba11d06
Author: Thierry Crozat (criezy at scummvm.org)
Date: 2020-10-03T23:04:25+01:00
Commit Message:
TESTBED: Add some more test for encoding conversion
The additional test cover the case were we convert UTF-16 or UTF-32
strings between different endianness.
Changed paths:
engines/testbed/encoding.cpp
diff --git a/engines/testbed/encoding.cpp b/engines/testbed/encoding.cpp
index 432da8b764..66ffd5535e 100644
--- a/engines/testbed/encoding.cpp
+++ b/engines/testbed/encoding.cpp
@@ -239,6 +239,18 @@ TestExitStatus Encodingtests::testConversionUnicodeBigEndian() {
//| dolar | cent | euro
unsigned char utf32be[] = {0, 0, 0, 0x24, 0, 0, 0, 0xA2, 0, 0, 0x20, 0xAC, 0, 0, 0, 0};
+#if SCUMM_BIG_ENDIAN
+ //| dolar | cent | euro |
+ unsigned char utf16[] = {0, 0x24, 0, 0xA2, 0x20, 0xAC, 0, 0};
+ //| dolar | cent | euro
+ unsigned char utf32[] = {0, 0, 0, 0x24, 0, 0, 0, 0xA2, 0, 0, 0x20, 0xAC, 0, 0, 0, 0};
+#else
+ //| dolar | cent | euro |
+ unsigned char utf16[] = {0x24, 0, 0xA2, 0, 0xAC, 0x20, 0, 0};
+ //| dolar | cent | euro
+ unsigned char utf32[] = {0x24, 0, 0, 0, 0xA2, 0, 0, 0, 0xAC, 0x20, 0, 0, 0, 0, 0, 0};
+#endif
+
// UTF16 to UTF8
Common::Encoding converter("UTF-8", "UTF-16BE");
@@ -402,6 +414,69 @@ TestExitStatus Encodingtests::testConversionUnicodeBigEndian() {
return kTestFailed;
}
free(result);
+
+ // UTF16 to UTF16
+ converter.setFrom("UTF-16BE");
+ converter.setTo("UTF-16");
+
+ result = converter.convert((char *) utf16be, 6);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-16BE to UTF-16 conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf16, 8)) {
+ Testsuite::logPrintf("UTF-16BE to UTF-16 conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ converter.setFrom("UTF-16");
+ converter.setTo("UTF-16BE");
+
+ result = converter.convert((char *) utf16, 6);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-16 to UTF-16BE conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf16be, 8)) {
+ Testsuite::logPrintf("UTF-16 to UTF-16BE conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ // UTF32 to UTF32
+ converter.setFrom("UTF-32BE");
+ converter.setTo("UTF-32");
+
+ result = converter.convert((char *) utf32be, 12);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-32BE to UTF-32 conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf32, 16)) {
+ Testsuite::logPrintf("UTF-32BE to UTF-32 conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ converter.setFrom("UTF-32");
+ converter.setTo("UTF-32BE");
+
+ result = converter.convert((char *) utf32, 12);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-32 to UTF-32BE conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf32be, 16)) {
+ Testsuite::logPrintf("UTF-32 to UTF-32BE conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
return kTestPassed;
}
@@ -422,6 +497,17 @@ TestExitStatus Encodingtests::testConversionUnicodeLittleEndian() {
//| dolar | cent | euro
unsigned char utf32le[] = {0x24, 0, 0, 0, 0xA2, 0, 0, 0, 0xAC, 0x20, 0, 0, 0, 0, 0, 0};
+#if SCUMM_BIG_ENDIAN
+ unsigned char utf16[] = {0, 0x24, 0, 0xA2, 0x20, 0xAC, 0, 0};
+ //| dolar | cent | euro
+ unsigned char utf32[] = {0, 0, 0, 0x24, 0, 0, 0, 0xA2, 0, 0, 0x20, 0xAC, 0, 0, 0, 0};
+#else
+ //| dolar | cent | euro |
+ unsigned char utf16[] = {0x24, 0, 0xA2, 0, 0xAC, 0x20, 0, 0};
+ //| dolar | cent | euro
+ unsigned char utf32[] = {0x24, 0, 0, 0, 0xA2, 0, 0, 0, 0xAC, 0x20, 0, 0, 0, 0, 0, 0};
+#endif
+
// UTF16 to UTF8
Common::Encoding converter("UTF-8", "UTF-16LE");
@@ -585,6 +671,69 @@ TestExitStatus Encodingtests::testConversionUnicodeLittleEndian() {
return kTestFailed;
}
free(result);
+
+ // UTF16 to UTF16
+ converter.setFrom("UTF-16LE");
+ converter.setTo("UTF-16");
+
+ result = converter.convert((char *) utf16le, 6);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-16LE to UTF-16 conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf16, 8)) {
+ Testsuite::logPrintf("UTF-16LE to UTF-16 conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ converter.setFrom("UTF-16");
+ converter.setTo("UTF-16LE");
+
+ result = converter.convert((char *) utf16, 6);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-16 to UTF-16LE conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf16le, 8)) {
+ Testsuite::logPrintf("UTF-16 to UTF-16LE conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ // UTF32 to UTF32
+ converter.setFrom("UTF-32LE");
+ converter.setTo("UTF-32");
+
+ result = converter.convert((char *) utf32le, 12);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-32LE to UTF-32 conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf32, 16)) {
+ Testsuite::logPrintf("UTF-32LE to UTF-32 conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
+ converter.setFrom("UTF-32");
+ converter.setTo("UTF-32LE");
+
+ result = converter.convert((char *) utf32, 12);
+ if (result == NULL) {
+ Testsuite::logPrintf("UTF-32 to UTF-32LE conversion isn't available");
+ return kTestFailed;
+ }
+ if (memcmp(result, utf32le, 16)) {
+ Testsuite::logPrintf("UTF-32 to UTF-32LE conversion differs from the expected result.");
+ free(result);
+ return kTestFailed;
+ }
+ free(result);
+
return kTestPassed;
}
Commit: 85554637588517465a1434908dd5778652354e28
https://github.com/scummvm/scummvm/commit/85554637588517465a1434908dd5778652354e28
Author: Thierry Crozat (criezy at scummvm.org)
Date: 2020-10-03T23:04:25+01:00
Commit Message:
COMMON: Fix Encoding switchEndian function
The result string was missing the null terminator.
Changed paths:
common/encoding.cpp
diff --git a/common/encoding.cpp b/common/encoding.cpp
index 0b7fbab899..d8f654bab3 100644
--- a/common/encoding.cpp
+++ b/common/encoding.cpp
@@ -55,7 +55,7 @@ Encoding::Encoding(const String &to, const String &from)
char *Encoding::switchEndian(const char *string, int length, int bitCount) {
assert(bitCount % 8 == 0);
assert(length % (bitCount / 8) == 0);
- char *newString = (char *)malloc(length);
+ char *newString = (char *)calloc(sizeof(char), length + 4);
if (!newString) {
warning("Could not allocate memory for string conversion");
return nullptr;
Commit: 64f727ca3d32e15aa126857424fdb0f55acdc4c4
https://github.com/scummvm/scummvm/commit/64f727ca3d32e15aa126857424fdb0f55acdc4c4
Author: Thierry Crozat (criezy at scummvm.org)
Date: 2020-10-03T23:04:25+01:00
Commit Message:
COMMON: Fix performance issues in Encoding when checking endianness
Changed paths:
common/encoding.cpp
diff --git a/common/encoding.cpp b/common/encoding.cpp
index d8f654bab3..5d701d3cd1 100644
--- a/common/encoding.cpp
+++ b/common/encoding.cpp
@@ -44,7 +44,7 @@ String addUtfEndianness(const String &str) {
return str + "LE";
#endif
} else
- return String(str);
+ return str;
}
Encoding::Encoding(const String &to, const String &from)
@@ -98,19 +98,33 @@ char *Encoding::convertWithTransliteration(const String &to, const String &from,
return result;
}
- if ((addUtfEndianness(to).equalsIgnoreCase("utf-16be") &&
- addUtfEndianness(from).equalsIgnoreCase("utf-16le")) ||
- (addUtfEndianness(to).equalsIgnoreCase("utf-16le") &&
- addUtfEndianness(from).equalsIgnoreCase("utf-16be")) ||
- (addUtfEndianness(to).equalsIgnoreCase("utf-32be") &&
- addUtfEndianness(from).equalsIgnoreCase("utf-32le")) ||
- (addUtfEndianness(to).equalsIgnoreCase("utf-32le") &&
- addUtfEndianness(from).equalsIgnoreCase("utf-32be"))) {
- // The encoding is the same, we just need to switch the endianness
- if (to.hasPrefixIgnoreCase("utf-16"))
- return switchEndian(string, length, 16);
- else
- return switchEndian(string, length, 32);
+ if ((to.hasPrefixIgnoreCase("utf-16") && from.hasPrefixIgnoreCase("utf-16")) ||
+ (to.hasPrefixIgnoreCase("utf-32") && from.hasPrefixIgnoreCase("utf-32"))) {
+ // Since the two strings are not equal as this is already checked above,
+ // this likely mean that one or both has an endianness suffix, and we
+ // just need to switch the endianess.
+#ifdef SCUMM_BIG_ENDIAN
+ bool fromBigEndian = !from.hasSuffixIgnoreCase("le");
+ bool toBigEndian = !to.hasSuffixIgnoreCase("le");
+#else
+ bool fromBigEndian = from.hasSuffixIgnoreCase("be");
+ bool toBigEndian = to.hasSuffixIgnoreCase("be");
+#endif
+ if (fromBigEndian == toBigEndian) {
+ // don't convert, just copy the string and return it
+ char *result = (char *)calloc(sizeof(char), length + 4);
+ if (!result) {
+ warning("Could not allocate memory for string conversion");
+ return nullptr;
+ }
+ memcpy(result, string, length);
+ return result;
+ } else {
+ if (to.hasPrefixIgnoreCase("utf-16"))
+ return switchEndian(string, length, 16);
+ else
+ return switchEndian(string, length, 32);
+ }
}
char *newString = nullptr;
More information about the Scummvm-git-logs
mailing list