[Scummvm-git-logs] scummvm master -> 08c46ec41e26003091a8586439b6cd9aa851fba3
lephilousophe
lephilousophe at users.noreply.github.com
Sat Apr 10 21:37:15 UTC 2021
This automated email contains information about 3 new commits which have been
pushed to the 'scummvm' repo located at https://github.com/scummvm/scummvm .
Summary:
6f4359e6c2 DEVTOOLS: Make create_encoding.py Python3 compliant
5cdb300534 COMMON: Fix CJK encode/decode and make valgrind quiet
08c46ec41e TESTS: Cleanup allocation in array tests
Commit: 6f4359e6c2a421979b88965889f2ce9aefbc2723
https://github.com/scummvm/scummvm/commit/6f4359e6c2a421979b88965889f2ce9aefbc2723
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00
Commit Message:
DEVTOOLS: Make create_encoding.py Python3 compliant
Rename it to fix typo
Changed paths:
A devtools/create_encoding/create_encoding.py
R devtools/create_encoding/create_encoging.py
diff --git a/devtools/create_encoding/create_encoging.py b/devtools/create_encoding/create_encoding.py
similarity index 76%
rename from devtools/create_encoding/create_encoging.py
rename to devtools/create_encoding/create_encoding.py
index 874500945a..4c267ff45d 100755
--- a/devtools/create_encoding/create_encoging.py
+++ b/devtools/create_encoding/create_encoding.py
@@ -1,8 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# encoding: utf-8
import struct
+def merge_ranges(*ranges):
+ # Use a set for efficient lookup
+ result = set()
+ for rng in ranges:
+ result.update(rng)
+ return result
+
def processtable(inputfilename, outfile, highrange, lowrange):
with open(inputfilename) as f:
res = dict(((x << 8) | y, 0) for x in highrange for y in lowrange)
@@ -37,18 +44,18 @@ FILE = 'file'
tables = [
{
FILE: "CP932.TXT",
- HIGH: range(0x81, 0x85) + range(0x87, 0xa0) + range(0xe0, 0xef) + range(0xfa, 0xfd),
+ HIGH: merge_ranges(range(0x81, 0x85), range(0x87, 0xa0), range(0xe0, 0xef), range(0xfa, 0xfd)),
LOW: range(0x40, 0x100)
},
{
FILE: "CP949.TXT",
HIGH: range(0x81, 0xFF),
- LOW: range(0x41, 0x5b) + range(0x61, 0x7b) + range(0x81, 0xFF)
+ LOW: merge_ranges(range(0x41, 0x5b), range(0x61, 0x7b), range(0x81, 0xFF))
},
{
FILE: "CP950.TXT",
HIGH: range(0xA1, 0xFA),
- LOW: range(0x40, 0x7f) + range(0xa1, 0xff)
+ LOW: merge_ranges(range(0x40, 0x7f), range(0xa1, 0xff))
}
]
Commit: 5cdb30053411dae90de014c50b22b1349b0fd528
https://github.com/scummvm/scummvm/commit/5cdb30053411dae90de014c50b22b1349b0fd528
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00
Commit Message:
COMMON: Fix CJK encode/decode and make valgrind quiet
CJK offsets were wrong compared to encodings.dat file and buffers were
read past their end.
Rewrote indexing to make it match the Python script and optimize
slightly
Changed paths:
common/str-enc.cpp
test/common/encoding.h
diff --git a/common/str-enc.cpp b/common/str-enc.cpp
index 0bb288eeac..9ffb200d56 100644
--- a/common/str-enc.cpp
+++ b/common/str-enc.cpp
@@ -175,12 +175,14 @@ void U32String::decodeWindows932(const char *src, uint32 len) {
uint8 lowidx = low - 0x40;
uint8 highidx;
- if (high >= 0x81 && high <= 0x84)
+ if (high >= 0x81 && high < 0x85)
highidx = high - 0x81;
- else if (high >= 0x87 && high <= 0x9f)
+ else if (high >= 0x87 && high < 0xa0)
highidx = high - 0x87 + 4;
- else if (high >= 0xe0 && high <= 0xee)
+ else if (high >= 0xe0 && high < 0xef)
highidx = high - 0xe0 + 29;
+ else if (high >= 0xfa && high < 0xfd)
+ highidx = high - 0xfa + 44;
else {
operator+=(invalidCode);
continue;
@@ -199,11 +201,11 @@ void U32String::decodeWindows932(const char *src, uint32 len) {
static uint16 convertUHCToUCSReal(uint8 high, uint8 low) {
uint lowidx = 0;
- if (low >= 0x41 && low <= 0x5a)
+ if (low >= 0x41 && low < 0x5b)
lowidx = low - 0x41;
- else if (low >= 0x61 && low <= 0x7a)
+ else if (low >= 0x61 && low < 0x7b)
lowidx = low - 0x61 + 0x1a;
- else if (low >= 0x81 && low <= 0xfe)
+ else if (low >= 0x81 && low < 0xff)
lowidx = low - 0x81 + 0x1a * 2;
else
return 0;
@@ -283,7 +285,7 @@ void U32String::decodeWindows950(const char *src, uint32 len) {
}
uint8 low = src[i++];
- uint8 lowidx = low < 0x80 ? low - 0x40 : low - 0x62;
+ uint8 lowidx = low < 0x80 ? low - 0x40 : (low - 0xa1 + 0x3f);
// Main range
if (high >= 0xa1 && high < 0xfa) {
@@ -320,22 +322,24 @@ void String::encodeWindows932(const U32String &src) {
if (!reverseTable && windows932ConversionTable) {
uint16 *rt = new uint16[0x10000];
memset(rt, 0, sizeof(rt[0]) * 0x10000);
- for (uint highidx = 0; highidx < 58; highidx++)
+ for (uint highidx = 0; highidx < 47; highidx++) {
+ uint8 high = 0;
+ if (highidx < 4)
+ high = highidx + 0x81;
+ else if (highidx < 29)
+ high = highidx + 0x87 - 4;
+ else if (highidx < 44)
+ high = highidx + 0xe0 - 29;
+ else
+ high = highidx + 0xfa - 44;
+
for (uint lowidx = 0; lowidx < 192; lowidx++) {
- uint8 high = 0;
uint8 low = lowidx + 0x40;
uint16 unicode = windows932ConversionTable[highidx * 192 + lowidx];
- if (highidx < 4)
- high = highidx + 0x81;
- else if (highidx < 29)
- high = highidx + 0x87 - 4;
- else
- high = highidx + 0xe0 - 29;
-
rt[unicode] = (high << 8) | low;
}
-
+ }
reverseTable = rt;
}
@@ -389,20 +393,22 @@ void String::encodeWindows949(const U32String &src) {
uint16 *rt = new uint16[0x10000];
memset(rt, 0, sizeof(rt[0]) * 0x10000);
- for (uint highidx = 0; highidx < 0x7e; highidx++)
- for (uint lowidx = 0; lowidx < 0xb2; lowidx++) {
+ for (uint lowidx = 0; lowidx < 0xb2; lowidx++) {
+ uint8 low = 0;
+ if (lowidx < 0x1a)
+ low = 0x41 + lowidx;
+ else if (lowidx < 0x1a * 2)
+ low = 0x61 + lowidx - 0x1a;
+ else
+ low = 0x81 + lowidx - 0x1a * 2;
+
+ for (uint highidx = 0; highidx < 0x7e; highidx++) {
uint8 high = highidx + 0x81;
- uint8 low = 0;
uint16 unicode = windows949ConversionTable[highidx * 0xb2 + lowidx];
- if (lowidx < 0x1a)
- low = 0x41 + lowidx;
- else if (lowidx < 0x1a * 2)
- low = 0x61 + lowidx - 0x1a;
- else
- low = 0x81 + lowidx - 0x1a * 2;
rt[unicode] = (high << 8) | low;
}
+ }
reverseTable = rt;
}
@@ -477,18 +483,20 @@ void String::encodeWindows950(const U32String &src, bool transliterate) {
uint16 *rt = new uint16[0x10000];
memset(rt, 0, sizeof(rt[0]) * 0x10000);
- for (uint highidx = 0; highidx < 90; highidx++)
- for (uint lowidx = 0; lowidx < 157; lowidx++) {
+ for (uint lowidx = 0; lowidx < 157; lowidx++) {
+ uint8 low = 0;
+ if (lowidx < 0x3f)
+ low = 0x40 + lowidx;
+ else
+ low = 0xa1 + lowidx - 0x3f;
+
+ for (uint highidx = 0; highidx < 89; highidx++) {
uint8 high = highidx + 0xa1;
- uint8 low = 0;
uint16 unicode = windows950ConversionTable[highidx * 157 + lowidx];
- if (lowidx <= 0x3e)
- low = 0x40 + lowidx;
- else
- low = 0x62 + lowidx;
rt[unicode] = (high << 8) | low;
}
+ }
reverseTable = rt;
}
diff --git a/test/common/encoding.h b/test/common/encoding.h
index 5af9751209..c6b73f5383 100644
--- a/test/common/encoding.h
+++ b/test/common/encoding.h
@@ -173,14 +173,14 @@ public:
uint16 *result16 = Common::U32String((uint32 *) utf32, 3).encodeUTF16Native(NULL);
TS_ASSERT(result16 != NULL);
TS_ASSERT_EQUALS(memcmp(result16, utf16, 8), 0);
- free(result16);
+ delete[] result16;
// UTF8 to UTF16
result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16Native(NULL);
TS_ASSERT(result16 != NULL);
TS_ASSERT_EQUALS(memcmp(result16, utf16, 8), 0);
- free(result16);
+ delete[] result16;
// UTF8 to UTF32
Common::U32String resultustr = Common::String((const char *) utf8, 6).decode(Common::kUtf8);
@@ -207,7 +207,7 @@ public:
uint16 *result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16BE(NULL);
TS_ASSERT(result16 != NULL);
TS_ASSERT_EQUALS(memcmp(result16, utf16be, 8), 0);
- free(result16);
+ delete[] result16;
}
@@ -226,7 +226,7 @@ public:
uint16 *result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16LE(NULL);
TS_ASSERT(result16 != NULL);
TS_ASSERT_EQUALS(memcmp(result16, utf16le, 8), 0);
- free(result16);
+ delete[] result16;
}
Commit: 08c46ec41e26003091a8586439b6cd9aa851fba3
https://github.com/scummvm/scummvm/commit/08c46ec41e26003091a8586439b6cd9aa851fba3
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00
Commit Message:
TESTS: Cleanup allocation in array tests
Changed paths:
test/common/array.h
diff --git a/test/common/array.h b/test/common/array.h
index f86d291ab2..d382838205 100644
--- a/test/common/array.h
+++ b/test/common/array.h
@@ -459,6 +459,11 @@ public:
}
TS_ASSERT_EQUALS(iter, container.end());
+
+ // Cleanup
+ for(iter = container.begin(); iter != container.end(); iter++) {
+ delete *iter;
+ }
}
void test_stability() {
@@ -485,6 +490,11 @@ public:
}
TS_ASSERT_EQUALS(iter, container.end());
+
+ // Cleanup
+ for(iter = container.begin(); iter != container.end(); iter++) {
+ delete *iter;
+ }
}
};
More information about the Scummvm-git-logs
mailing list