[Scummvm-git-logs] scummvm master -> 08c46ec41e26003091a8586439b6cd9aa851fba3

lephilousophe lephilousophe at users.noreply.github.com
Sat Apr 10 21:37:15 UTC 2021


This automated email contains information about 3 new commits which have been
pushed to the 'scummvm' repo located at https://github.com/scummvm/scummvm .

Summary:
6f4359e6c2 DEVTOOLS: Make create_encoding.py Python3 compliant
5cdb300534 COMMON: Fix CJK encode/decode and make valgrind quiet
08c46ec41e TESTS: Cleanup allocation in array tests


Commit: 6f4359e6c2a421979b88965889f2ce9aefbc2723
    https://github.com/scummvm/scummvm/commit/6f4359e6c2a421979b88965889f2ce9aefbc2723
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00

Commit Message:
DEVTOOLS: Make create_encoding.py Python3 compliant

Rename it to fix typo

Changed paths:
  A devtools/create_encoding/create_encoding.py
  R devtools/create_encoding/create_encoging.py


diff --git a/devtools/create_encoding/create_encoging.py b/devtools/create_encoding/create_encoding.py
similarity index 76%
rename from devtools/create_encoding/create_encoging.py
rename to devtools/create_encoding/create_encoding.py
index 874500945a..4c267ff45d 100755
--- a/devtools/create_encoding/create_encoging.py
+++ b/devtools/create_encoding/create_encoding.py
@@ -1,8 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 import struct
 
+def merge_ranges(*ranges):
+    # Use a set for efficient lookup
+    result = set()
+    for rng in ranges:
+        result.update(rng)
+    return result
+
 def processtable(inputfilename, outfile, highrange, lowrange):
     with open(inputfilename) as f:
         res = dict(((x << 8) | y, 0) for x in highrange for y in lowrange) 
@@ -37,18 +44,18 @@ FILE = 'file'
 tables = [
     {
         FILE: "CP932.TXT",
-        HIGH: range(0x81, 0x85) + range(0x87, 0xa0) + range(0xe0, 0xef) + range(0xfa, 0xfd),
+        HIGH: merge_ranges(range(0x81, 0x85), range(0x87, 0xa0), range(0xe0, 0xef), range(0xfa, 0xfd)),
         LOW: range(0x40, 0x100)
     },
     {
         FILE: "CP949.TXT",
         HIGH: range(0x81, 0xFF),
-        LOW: range(0x41, 0x5b) + range(0x61, 0x7b) + range(0x81, 0xFF)
+        LOW: merge_ranges(range(0x41, 0x5b), range(0x61, 0x7b), range(0x81, 0xFF))
     },
     {
         FILE: "CP950.TXT",
         HIGH: range(0xA1, 0xFA),
-        LOW: range(0x40, 0x7f) + range(0xa1, 0xff)
+        LOW: merge_ranges(range(0x40, 0x7f), range(0xa1, 0xff))
     }
 ]
 


Commit: 5cdb30053411dae90de014c50b22b1349b0fd528
    https://github.com/scummvm/scummvm/commit/5cdb30053411dae90de014c50b22b1349b0fd528
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00

Commit Message:
COMMON: Fix CJK encode/decode and make valgrind quiet

CJK offsets were wrong compared to encodings.dat file and buffers were
read past their end.
Rewrote indexing to make it match the Python script and optimize
slightly

Changed paths:
    common/str-enc.cpp
    test/common/encoding.h


diff --git a/common/str-enc.cpp b/common/str-enc.cpp
index 0bb288eeac..9ffb200d56 100644
--- a/common/str-enc.cpp
+++ b/common/str-enc.cpp
@@ -175,12 +175,14 @@ void U32String::decodeWindows932(const char *src, uint32 len) {
 		uint8 lowidx = low - 0x40;
 		uint8 highidx;
 
-		if (high >= 0x81 && high <= 0x84)
+		if (high >= 0x81 && high < 0x85)
 			highidx = high - 0x81;
-		else if (high >= 0x87 && high <= 0x9f)
+		else if (high >= 0x87 && high < 0xa0)
 			highidx = high - 0x87 + 4;
-		else if (high >= 0xe0 && high <= 0xee)
+		else if (high >= 0xe0 && high < 0xef)
 			highidx = high - 0xe0 + 29;
+		else if (high >= 0xfa && high < 0xfd)
+			highidx = high - 0xfa + 44;
 		else {
 			operator+=(invalidCode);
 			continue;
@@ -199,11 +201,11 @@ void U32String::decodeWindows932(const char *src, uint32 len) {
 
 static uint16 convertUHCToUCSReal(uint8 high, uint8 low) {
 	uint lowidx = 0;
-	if (low >= 0x41 && low <= 0x5a)
+	if (low >= 0x41 && low < 0x5b)
 		lowidx = low - 0x41;
-	else if (low >= 0x61 && low <= 0x7a)
+	else if (low >= 0x61 && low < 0x7b)
 		lowidx = low - 0x61 + 0x1a;
-	else if (low >= 0x81 && low <= 0xfe)
+	else if (low >= 0x81 && low < 0xff)
 		lowidx = low - 0x81 + 0x1a * 2;
 	else
 		return 0;
@@ -283,7 +285,7 @@ void U32String::decodeWindows950(const char *src, uint32 len) {
 		}
 
 		uint8 low = src[i++];
-		uint8 lowidx = low < 0x80 ? low - 0x40 : low - 0x62;
+		uint8 lowidx = low < 0x80 ? low - 0x40 : (low - 0xa1 + 0x3f);
 
 		// Main range
 		if (high >= 0xa1 && high < 0xfa) {
@@ -320,22 +322,24 @@ void String::encodeWindows932(const U32String &src) {
 	if (!reverseTable && windows932ConversionTable) {
 		uint16 *rt = new uint16[0x10000];
 		memset(rt, 0, sizeof(rt[0]) * 0x10000);
-		for (uint highidx = 0; highidx < 58; highidx++)
+		for (uint highidx = 0; highidx < 47; highidx++) {
+			uint8 high = 0;
+			if (highidx < 4)
+				high = highidx + 0x81;
+			else if (highidx < 29)
+				high = highidx + 0x87 - 4;
+			else if (highidx < 44)
+				high = highidx + 0xe0 - 29;
+			else
+				high = highidx + 0xfa - 44;
+
 			for (uint lowidx = 0; lowidx < 192; lowidx++) {
-				uint8 high = 0;
 				uint8 low = lowidx + 0x40;
 				uint16 unicode = windows932ConversionTable[highidx * 192 + lowidx];
 
-				if (highidx < 4)
-					high = highidx + 0x81;
-				else if (highidx < 29)
-					high = highidx + 0x87 - 4;
-				else
-					high = highidx + 0xe0 - 29;
-
 				rt[unicode] = (high << 8) | low;
 			}
-
+		}
 		reverseTable = rt;
 	}
 
@@ -389,20 +393,22 @@ void String::encodeWindows949(const U32String &src) {
 		uint16 *rt = new uint16[0x10000];
 		memset(rt, 0, sizeof(rt[0]) * 0x10000);
 
-		for (uint highidx = 0; highidx < 0x7e; highidx++)
-			for (uint lowidx = 0; lowidx < 0xb2; lowidx++) {
+		for (uint lowidx = 0; lowidx < 0xb2; lowidx++) {
+			uint8 low = 0;
+			if (lowidx < 0x1a)
+				low = 0x41 + lowidx;
+			else if (lowidx < 0x1a * 2)
+				low = 0x61 + lowidx - 0x1a;
+			else
+				low = 0x81 + lowidx - 0x1a * 2;
+
+			for (uint highidx = 0; highidx < 0x7e; highidx++) {
 				uint8 high = highidx + 0x81;
-				uint8 low = 0;
 				uint16 unicode = windows949ConversionTable[highidx * 0xb2 + lowidx];
 
-				if (lowidx < 0x1a)
-					low = 0x41 + lowidx;
-				else if (lowidx < 0x1a * 2)
-					low = 0x61 + lowidx - 0x1a;
-				else
-					low = 0x81 + lowidx - 0x1a * 2;
 				rt[unicode] = (high << 8) | low;
 			}
+		}
 
 		reverseTable = rt;
 	}
@@ -477,18 +483,20 @@ void String::encodeWindows950(const U32String &src, bool transliterate) {
 		uint16 *rt = new uint16[0x10000];
 		memset(rt, 0, sizeof(rt[0]) * 0x10000);
 
-		for (uint highidx = 0; highidx < 90; highidx++)
-			for (uint lowidx = 0; lowidx < 157; lowidx++) {
+		for (uint lowidx = 0; lowidx < 157; lowidx++) {
+			uint8 low = 0;
+			if (lowidx < 0x3f)
+				low = 0x40 + lowidx;
+			else
+				low = 0xa1 + lowidx - 0x3f;
+
+			for (uint highidx = 0; highidx < 89; highidx++) {
 				uint8 high = highidx + 0xa1;
-				uint8 low = 0;
 				uint16 unicode = windows950ConversionTable[highidx * 157 + lowidx];
 
-				if (lowidx <= 0x3e)
-					low = 0x40 + lowidx;
-				else
-					low = 0x62 + lowidx;
 				rt[unicode] = (high << 8) | low;
 			}
+		}
 
 		reverseTable = rt;
 	}
diff --git a/test/common/encoding.h b/test/common/encoding.h
index 5af9751209..c6b73f5383 100644
--- a/test/common/encoding.h
+++ b/test/common/encoding.h
@@ -173,14 +173,14 @@ public:
 		uint16 *result16 = Common::U32String((uint32 *) utf32, 3).encodeUTF16Native(NULL);
 		TS_ASSERT(result16 != NULL);
 		TS_ASSERT_EQUALS(memcmp(result16, utf16, 8), 0);
-		free(result16);
+		delete[] result16;
 
 		// UTF8 to UTF16
 
 		result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16Native(NULL);
 		TS_ASSERT(result16 != NULL);
 		TS_ASSERT_EQUALS(memcmp(result16, utf16, 8), 0);
-		free(result16);
+		delete[] result16;
 
 		// UTF8 to UTF32
 		Common::U32String resultustr = Common::String((const char *) utf8, 6).decode(Common::kUtf8);
@@ -207,7 +207,7 @@ public:
 		uint16 *result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16BE(NULL);
 		TS_ASSERT(result16 != NULL);
 		TS_ASSERT_EQUALS(memcmp(result16, utf16be, 8), 0);
-		free(result16);
+		delete[] result16;
 
 	}
 
@@ -226,7 +226,7 @@ public:
 		uint16 *result16 = Common::U32String((char *) utf8, 6, Common::kUtf8).encodeUTF16LE(NULL);
 		TS_ASSERT(result16 != NULL);
 		TS_ASSERT_EQUALS(memcmp(result16, utf16le, 8), 0);
-		free(result16);
+		delete[] result16;
 	
 	}
 


Commit: 08c46ec41e26003091a8586439b6cd9aa851fba3
    https://github.com/scummvm/scummvm/commit/08c46ec41e26003091a8586439b6cd9aa851fba3
Author: Le Philousophe (lephilousophe at users.noreply.github.com)
Date: 2021-04-10T23:36:55+02:00

Commit Message:
TESTS: Cleanup allocation in array tests

Changed paths:
    test/common/array.h


diff --git a/test/common/array.h b/test/common/array.h
index f86d291ab2..d382838205 100644
--- a/test/common/array.h
+++ b/test/common/array.h
@@ -459,6 +459,11 @@ public:
 		}
 
 		TS_ASSERT_EQUALS(iter, container.end());
+
+		// Cleanup
+		for(iter = container.begin(); iter != container.end(); iter++) {
+			delete *iter;
+		}
 	}
 
 	void test_stability() {
@@ -485,6 +490,11 @@ public:
 		}
 
 		TS_ASSERT_EQUALS(iter, container.end());
+
+		// Cleanup
+		for(iter = container.begin(); iter != container.end(); iter++) {
+			delete *iter;
+		}
 	}
 
 };




More information about the Scummvm-git-logs mailing list