[Scummvm-git-logs] scummvm master -> b021957fb378811d8d59efa466b02e16f1d99fb0

sev- noreply at scummvm.org
Mon Mar 13 19:17:41 UTC 2023 

This automated email contains information about 6 new commits which have been
pushed to the 'scummvm' repo located at https://github.com/scummvm/scummvm .

Summary:
5b3fe1d916 COMMON: Error-out on an invalid clickteam compressed block
c33ce058f8 COMMON: Add createReadStreamForMemberNext for the ease of using patcher archives
6476e55aab COMMON: Allow memcaching archive to bypass caching part and return a stream
f513d6eee9 COMMON: Support patching clickteam installer and version used by PRCA
2f70d7dad0 SCUMM: Detect PRCA patch
b021957fb3 SCUMM: Support playing most RuSCUMM versions without patching manually


Commit: 5b3fe1d9163e7e35eb028a44af9d52737c4cccd3
    https://github.com/scummvm/scummvm/commit/5b3fe1d9163e7e35eb028a44af9d52737c4cccd3
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
COMMON: Error-out on an invalid clickteam compressed block

Changed paths:
    common/compression/gzio.cpp


diff --git a/common/compression/gzio.cpp b/common/compression/gzio.cpp
index c7857dc862e..d9d85c9fcbe 100644
--- a/common/compression/gzio.cpp
+++ b/common/compression/gzio.cpp
@@ -34,6 +34,7 @@
    comments to that effect with your name and the date.  Thank you.
  */
 
+#include "common/debug.h"
 #include "common/endian.h"
 #include "common/stream.h"
 #include "common/ptr.h"
@@ -881,6 +882,8 @@ GzioReadStream::get_new_block()
     case 7:
       _blockType = INFLATE_STORED;
       break;
+    default:
+      error("Unsupported clickteam block type %d", (int)(b & 7));
     }
     DUMPBITS (3);
 


Commit: c33ce058f890982fffbafb42f8dc8ee04542c363
    https://github.com/scummvm/scummvm/commit/c33ce058f890982fffbafb42f8dc8ee04542c363
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
COMMON: Add createReadStreamForMemberNext for the ease of using patcher archives

Changed paths:
    common/archive.cpp
    common/archive.h


diff --git a/common/archive.cpp b/common/archive.cpp
index 07400e4d1d0..48f754d7337 100644
--- a/common/archive.cpp
+++ b/common/archive.cpp
@@ -356,6 +356,24 @@ SeekableReadStream *SearchSet::createReadStreamForMember(const Path &path) const
 	return nullptr;
 }
 
+SeekableReadStream *SearchSet::createReadStreamForMemberNext(const Path &path, const Archive *starting) const {
+	if (path.empty())
+		return nullptr;
+
+	ArchiveNodeList::const_iterator it = _list.begin();
+	for (; it != _list.end(); ++it)
+		if (it->_arc == starting) {
+			++it;
+			break;
+		}
+	for (; it != _list.end(); ++it) {
+		SeekableReadStream *stream = it->_arc->createReadStreamForMember(path);
+		if (stream)
+			return stream;
+	}
+
+	return nullptr;
+}
 
 SearchManager::SearchManager() {
 	clear(); // Force a reset
diff --git a/common/archive.h b/common/archive.h
index 1ed647353d8..089d0b7a218 100644
--- a/common/archive.h
+++ b/common/archive.h
@@ -143,6 +143,14 @@ public:
 	 */
 	virtual SeekableReadStream *createReadStreamForMember(const Path &path) const = 0;
 
+	/**
+	 * For most archives: same as previous. For SearchSet see SearchSet
+	 * documentation.
+	 */
+	virtual SeekableReadStream *createReadStreamForMemberNext(const Path &path, const Archive *starting) const {
+		return createReadStreamForMember(path);
+	}
+
 	/**
 	 * Dump all files from the archive to the given directory
 	 */
@@ -342,6 +350,11 @@ public:
 	 */
 	SeekableReadStream *createReadStreamForMember(const Path &path) const override;
 
+	/**
+	 * Similar to above but exclude matches from archives before starting and starting itself.
+	 */
+	SeekableReadStream *createReadStreamForMemberNext(const Path &path, const Archive *starting) const override;
+
 	/**
 	 * Ignore clashes when adding directories. For more details, see the corresponding parameter
 	 * in @ref FSDirectory documentation.


Commit: 6476e55aab6be73bdacb34eef4c80a5195b2d02c
    https://github.com/scummvm/scummvm/commit/6476e55aab6be73bdacb34eef4c80a5195b2d02c
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
COMMON: Allow memcaching archive to bypass caching part and return a stream

It's useful for patchers if they can easily create a stream in some cases but
not others.

Changed paths:
    common/archive.cpp
    common/archive.h


diff --git a/common/archive.cpp b/common/archive.cpp
index 48f754d7337..62b7e277b9e 100644
--- a/common/archive.cpp
+++ b/common/archive.cpp
@@ -107,7 +107,10 @@ SeekableReadStream *MemcachingCaseInsensitiveArchive::createReadStreamForMember(
 	String translated = translatePath(path);
 	bool isNew = false;
 	if (!_cache.contains(translated)) {
-		_cache[translated] = readContentsForPath(translated);
+		SharedArchiveContents readResult = readContentsForPath(translated);
+		if (readResult._bypass)
+			return readResult._bypass;
+		_cache[translated] = readResult;
 		isNew = true;
 	}
 
@@ -121,7 +124,10 @@ SeekableReadStream *MemcachingCaseInsensitiveArchive::createReadStreamForMember(
 	// Check whether the entry is still valid as WeakPtr might have expired.
 	if (!entry->makeStrong()) {
 		// If it's expired, recreate the entry.
-		_cache[translated] = readContentsForPath(translated);
+		SharedArchiveContents readResult = readContentsForPath(translated);
+		if (readResult._bypass)
+			return readResult._bypass;
+		_cache[translated] = readResult;
 		entry = &_cache[translated];
 		isNew = true;
 	}
diff --git a/common/archive.h b/common/archive.h
index 089d0b7a218..462ac4ec279 100644
--- a/common/archive.h
+++ b/common/archive.h
@@ -169,10 +169,15 @@ class SharedArchiveContents {
 public:
 	SharedArchiveContents(byte *contents, uint32 contentSize) :
 		_strongRef(contents, ArrayDeleter<byte>()), _weakRef(_strongRef),
-		_contentSize(contentSize), _missingFile(false) {}
-	SharedArchiveContents() : _strongRef(nullptr), _weakRef(nullptr), _contentSize(0), _missingFile(true) {}
+		_contentSize(contentSize), _missingFile(false), _bypass(nullptr) {}
+	SharedArchiveContents() : _strongRef(nullptr), _weakRef(nullptr), _contentSize(0), _missingFile(true), _bypass(nullptr) {}
+	static SharedArchiveContents bypass(SeekableReadStream *stream) {
+		return SharedArchiveContents(stream);
+	}
 
 private:
+	SharedArchiveContents(SeekableReadStream *stream) : _strongRef(nullptr), _weakRef(nullptr), _contentSize(0), _missingFile(false), _bypass(stream) {}
+
 	bool isFileMissing() const { return _missingFile; }
 	SharedPtr<byte> getContents() const { return _strongRef; }
 	uint32 getSize() const { return _contentSize; }
@@ -197,6 +202,7 @@ private:
 	WeakPtr<byte> _weakRef;
 	uint32 _contentSize;
 	bool _missingFile;
+	SeekableReadStream *_bypass;
 
 	friend class MemcachingCaseInsensitiveArchive;
 };


Commit: f513d6eee988220bf73ecbbcd21d366c28325bcd
    https://github.com/scummvm/scummvm/commit/f513d6eee988220bf73ecbbcd21d366c28325bcd
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
COMMON: Support patching clickteam installer and version used by PRCA

Changed paths:
    common/compression/clickteam.cpp
    common/compression/clickteam.h


diff --git a/common/compression/clickteam.cpp b/common/compression/clickteam.cpp
index c06f3b4f883..934dcb5025a 100644
--- a/common/compression/clickteam.cpp
+++ b/common/compression/clickteam.cpp
@@ -27,48 +27,131 @@
 #include "common/substream.h"
 #include "common/memstream.h"
 
-#define STUB_SIZE 0x16000
 #define FLAG_COMPRESSED 1
 
 namespace Common {
 
 ClickteamInstaller::ClickteamFileDescriptor::ClickteamFileDescriptor(const ClickteamTag& contentsTag, uint32 off)
-  : _fileDataOffset(0), _fileDescriptorOffset(0), _compressedSize(0), _uncompressedSize(0) {
-	uint32 stringsOffset = 36;
-	byte *tag = contentsTag._contents + off;
-	uint32 lmax = contentsTag._size - off;
-	if (lmax < 0x24)
-		return;
-	uint16 ls = READ_LE_UINT32(tag), l;
-	if (ls < 0x24)
-		return;
-
-	l = MIN((uint32)ls, lmax);
-
-	uint16 flags = READ_LE_UINT32(tag+22);
-	if (flags & 6)
-		stringsOffset += 0x12;
-	if (flags & 8)
-		stringsOffset += 0x18;
-	if (stringsOffset >= l) {
-		return;
+	: _fileDataOffset(0), _fileDescriptorOffset(0), _compressedSize(0), _uncompressedSize(0), _isReferenceMissing(false) {
+	switch (contentsTag._tagId) {
+	case (uint16)ClickteamTagId::FILE_LIST: {
+		uint32 stringsOffset = 36;
+		byte *tag = contentsTag._contents + off;
+		uint32 lmax = contentsTag._size - off;
+		if (lmax < 0x24)
+			return;
+		uint32 ls = READ_LE_UINT32(tag), l;
+		if (ls < 0x24)
+			return;
+
+		l = MIN((uint32)ls, lmax);
+
+		uint16 flags = READ_LE_UINT32(tag+22);
+		if (flags & 6)
+			stringsOffset += 0x12;
+		if (flags & 8)
+			stringsOffset += 0x18;
+		if (stringsOffset >= l) {
+			return;
+		}
+		_fileDataOffset = READ_LE_UINT32(tag + 6);
+		_compressedSize = READ_LE_UINT32(tag + 10);
+		_uncompressedSize = READ_LE_UINT32(tag + 14);
+		_expectedCRC = READ_LE_UINT32(tag + 18);
+		char *strings = (char *)tag + stringsOffset;
+		char *p;
+		for (p = strings; p < (char*)tag + lmax && *p; p++);
+		_fileName = Common::String(strings, p - strings);
+		_fileDescriptorOffset = off;
+		_supported = true;
+		_isPatchFile = false;
+		_crcIsXorred = true;
+		break;
+	}
+	case (uint16)ClickteamTagId::FILE_PATCHING_LIST: {
+		uint32 stringsOffset = 0x36;
+		byte *tag = contentsTag._contents + off;
+		uint32 lmax = contentsTag._size - off;
+		if (lmax < 33)
+			return;
+		uint32 ls = READ_LE_UINT32(tag);
+		if (ls < 33)
+			return;
+
+		byte type = tag[7];
+		if (type != 0) {
+			_supported = false;
+			_fileName = "";
+			_fileDataOffset = 0;
+			_fileDescriptorOffset = off;
+			_compressedSize = 0;
+			_uncompressedSize = 0;
+			_expectedCRC = 0;
+			_isPatchFile = false;
+			_crcIsXorred = false;
+			return;
+		}
+
+		// Layout:
+		// 0-3: tag size
+		// 4-6: ???
+		// 7: operation
+		// 8: file type
+		// 9: ???
+		// a-d: uncompressed size
+		// e-11: unxorred uncompressed CRC
+		// 12-15: number of original files entries
+		// 16-17: pointer to original files entries
+		// 18-1d: ???
+		// 1e-36: 3 blocks of 8 bytes relating to some timestamps..
+
+		// Original files entries. Array of:
+		// 0-3: original CRC
+		// 4-7: file size before patching
+		// 8-b: patch data offset
+		// c-f: patch size
+
+		_expectedCRC = READ_LE_UINT32(tag + 0xe);
+		int numPatchEntries = READ_LE_UINT16(tag + 0x12);
+		byte *blockb = tag + READ_LE_UINT16(tag + 0x16);
+
+		_uncompressedSize = READ_LE_UINT32(tag + 0xa);
+		char *strings = (char *)tag + stringsOffset;
+		char *p;
+		for (p = strings; p < (char*)tag + lmax && *p; p++);
+		_fileName = Common::String(strings, p - strings);
+		_fileDescriptorOffset = off;
+		_compressedSize = 0;
+		_fileDataOffset = 0;
+		_supported = true;
+		_isPatchFile = !(tag[8] & 2);
+
+		if (!_isPatchFile && numPatchEntries > 0) {
+			_compressedSize = READ_LE_UINT32(blockb + 0xc);
+			_fileDataOffset = READ_LE_UINT32(blockb + 0x8);
+		}
+
+		_patchEntries.resize(numPatchEntries);
+
+		for (int i = 0; i < numPatchEntries; i++) {
+			_patchEntries[i]._originalCRC = READ_LE_UINT32(blockb + 0x10 * i);
+			_patchEntries[i]._originalSize = READ_LE_UINT32(blockb + 0x10 * i + 4);
+			_patchEntries[i]._patchDataOffset = READ_LE_UINT32(blockb + 0x10 * i + 8);
+			_patchEntries[i]._patchSize = READ_LE_UINT32(blockb + 0x10 * i + 12);
+		}
+
+		_crcIsXorred = false;
+		break;
+	}
 	}
-	_fileDataOffset = READ_LE_UINT32(tag + 6);
-	_compressedSize = READ_LE_UINT32(tag + 10);
-	_uncompressedSize = READ_LE_UINT32(tag + 14);
-	_expectedCRC = READ_LE_UINT32(tag + 18);
-	char *strings = (char *)tag + stringsOffset;
-	char *p;
-	for (p = strings; p < (char*)tag + lmax && *p; p++);
-	_fileName = Common::String(strings, p - strings);
-	_fileDescriptorOffset = off;
 }
 
 ClickteamInstaller::ClickteamTag* ClickteamInstaller::getTag(ClickteamTagId tagId) const {
 	return _tags.getValOrDefault((uint16) tagId).get();
 }
 
-static uint32 computeCRC(byte *buf, uint32 sz, uint32 previous) {
+namespace {
+uint32 computeCRC(byte *buf, uint32 sz, uint32 previous) {
 	uint32 cur = previous;
 	byte *ptr = buf;
 	uint32 i;
@@ -83,47 +166,137 @@ static uint32 computeCRC(byte *buf, uint32 sz, uint32 previous) {
 }
 
 bool checkStubAndComputeCRC1(Common::SeekableReadStream *stream, uint32 &crc) {
-	if (stream->size() <= STUB_SIZE) {
+	static const byte BLOCK1_MAGIC_START[] = { 0x77, 0x77, 0x49, 0x4e, 0x53, 0x53 };
+	static const byte BLOCK1_MAGIC_END[] = { 0x77, 0x77, 0x49, 0x4e, 0x53, 0x45 };
+	static const byte STUB_SIZE_MAGIC[] = { 0x77, 0x77, 0x67, 0x54, 0x29, 0x48 };
+	static const uint32 MAX_SEARCH_RANGE = 0x16000; // So far, if needed increase
+	uint32 blockSearchRange = MIN<uint32>(MAX_SEARCH_RANGE, stream->size());
+
+	if (blockSearchRange <= sizeof(STUB_SIZE_MAGIC) + 4 + sizeof(BLOCK1_MAGIC_START) + sizeof(BLOCK1_MAGIC_END)) {
 		return false;
 	}
 
-	byte *stub = new byte[STUB_SIZE];
-	static const byte BLOCK1_MAGIC_START[] = { 0x77, 0x77, 0x49, 0x4e, 0x53, 0x53 };
-	static const byte BLOCK1_MAGIC_END[] = { 0x77, 0x77, 0x49, 0x4e, 0x53, 0x45 };
+	byte *stub = new byte[blockSearchRange];
 
 	stream->seek(0);
-	stream->read(stub, STUB_SIZE);
+	stream->read(stub, blockSearchRange);
 
+	byte *block1start = nullptr;
+	byte *block1end = nullptr;
+	byte *stubSizePtr = nullptr;
 	byte *ptr;
 
-	for (ptr = stub; ptr < stub + STUB_SIZE - sizeof(BLOCK1_MAGIC_START); ptr++) {
+	for (ptr = stub; ptr < stub + blockSearchRange - sizeof(STUB_SIZE_MAGIC) - 3; ptr++) {
+		if (memcmp(ptr, STUB_SIZE_MAGIC, sizeof(STUB_SIZE_MAGIC)) == 0)
+			stubSizePtr = ptr;
+		if (block1start && memcmp(ptr, BLOCK1_MAGIC_END, sizeof(BLOCK1_MAGIC_END)) == 0)
+			block1end = ptr;
 		if (memcmp(ptr, BLOCK1_MAGIC_START, sizeof(BLOCK1_MAGIC_START)) == 0)
+			block1start = ptr;
+		if (block1start && block1end && stubSizePtr)
 			break;
 	}
 
-	if (ptr == stub + STUB_SIZE - sizeof(BLOCK1_MAGIC_START)) {
+	if (!block1start || !block1end || !stubSizePtr) {
 		delete[] stub;
 		return false;
 	}
 
-	byte *block1start = ptr;
-	ptr += sizeof(BLOCK1_MAGIC_START);
+	uint32 stubSize = READ_LE_UINT32(stubSizePtr + sizeof(STUB_SIZE_MAGIC));
+	crc = computeCRC(block1start, block1end - block1start, 0);
+	delete[] stub;
 
-	for (; ptr < stub + STUB_SIZE - sizeof(BLOCK1_MAGIC_END); ptr++) {
-		if (memcmp(ptr, BLOCK1_MAGIC_END, sizeof(BLOCK1_MAGIC_END)) == 0)
-			break;
+	stream->seek(stubSize);
+	return true;
+}
+
+int32 signExtendAndOffset(uint32 val, int bit, uint32 offset) {
+	if (val & (1 << bit)) {
+		return (val | (0xffffffff << bit)) - offset;
 	}
+	return val + offset;
+}
 
-	if (ptr == stub + STUB_SIZE - sizeof(BLOCK1_MAGIC_END)) {
-		delete[] stub;
-		return false;
+void applyClickteamPatch(Common::WriteStream *outStream, Common::SeekableReadStream *refStream,
+			 Common::SeekableReadStream *patchStream, Common::SeekableReadStream *literalsStream) {
+	uint32 referenceBaseOffset = 0;
+
+	while (!patchStream->eos() && !outStream->err()) {
+                uint32 referenceReadSize = 0;
+		byte patchByte = patchStream->readByte();
+                if (patchByte & 0x80) {
+			uint32 litteralReadSize = (patchByte >> 5) & 3;
+			if (litteralReadSize == 0)
+				litteralReadSize = ((patchByte & 0x1f) + 1);
+			else
+				referenceReadSize = (patchByte & 0x1f) + 2;
+			byte *buf = new byte[litteralReadSize]; // optimize this
+			literalsStream->read(buf, litteralReadSize);
+			outStream->write(buf,litteralReadSize);
+			delete[] buf;
+                } else if (patchByte == 0)
+			referenceReadSize = patchStream->readUint16LE() + 0x81;
+                else
+			referenceReadSize = patchByte + 1;
+                if (referenceReadSize != 0) {
+			int referenceOffsetDelta;
+			patchByte = patchStream->readByte();
+			if (patchByte & 0x80) {
+				if ((patchByte & 0x40) == 0)
+					referenceOffsetDelta = signExtendAndOffset(patchByte & 0x3f, 5, 0);
+				else {
+					referenceOffsetDelta = ((patchByte & 0x3f) << 16) | patchStream->readUint16BE();
+					if (referenceOffsetDelta == 0x100000)
+						referenceOffsetDelta = patchStream->readSint32BE();
+					else
+						referenceOffsetDelta = signExtendAndOffset(referenceOffsetDelta, 21, 0x4020);
+				}
+			} else
+				referenceOffsetDelta = signExtendAndOffset(((patchByte & 0x7f) << 8) | patchStream->readByte(), 14, 0x20);
+			uint32 referenseOffset = referenceOffsetDelta + referenceBaseOffset;
+			byte *buf = new byte[referenceReadSize]; // optimize this
+			if (referenseOffset < refStream->size()) {
+				refStream->seek(referenseOffset);
+				refStream->read(buf, referenceReadSize);
+			} else {
+				memset(buf, 0, referenceReadSize);
+			}
+			// TODO: Handle zero-out blocks. We never encountered any so far
+			outStream->write(buf, referenceReadSize);
+			delete[] buf;
+			referenceBaseOffset += referenceOffsetDelta + referenceReadSize;
+                }
 	}
+}
 
-	byte *block1end = ptr;
-	crc = computeCRC(block1start, block1end - block1start, 0);
-	delete[] stub;
+bool readBlockHeader(Common::SeekableReadStream *stream, uint32 &compressedSize, uint32 &uncompressedSize, bool &isCompressed) {
+	byte codec = stream->readByte();
+	if (codec > 7) {
+		warning("Unknown block codec %d", codec);
+		return false;
+	}
+	if (codec & 2)
+		uncompressedSize = stream->readUint32LE();
+	else
+		uncompressedSize = stream->readUint16LE();
+	switch (codec & 5) {
+	case 5:
+		compressedSize = stream->readUint32LE();
+		break;
+	case 1:
+		compressedSize = stream->readUint16LE();
+		break;
+	case 0:
+		compressedSize = uncompressedSize;
+		break;
+	default:
+		warning("Unknown block codec %d", codec);
+		return false;
+	}
+	isCompressed = codec & 1;
 	return true;
 }
+}  // end of anonymous namespace
 
 struct TagHead {
 	uint16 id;
@@ -131,7 +304,50 @@ struct TagHead {
 	uint32 compressedLen;
 };
 
+int ClickteamInstaller::findPatchIdx(const ClickteamFileDescriptor &desc, Common::SeekableReadStream *refStream,
+				     const Common::String &fileName,
+				     uint32 crcXor, bool doWarn) {
+	bool hasMatching = refStream->size() == desc._uncompressedSize; // Maybe already patched?
+	for (uint i = 0; !hasMatching && i < desc._patchEntries.size(); i++)
+		if (desc._patchEntries[i]._originalSize == refStream->size()) {
+			hasMatching = true;
+			break;
+		}
+	if (!hasMatching) {
+		if (doWarn)
+			warning("Couldn't find matching patch entry for file %s size %d", fileName.c_str(), (int)refStream->size());
+		return -1;
+	}
+	uint32 crcOriginal = 0;
+	{
+		byte buf[0x1000]; // Must be divisible by 4
+		while (!refStream->eos()) {
+			uint32 actual = refStream->read(buf, sizeof(buf));
+			crcOriginal = computeCRC(buf, actual, crcOriginal);
+		}
+	}
+	int patchDescIdx = -1;
+	for (uint i = 0; i < desc._patchEntries.size(); i++)
+		if (desc._patchEntries[i]._originalSize == refStream->size() && desc._patchEntries[i]._originalCRC == (crcOriginal ^ crcXor)) {
+			patchDescIdx = i;
+			break;
+		}
+	 // Maybe already patched if nothing else is found?
+	if (patchDescIdx == -1 && refStream->size() == desc._uncompressedSize && crcOriginal == desc._expectedCRC)
+		return -2;
+	if (patchDescIdx < 0 && doWarn) {
+		warning("Couldn't find matching patch entry for file %s size %d and CRC 0x%x", fileName.c_str(), (int)refStream->size(), crcOriginal);
+	}
+	return patchDescIdx;
+}
+
+
 ClickteamInstaller* ClickteamInstaller::open(Common::SeekableReadStream *stream, DisposeAfterUse::Flag dispose) {
+	return openPatch(stream, false, true, nullptr, dispose);
+}
+
+ClickteamInstaller* ClickteamInstaller::openPatch(Common::SeekableReadStream *stream, bool verifyOriginal, bool verifyAllowSkip,
+						  Common::Archive *reference, DisposeAfterUse::Flag dispose) {
 	Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo> files;
 	HashMap<uint16, Common::SharedPtr<ClickteamTag>> tags;
 	uint32 crc_xor;
@@ -169,12 +385,12 @@ ClickteamInstaller* ClickteamInstaller::open(Common::SeekableReadStream *stream,
 			uint32 uncompressedPayloadLen = READ_LE_UINT32(compressedPayload);
 			byte *uncompressedPayload = new byte[uncompressedPayloadLen];
 			int32 ret = Common::GzioReadStream::clickteamDecompress(uncompressedPayload,
-										  uncompressedPayloadLen,
-										  compressedPayload + 4,
-										  compressedPayloadLen - 4);
+										uncompressedPayloadLen,
+										compressedPayload + 4,
+										compressedPayloadLen - 4);
 			delete[] compressedPayload;
 			if (ret < 0) {
-				debug ("Decompression error");
+				warning("Decompression error for tag 0x%04x", tagId);
 				continue;
 			}
 			tag = new ClickteamTag(tagId, uncompressedPayload, uncompressedPayloadLen);
@@ -183,18 +399,23 @@ ClickteamInstaller* ClickteamInstaller::open(Common::SeekableReadStream *stream,
 		}
 		tags[tagId].reset(tag);
 		switch (tag->_tagId) {
-		case (uint16) ClickteamTagId::FILE_LIST: {
+		case (uint16) ClickteamTagId::FILE_LIST:
+		case (uint16) ClickteamTagId::FILE_PATCHING_LIST: {
 			if (tag->_size < 4) {
 				return nullptr;
 			}
 			uint32 count = READ_LE_UINT32(tag->_contents);
 			uint32 off = 4;
 			for (unsigned i = 0; i < count && off + 0x24 < tag->_size; i++) {
-				uint16 l = READ_LE_UINT16(tag->_contents + off);
-				if (l < 0x24)
+				uint32 l = READ_LE_UINT32(tag->_contents + off);
+				if (l < 33)
 					break;
 				ClickteamFileDescriptor desc(*tag, off);
-				files[desc._fileName] = desc;
+				if (desc._supported) {
+					// Prefer non-patches
+					if (!desc._isPatchFile || ! files.contains(desc._fileName))
+						files[desc._fileName] = desc;
+				}
 				off += l;
 			}
 			break;
@@ -213,7 +434,34 @@ ClickteamInstaller* ClickteamInstaller::open(Common::SeekableReadStream *stream,
 	if (block3_offset <= 0 || block3_len <= 0)
 		return nullptr;
 
-	return new ClickteamInstaller(files, tags, crc_xor, block3_offset, block3_len, stream, dispose);
+	if (verifyOriginal && reference) {
+		for (Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo>::iterator i = files.begin(), end = files.end();
+		     i != end; ++i) {
+			if (i->_value._isPatchFile) {
+				Common::ScopedPtr<Common::SeekableReadStream> refStream(reference->createReadStreamForMember(Common::Path(i->_key, '\\')));
+				if (!refStream) {
+					if (verifyAllowSkip) {
+						i->_value._isReferenceMissing = true;
+						continue;
+					}
+					return nullptr;
+				}
+				if (findPatchIdx(i->_value, refStream.get(), i->_key, crc_xor, false) == -1)
+					return nullptr;
+			}
+		}
+	}
+
+	if (!reference) {
+		for (Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo>::iterator i = files.begin(), end = files.end();
+		     i != end; ++i) {
+			if (i->_value._isPatchFile) {
+				i->_value._isReferenceMissing = true;
+			}
+		}
+	}
+
+	return new ClickteamInstaller(files, tags, crc_xor, block3_offset, block3_len, stream, reference, dispose);
 }
 
 bool ClickteamInstaller::hasFile(const Path &path) const {
@@ -225,8 +473,10 @@ int ClickteamInstaller::listMembers(ArchiveMemberList &list) const {
 
 	for (Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo>::const_iterator i = _files.begin(), end = _files.end();
 	     i != end; ++i) {
-		list.push_back(ArchiveMemberList::value_type(new GenericArchiveMember(i->_key, this)));
-		++members;
+		if (!i->_value._isReferenceMissing) {
+			list.push_back(ArchiveMemberList::value_type(new GenericArchiveMember(i->_key, this)));
+			++members;
+		}
 	}
 
 	return members;
@@ -234,43 +484,131 @@ int ClickteamInstaller::listMembers(ArchiveMemberList &list) const {
 
 const ArchiveMemberPtr ClickteamInstaller::getMember(const Path &path) const {
 	Common::String translated = translatePath(path);
-	if (!_files.contains(translated))
+	ClickteamFileDescriptor el;
+	if (!_files.tryGetVal(translated, el))
 		return nullptr;
 
-	return Common::SharedPtr<Common::ArchiveMember>(new GenericArchiveMember(_files.getVal(translated)._fileName, this));
+	if (el._isReferenceMissing)
+		return nullptr;
+
+	return Common::SharedPtr<Common::ArchiveMember>(new GenericArchiveMember(el._fileName, this));
 }
 
 Common::SharedArchiveContents ClickteamInstaller::readContentsForPath(const Common::String& translated) const {
-	if (!_files.contains(translated))
-		return Common::SharedArchiveContents();
-	ClickteamFileDescriptor desc = _files.getVal(translated);
-	Common::SeekableReadStream *subStream = new Common::SeekableSubReadStream(_stream.get(), _block3Offset + desc._fileDataOffset,
-										  _block3Offset + desc._fileDataOffset + desc._compressedSize);
-	if (!subStream) {
-		debug("Decompression error");
+	ClickteamFileDescriptor desc;
+	byte *uncompressedBuffer = nullptr;
+
+	if (!_files.tryGetVal(translated, desc))
 		return Common::SharedArchiveContents();
-	}
-	Common::ScopedPtr<Common::SeekableReadStream> uncStream(GzioReadStream::openClickteam(subStream, desc._uncompressedSize, DisposeAfterUse::YES));
-	if (!uncStream) {
-		debug("Decompression error");
+	if (desc._isReferenceMissing)
 		return Common::SharedArchiveContents();
-	}
 
-	byte *uncompressedBuffer = new byte[desc._uncompressedSize];
+	if (desc._isPatchFile) {
+		Common::ScopedPtr<Common::SeekableReadStream> refStream(_reference->createReadStreamForMemberNext(Common::Path(translated, '\\'), this));
+		if (!refStream) {
+			warning("Couldn't open reference file for %s. Skipping", translated.c_str());
+			return Common::SharedArchiveContents();
+		}
+		int patchDescIdx = findPatchIdx(desc, refStream.get(), translated, _crcXor, true);
+		if (patchDescIdx == -1 || patchDescIdx < -2)
+			return Common::SharedArchiveContents();
 
-	int64 ret = uncStream->read(uncompressedBuffer, desc._uncompressedSize);
-	if (ret < 0 || ret < desc._uncompressedSize) {
-		debug ("Decompression error");
-		delete[] uncompressedBuffer;
-		return Common::SharedArchiveContents();
+		refStream->seek(0);
+
+		 // Already patched
+		if (patchDescIdx == -2) {
+			return Common::SharedArchiveContents::bypass(refStream.release());
+		}
+
+		uint32 patchDataOffset = _block3Offset + desc._patchEntries[patchDescIdx]._patchDataOffset;
+		_stream->seek(patchDataOffset);
+		uint32 patchCompressedSize, patchUncompressedSize;
+		bool patchIsCompressed;
+		if (!readBlockHeader(_stream.get(), patchCompressedSize, patchUncompressedSize, patchIsCompressed))
+			return Common::SharedArchiveContents();
+
+		uint32 patchStart = _stream->pos();
+		_stream->skip(patchCompressedSize);
+
+		uint32 literalsCompressedSize, literalsUncompressedSize;
+		bool literalsIsCompressed;
+		if (!readBlockHeader(_stream.get(), literalsCompressedSize, literalsUncompressedSize, literalsIsCompressed)) {
+			return Common::SharedArchiveContents();
+		}
+		uint32 literalsStart = _stream->pos();
+
+		Common::ScopedPtr<Common::SeekableReadStream> uncompressedPatchStream, uncompressedLiteralsStream;
+
+		if (patchIsCompressed) {
+			Common::SeekableReadStream *compressedPatchStream = new Common::SafeSeekableSubReadStream(
+				_stream.get(), patchStart, patchStart + patchCompressedSize);
+			if (!compressedPatchStream) {
+				warning("Decompression error");
+				return Common::SharedArchiveContents();
+			}
+
+			uncompressedPatchStream.reset(GzioReadStream::openClickteam(compressedPatchStream, patchUncompressedSize, DisposeAfterUse::YES));
+		} else {
+			uncompressedPatchStream.reset(new Common::SafeSeekableSubReadStream(
+							      _stream.get(), patchStart, patchStart + patchCompressedSize));
+
+		}
+		if (!uncompressedPatchStream) {
+			warning("Decompression error");
+			return Common::SharedArchiveContents();
+		}
+
+		if (literalsIsCompressed) {
+			Common::SeekableReadStream *compressedLiteralsStream = new Common::SafeSeekableSubReadStream(
+				_stream.get(), literalsStart, literalsStart + literalsCompressedSize);
+			if (!compressedLiteralsStream) {
+				warning("Decompression error");
+				return Common::SharedArchiveContents();
+			}
+
+			uncompressedLiteralsStream.reset(GzioReadStream::openClickteam(compressedLiteralsStream, literalsUncompressedSize, DisposeAfterUse::YES));
+		} else {
+			uncompressedLiteralsStream.reset(new Common::SafeSeekableSubReadStream(
+								 _stream.get(), literalsStart, literalsStart + literalsCompressedSize));
+		}
+		if (!uncompressedLiteralsStream) {
+			warning("Decompression error");
+			return Common::SharedArchiveContents();
+		}
+
+		uncompressedBuffer = new byte[desc._uncompressedSize];
+		Common::MemoryWriteStream outStream(uncompressedBuffer, desc._uncompressedSize);
+		applyClickteamPatch(&outStream, refStream.get(), uncompressedPatchStream.get(), uncompressedLiteralsStream.get());
+	} else {
+		Common::SeekableReadStream *subStream = new Common::SeekableSubReadStream(_stream.get(), _block3Offset + desc._fileDataOffset,
+											  _block3Offset + desc._fileDataOffset + desc._compressedSize);
+		if (!subStream) {
+			warning("Decompression error");
+			return Common::SharedArchiveContents();
+		}
+
+		Common::ScopedPtr<Common::SeekableReadStream> uncStream(GzioReadStream::openClickteam(subStream, desc._uncompressedSize, DisposeAfterUse::YES));
+		if (!uncStream) {
+			warning("Decompression error");
+			return Common::SharedArchiveContents();
+		}
+
+		uncompressedBuffer = new byte[desc._uncompressedSize];
+
+		int64 ret = uncStream->read(uncompressedBuffer, desc._uncompressedSize);
+		if (ret < 0 || ret < desc._uncompressedSize) {
+			warning ("Decompression error");
+			delete[] uncompressedBuffer;
+			return Common::SharedArchiveContents();
+		}
 	}
 
 	if (desc._expectedCRC != 0 || !desc._fileName.equalsIgnoreCase("Uninstal.exe")) {
-		uint32 expectedCrc = desc._expectedCRC ^ _crcXor;
+		uint32 expectedCrc = desc._crcIsXorred ? desc._expectedCRC ^ _crcXor : desc._expectedCRC;
 		uint32 actualCrc = computeCRC(uncompressedBuffer, desc._uncompressedSize, 0);
 
 		if (actualCrc != expectedCrc) {
-			debug("CRC mismatch for %s: expected=%08x (obfuscated %08x), actual=%08x", desc._fileName.c_str(), expectedCrc, desc._expectedCRC, actualCrc);
+			warning("CRC mismatch for %s: expected=%08x (obfuscated %08x), actual=%08x (back %08x)", desc._fileName.c_str(), expectedCrc, desc._expectedCRC, actualCrc, actualCrc ^ _crcXor);
 			delete[] uncompressedBuffer;
 			return Common::SharedArchiveContents();
 		}
diff --git a/common/compression/clickteam.h b/common/compression/clickteam.h
index 0eecfe1a007..bd5ef41091b 100644
--- a/common/compression/clickteam.h
+++ b/common/compression/clickteam.h
@@ -35,7 +35,8 @@ public:
 		BANNER_IMAGE = 0x1235,
 		FILE_LIST = 0x123a,
 		STRINGS = 0x123e,
-		UNINSTALLER = 0x123f
+		UNINSTALLER = 0x123f,
+		FILE_PATCHING_LIST = 0x1242,
 	};
 
 	class ClickteamTag : Common::NonCopyable {
@@ -53,16 +54,24 @@ public:
 		}
 	};
 
-	bool hasFile(const Path &path) const override;
+	bool hasFile(const Common::Path &path) const override;
 	int listMembers(Common::ArchiveMemberList&) const override;
-	const ArchiveMemberPtr getMember(const Path &path) const override;
+	const ArchiveMemberPtr getMember(const Common::Path &path) const override;
 	Common::SharedArchiveContents readContentsForPath(const Common::String& translated) const override;
 
 	ClickteamTag* getTag(ClickteamTagId tagId) const;
 
+	static ClickteamInstaller* openPatch(Common::SeekableReadStream *stream, bool verifyOriginal = true, bool verifyAllowSkip = true,
+					     Common::Archive *reference = &SearchMan, DisposeAfterUse::Flag dispose = DisposeAfterUse::NO);
 	static ClickteamInstaller* open(Common::SeekableReadStream *stream, DisposeAfterUse::Flag dispose = DisposeAfterUse::NO);
 
 private:
+	struct ClickteamPatchDescriptor {
+		uint32 _originalCRC;
+		uint32 _originalSize;
+		uint32 _patchDataOffset;
+		uint32 _patchSize;
+	};
 	class ClickteamFileDescriptor {
 	private:
 		Common::String _fileName;
@@ -74,6 +83,12 @@ private:
 		uint32 _compressedSize;
 		uint32 _uncompressedSize;
 		uint32 _expectedCRC;
+		bool _supported;
+		bool _isPatchFile;
+		bool _crcIsXorred;
+		bool _isReferenceMissing;
+		uint16 _field1c;
+		Common::Array<ClickteamPatchDescriptor> _patchEntries;
 
 		ClickteamFileDescriptor(const ClickteamTag& contentsTag, uint32 off);
 		friend class ClickteamInstaller;
@@ -84,14 +99,20 @@ private:
 
 	ClickteamInstaller(Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo> files,
 			   Common::HashMap<uint16, Common::SharedPtr<ClickteamTag>> tags,
-			   uint32 crcXor, uint32 block3Offset, uint32 block3Size, Common::SeekableReadStream *stream, DisposeAfterUse::Flag dispose)
-		: _files(files), _tags(tags), _crcXor(crcXor), _block3Offset(block3Offset), /*_block3Size(block3Size), */_stream(stream, dispose) {
+			   uint32 crcXor, uint32 block3Offset, uint32 block3Size, Common::SeekableReadStream *stream,
+			   Common::Archive *reference,
+			   DisposeAfterUse::Flag dispose)
+		: _files(files), _tags(tags), _crcXor(crcXor), _block3Offset(block3Offset), /*_block3Size(block3Size), */_stream(stream, dispose),
+		  _reference(reference) {
 	}
 
+	static int findPatchIdx(const ClickteamFileDescriptor &desc, Common::SeekableReadStream *refStream, const Common::String &fileName,
+				uint32 crcXor, bool doWarn);
 	Common::HashMap<Common::String, ClickteamFileDescriptor, Common::IgnoreCase_Hash, Common::IgnoreCase_EqualTo> _files;
 	Common::HashMap<uint16, Common::SharedPtr<ClickteamTag>> _tags;
 	Common::DisposablePtr<Common::SeekableReadStream> _stream;
 	uint32 _crcXor, _block3Offset/*, _block3Size*/;
+	Common::Archive *_reference;
 };
 }
 #endif


Commit: 2f70d7dad0eca332b0baacc8a9c47a7150b9d47e
    https://github.com/scummvm/scummvm/commit/2f70d7dad0eca332b0baacc8a9c47a7150b9d47e
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
SCUMM: Detect PRCA patch

Changed paths:
    engines/scumm/detection.h
    engines/scumm/detection_internal.h
    engines/scumm/detection_tables.h


diff --git a/engines/scumm/detection.h b/engines/scumm/detection.h
index 5ccde31d359..e4fb7ab5ebf 100644
--- a/engines/scumm/detection.h
+++ b/engines/scumm/detection.h
@@ -247,6 +247,11 @@ enum ScummGameId {
 	GID_HECUP		// CUP demos
 };
 
+struct RuScummPatcher {
+	ScummGameId gameid;
+	const char *variant;
+	const char *patcherName;
+};
 } // End of namespace Scumm
 
 
diff --git a/engines/scumm/detection_internal.h b/engines/scumm/detection_internal.h
index 8dc5a5f610f..f7e6bb76fba 100644
--- a/engines/scumm/detection_internal.h
+++ b/engines/scumm/detection_internal.h
@@ -24,6 +24,7 @@
 
 #include "common/debug.h"
 #include "common/md5.h"
+#include "common/punycode.h"
 
 #include "gui/error.h"
 
@@ -210,7 +211,7 @@ static bool detectSpeech(const Common::FSList &fslist, const GameSettings *gs) {
 }
 
 // The following function tries to detect the language.
-static Common::Language detectLanguage(const Common::FSList &fslist, byte id, Common::Language originalLanguage = Common::UNK_LANG) {
+static Common::Language detectLanguage(const Common::FSList &fslist, byte id, const char *variant, Common::Language originalLanguage = Common::UNK_LANG) {
 	// First try to detect Chinese translation.
 	Common::FSNode fontFile;
 
@@ -219,6 +220,15 @@ static Common::Language detectLanguage(const Common::FSList &fslist, byte id, Co
 		return Common::ZH_CHN;
 	}
 
+	for (uint i = 0; ruScummPatcherTable[i].patcherName; i++) {
+		Common::FSNode patchFile;
+		if (ruScummPatcherTable[i].gameid == id && (variant == nullptr || strcmp(variant, ruScummPatcherTable[i].variant) == 0)
+		    && searchFSNode(fslist, Common::punycode_decode(ruScummPatcherTable[i].patcherName), patchFile)) {
+			debugC(0, kDebugGlobalDetection, "Russian detected");
+			return Common::RU_RUS;
+		}
+	}
+
 	if (id != GID_CMI && id != GID_DIG) {
 		// Detect Korean fan translated games
 		Common::FSNode langFile;
@@ -385,7 +395,7 @@ static void computeGameSettingsFromMD5(const Common::FSList &fslist, const GameF
 
 				// HACK: Try to detect languages for translated games.
 				if (dr.language == UNK_LANG || dr.language == Common::EN_ANY) {
-					dr.language = detectLanguage(fslist, dr.game.id, dr.language);
+					dr.language = detectLanguage(fslist, dr.game.id, g->variant, dr.language);
 				}
 
 				// HACK: Detect between 68k and PPC versions.
@@ -572,7 +582,7 @@ static void detectGames(const Common::FSList &fslist, Common::List<DetectorResul
 			}
 
 			// HACK: Perhaps it is some modified translation?
-			dr.language = detectLanguage(fslist, g->id);
+			dr.language = detectLanguage(fslist, g->id, g->variant);
 
 			// Detect if there are speech files in this unknown game.
 			if (detectSpeech(fslist, g)) {
diff --git a/engines/scumm/detection_tables.h b/engines/scumm/detection_tables.h
index 2e7fa664e12..6a603509775 100644
--- a/engines/scumm/detection_tables.h
+++ b/engines/scumm/detection_tables.h
@@ -904,6 +904,20 @@ static const GameFilenamePattern gameFilenamesTable[] = {
 	{ NULL, NULL, kGenUnchanged, UNK_LANG, UNK, 0 }
 };
 
+static const RuScummPatcher ruScummPatcherTable[] = {
+	// cmi_rus_1.1b.exe is InstallShield
+	{ GID_MANIAC, "V2", "mm_ega_rus_1.0.exe" },
+	{ GID_MANIAC, "V2", "mm_ega_rus_1.1.exe" },
+	{ GID_INDY3, "VGA", "indy3vga_rus_1.01_final.exe" },
+	{ GID_INDY4, "", "indy_atlantis_rus_v1.0_cd.exe" }, // CD
+	{ GID_LOOM, "EGA", "loom_ega_1.0_rus.exe" },
+	{ GID_LOOM, "EGA", "loom_ega_1.1_rus.exe" },
+	{ GID_LOOM, "FM-TOWNS", "loom_fm_1.0_rus.exe" },
+	{ GID_ZAK, "V2", "zak_ega_rus_1.0.exe" },
+	{ GID_TENTACLE, "", "xn-- . -. .1.0.exe-vtlacb4b3avgf1ea6cs7c2brf8abfez6ajun7b8d6byezgg" }, // CD, "день щупальца. патчер-русификатор. вер.1.0.exe"
+	{ GID_CMI, NULL, NULL }
+};
+
 } // End of namespace Scumm
 
 #endif


Commit: b021957fb378811d8d59efa466b02e16f1d99fb0
    https://github.com/scummvm/scummvm/commit/b021957fb378811d8d59efa466b02e16f1d99fb0
Author: Vladimir Serbinenko (phcoder at gmail.com)
Date: 2023-03-13T20:17:33+01:00

Commit Message:
SCUMM: Support playing most RuSCUMM versions without patching manually

Changed paths:
    engines/scumm/scumm.cpp


diff --git a/engines/scumm/scumm.cpp b/engines/scumm/scumm.cpp
index 5afe9e21b6f..7ef252471c6 100644
--- a/engines/scumm/scumm.cpp
+++ b/engines/scumm/scumm.cpp
@@ -20,6 +20,7 @@
  */
 
 #include "common/config-manager.h"
+#include "common/compression/clickteam.h"
 #include "common/debug-channels.h"
 #include "common/macresman.h"
 #include "common/md5.h"
@@ -40,6 +41,7 @@
 #include "scumm/charset.h"
 #include "scumm/costume.h"
 #include "scumm/debugger.h"
+#include "scumm/detection_tables.h"
 #include "scumm/dialogs.h"
 #include "scumm/file.h"
 #include "scumm/file_nes.h"
@@ -888,6 +890,21 @@ Common::Error ScummEngine::init() {
 
 	const Common::FSNode gameDataDir(ConfMan.get("path"));
 
+	for (uint i = 0; ruScummPatcherTable[i].patcherName; i++) {
+		if (ruScummPatcherTable[i].gameid == _game.id && (_game.variant == nullptr || strcmp(_game.variant, ruScummPatcherTable[i].variant) == 0)) {
+			Common::File *f = new Common::File();
+			if (f->open(ruScummPatcherTable[i].patcherName)) {
+				Common::Archive *patcher = Common::ClickteamInstaller::openPatch(f, true, true, &SearchMan, DisposeAfterUse::YES);
+				if (patcher) {
+					SearchMan.add("ruscumm", patcher, 3);
+					break;
+				}
+			}
+			delete f;
+		}
+	}
+
+
 	ConfMan.registerDefault("original_gui", true);
 	if (ConfMan.hasKey("original_gui", _targetName)) {
 		_useOriginalGUI = ConfMan.getBool("original_gui");

More information about the Scummvm-git-logs mailing list