[Scummvm-git-logs] scummvm-sites integrity -> a7fb08e10cfd8eac890ef35f3bc990f990614f24
sev-
noreply at scummvm.org
Sat Nov 18 17:20:54 UTC 2023
This automated email contains information about 170 new commits which have been
pushed to the 'scummvm-sites' repo located at https://github.com/scummvm/scummvm-sites .
Summary:
bdd1802260 INTEGRITY: Add database schema file
ef382145ac INTEGRITY: Add database seed file
50d6c050ef INTEGRITY: Add PHP script for parsing DAT files
fbbcddb6bd INTEGRITY: Parse checksum data into key-value pair
7d71eb258c INTEGRITY: Wrap parsing code into its own function
83a1f7720e INTEGRITY: Add function to insert data into db
fc90472d2c INTEGRITY: Optimizations for dat_parser.php
78a8482c2b INTEGIRTY: Add compute_hash.py
b493e8bb31 INTEGRITY: Calculating various sizes of checksum
1a6843f12b INTEGRITY: Add command line args and options
b6b55571b7 INTEGRITY: Add function to create DAT file
e77641517e INTEGRITY: Add minor changes to DAT format
8ea71e27a4 INTEGRITY: Updates to parser
6a85968d3e Update database schema
f5488bc65d Add field `extra` to `game` table
5129972cfa INTEGRITY: Update type of file.size to BIGINT
f59bcdd6dc INTEGRITY: Update parser for detection entries
37d2616313 INTEGRITY: Add t: to tail checksums, update header
7465b20844 INTEGRITY: Create find_matching_game()
cf2b8c8a1b INTEGRITY: Update schema to contain indices
5a1303e486 INTEGRITY: Match game with filesets
b51b98e333 INTEGRITY: Separate game matching and db insertion
3e58080581 INTEGRITY: Create basic website with pagination
865a3925e7 INTEGRITY: Add page for browsing logs
c401b6c5b0 INTEGRITY: Add first,last page button to index.php
af0104cf55 INTEGRITY: Create log table, index engine.engineid
78b307cf9b INTEGRITY: Create log on matching game files
0aacb1ce26 INTEGRITY: Add depth option to compute_hash.py
23791d3177 INTEGRITY: Add mysql_config file for username/pass
7221a59fff INTEGRITY: Remove some PHP8 functions
accd4345f2 INTEGRITY: Calc fileset.key for detection entries
d24480df55 INTEGRITY: Create log on inserting into the db
5e7ac07d55 INTEGRITY: Fix name of files when using depth option
2d033622b6 INTEGRITY: Add version to header of DAT file
96096724b2 INTEGRITY: Add ON DELETE CASCADE to file, filechecksum
0c730a0092 INTEGRITY: Merge filesets on detection entry match
c75b15ef41 INTEGRITY: Fix formatting and text for webpages
77cc374020 INTEGRITY: Merge filesets based on all checksums
0ad938c731 INTEGRITY: Copy checksize, checktype to new entries
8ae58943e6 INTEGRITY: Remove status unconfirmed
e862900d41 INTEGRITY: Rename index.php to games_list.php
bb3507030f INTEGRITY: Create (new) index.php with page links
dfa9e869da INTEGRITY: Create fileset `history` table
748a35e341 INTEGRITY: games_list.php now uses table for data
3b62839dc2 INTEGRITY: Extract duplicated code into remove_quotes()
f2e799542e INTEGRITY: Improve styling for games_list.php
b0ce11fa04 INTEGRITY: Improve navigation styling
f65e67fecf INTEGRITY: Create pagination module
ef25db858e INTEGRITY: Add .gitignore, README and server config
4ea3eb0347 INTEGRITY: Updated table styling
203138d1be INTEGRITY: Add newlines to HTML tags
0b8a823c73 INTEGRITY: Add filters to games_list.php
ad8c9f2379 INTEGRITY: Filter now works for all columns
aac2dd095c INTEGRITY: Add filters to logs.php
8b0acd255c INTEGRITY: Replace value dropdown with textbox
b18651a284 INTEGRITY: Update apache to deny access to more files
24ebd9953f INTEGRITY: Update references of CLI to scan
13ce72e697 INTEGRITY: Add CLI options to dat_parser.php
097236bd6f INTEGRITY: Add support for uploading multiple DATs
1995d95f93 INTEGRITY: Clamp $page value to (1, $num_of_pages)
f734004b41 INTEGRITY: Preserve query variables across pages
89e371f0a0 INTEGRITY: Add text field to manually type page no
fbe6c1f29c INTEGRITY: Fetch db, servername from mysql config
090028d3c5 INTEGRITY: Update matching
8c22dac99a INTEGRITY: Create fileset webpage
6de87602e8 INTEGRITY: Fix - Duplicates deleted while matching
91a947a0e6 INTEGRITY: Remove borders from table
74c204f63b INTEGRITY: Add fileset data to fileset.php
f5410f5a45 INTEGRITY: Fileset files displayed with pagination
7dc630bdc2 INTEGRITY: Add fileset to log text
2e39b3a538 INTEGRITY: Add hyperlinks to fileset in logs.php
3a4f001e6f INTEGRITY: Add username to logs
9b0623370b INTEGRITY: Add logs to fileset history
8294ef1a6c INTEGRITY: Add Developer Actions to fileset page
62ff29f34f INTEGRITY: Add explicit ordering to pagination.php
af096c5646 INTEGRITY: Clamp min fileset id
1d6eb1d74c INTEGRITY: Update styling
e66c454f2e INTEGRITY: Display page navigation only on >1 page
4e7b47efb0 INTEGRITY: Add priority order for matching
314da9ce62 INTEGRITY: Add support for MacBinary checksums
bf8fd58032 INTEGRITY: Add support for custom checksum size
1075289c45 INTEGRITY: Extend MacBinary support
ec59d32447 INTEGRITY: Add check for MacBinary
403572ae85 INTEGRITY: Punyencode filenames if required
eff74a1813 INTEGRITY: Filters now present on each column
798fb543db INTEGRITY: Fix filtering over multiple tables
f16881e434 INTEGRITY: Add support for engine names/desc
507b15c5e2 INTEGRITY: Skip duplicate detection entries
c39736329b INTEGRITY: Add timestamp to fileset table
a9c32f5d5a INTEGRITY: Existing filesets are marked for deletion
22356ae983 INTEGRITY: Delete games without detection entries
3c0b75c563 INTEGRITY: Skip insertion of existing fileset
12c978776d INTEGRITY: Fix MacBinary check, add new CRC16 func
ef0c8a4108 INTEGRITY: Fix broken $checksize check
fe8ed8ef01 INTEGRITY: Sanitize values of filters
4446cdeca2 INTEGRITY: Skip insertion of duplicate DATs
4ad5db171d INTEGRITY: Add links to filesets of game entries
cbad3a300b INTEGRITY: Add all file checksums to fileset page
89c380da54 INTEGRITY: Index fileset.status for better perf
b8835088d2 INTEGRITY: Add prev and next buttons to navigation
9f60de0aa9 INTEGRITY: Remove hardcoded exception to filtering
17f0a4cc30 INTEGRITY: Fix - empty variables in URL
a3b045cfe8 INTEGRITY: Add expand table button to fileset page
681af1fd58 INTEGRITY: Update check for checktype
30bda49997 INTEGRITY: Create file to validate game files
7c93faf95f INTEGRITY: Check that checksum is not empty
961433bf28 INTEGRITY: Update JSON request schema
09a6d2e6b8 INTEGRITY: Update formatting for schema.php
0e537969ef INTEGRITY: Add detection_size to fileset table
02223fbc58 INTEGRITY: Add special checksum columns
5d154f93aa INTEGRITY: Fix comment in pagination.php
1678ae13fe INTEGRITY Checksum validation for user files
96b78ae53f INTEGRITY: Bugfixes for dat_parser.php
2eff930220 INTEGRITY: Change format of tail checksums
2773978c0a INTEGRITY: Checksum validation for user filesets
1c747e4ffd INTEGRITY: Fix warning while insertion of DATs
32f804546a INTEGRITY: Refactor - Create file db_functions.php
7ffbef0a50 INTEGRITY: Update error logs to end with newlines
e8519ab2b8 INTEGRITY: Improve formatting for schema.php
a3bb29e701 INTEGRITY: Update format of checkcodes
ecaadeafa1 INTEGRITY: Add functions to add user fileset to db
358e252540 INTEGRITY: Change format of checkcode
0e047b2bdf INTEGRITY: Fix punyencoding for filepaths
452ad3c9df INTEGRITY: Support 'm:' type checksums in scanner
0358983805 INTEGRITY: Fix macbin_get_resfork()
d6541a10a9 INTEGRITY: Add support for AppleDouble and .rsrc files
9ac93e363d INTEGRITY: Update datafork for AppleDouble, .rsrc
e4ce93644b INTEGRITY: Skip crc and sha1 checksums from DATs
6984317465 INTEGRITY: Use full md5 in file.checksum
7d9c180bd9 INTEGRITY: Fix $num_of_results while filtering
e915c4f81a INTEGRITY: Skip showing md5-0 in widetable fileset
d7d1d39daf INTEGRITY: Change log table order, filters
2b1ac71188 INTEGRITY: Add transactions table
ec9c0a5fcb INTEGRITY: Add message if paginated table is empty
d38e845990 INTEGRITY: Create log entries for every fileset
0989136e50 INTEGRITY: Update calc_key() for detection entries
b1d84ea6ac INTEGRITY: Make games_list rows link to filesets
863e4612f3 INTEGRITY: Add sorting to all paginated columns
7a321265c4 INTEGRITY: Full user checksums used for validation
c45e850a37 INTEGRITY: Add insert_user_queue() for user queue
bbd9cf8d2c INTEGRITY: Restructure source code
0fa955ee0b INTEGRITY: Create apache config file .htaccess
0675081f03 INTEGRITY: Fix variable not defined error
84bfff264c INTEGRITY: Create endpoint for checksum validation
c73a95096a INTEGRITY: Rename checksum_validation
161abc5bda INTEGRITY: Update fileset sorting algorithm
3dc5a3a86f INTEGRITY: Insert user fileset on unknown variants
3b7527c438 INTEGRITY: Fix user fileset insertion
9fd1d86f7e INTEGRITY: Include paths now use absolute paths
49d8b4dcd4 INTEGRITY: Fix user fileset insertion
5cc750df0d INTEGRITY: fileset.key no longer uses gameid
8a99da8324 INTEGRITY: Update response for unknown filesets
7115f285c7 INTEGRITY: Add delete_filesets for marked filesets
fce070e8ad INTEGRITY: Display user submitted filesets
a030fa15ce INTEGRITY: Chnage error code for unknown variants
b29080e508 INTEGRITY: Punyencode each directory in path
9c1bdb508c INTEGRITY: Handle filesets with no metadata/files
bae271732d INTEGRITY: Fix require paths
4b54e96fff INTEGRITY: User filesets added on all mismatches
1942aba276 INTEGRITY: Fix user fileset id in JSON response
7ee0516f6d INTEGRITY: Display error on providing wrong args
329b0989d7 INTEGRITY: Fix logs page no in fileset history
11ac2b3117 INTEGRITY: Improve fileset history in fileset.php
e4b3497ecb INTEGRITY: Add ip address in user fileset logs
e6db6133ad INTEGRITY: Remove DAT filename from fileset logs
6e4705b61a INTEGRITY: Add button to match user fileset
bb8d7fe7b1 INTEGRITY: Hyperlink in fileset only shows one log
8348d707e9 UNFINISHED: Add mod_actions developer dashboard
9acec1b3d5 INTEGRITY: Fix user filesets matching and merging
a7fb08e10c INTEGRITY: Add no_metadata error code
Commit: bdd18022603e49a4044156884f35c977821a745c
https://github.com/scummvm/scummvm-sites/commit/bdd18022603e49a4044156884f35c977821a745c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-05-29T20:57:32+05:30
Commit Message:
INTEGRITY: Add database schema file
Changed paths:
A schema.php
diff --git a/schema.php b/schema.php
new file mode 100644
index 0000000..964738e
--- /dev/null
+++ b/schema.php
@@ -0,0 +1,149 @@
+<?php
+
+$servername = "localhost";
+$username = "username";
+$password = "password";
+$dbname = "integrity";
+
+// Create connection
+$conn = new mysqli($servername, $username, $password);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+// Create database
+$sql = "CREATE DATABASE IF NOT EXISTS " . $dbname;
+if ($conn->query($sql) === TRUE) {
+ echo "Database created successfully<br/>";
+}
+else {
+ echo "Error creating database: " . $conn->error;
+ exit();
+}
+
+$conn->query("USE " . $dbname);
+
+
+///////////////////////// CREATE TABLES /////////////////////////
+
+// Create engine table
+$table = "CREATE TABLE IF NOT EXISTS engine (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ name VARCHAR(200) NOT NULL,
+ engineid VARCHAR(100) NOT NULL
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'engine' created successfully<br/>";
+}
+else {
+ echo "Error creating 'engine' table: " . $conn->error;
+}
+
+// Create game table
+$table = "CREATE TABLE IF NOT EXISTS game (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ name VARCHAR(200) NOT NULL,
+ engine INT NOT NULL,
+ gameid VARCHAR(100) NOT NULL,
+ FOREIGN KEY (engine) REFERENCES engine(id)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'game' created successfully<br/>";
+}
+else {
+ echo "Error creating 'game' table: " . $conn->error;
+}
+
+// Create file table
+$table = "CREATE TABLE IF NOT EXISTS file (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ name VARCHAR(200) NOT NULL,
+ size INT NOT NULL,
+ checksum VARCHAR(64) NOT NULL
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'file' created successfully<br/>";
+}
+else {
+ echo "Error creating 'file' table: " . $conn->error;
+}
+
+// Create fileset table
+$table = "CREATE TABLE IF NOT EXISTS fileset (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ game INT NOT NULL,
+ file INT NOT NULL,
+ status INT,
+ `key` VARCHAR(64),
+ FOREIGN KEY (game) REFERENCES game(id),
+ FOREIGN KEY (file) REFERENCES file(id)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'fileset' created successfully<br/>";
+}
+else {
+ echo "Error creating 'fileset' table: " . $conn->error;
+}
+
+// Create filechecksum table
+$table = "CREATE TABLE IF NOT EXISTS filechecksum (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ file INT NOT NULL,
+ checksize INT NOT NULL,
+ checktype INT NOT NULL,
+ checksum VARCHAR(64) NOT NULL,
+ FOREIGN KEY (file) REFERENCES file(id)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'filechecksum' created successfully<br/>";
+}
+else {
+ echo "Error creating 'filechecksum' table: " . $conn->error;
+}
+
+// Create fileset_detection table
+$table = "CREATE TABLE IF NOT EXISTS fileset_detection (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ fileset INT NOT NULL,
+ checksum INT NOT NULL,
+ FOREIGN KEY (fileset) REFERENCES fileset(id),
+ FOREIGN KEY (checksum) REFERENCES filechecksum(id)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'fileset_detection' created successfully<br/>";
+}
+else {
+ echo "Error creating 'fileset_detection' table: " . $conn->error;
+}
+
+// Create queue table
+$table = "CREATE TABLE IF NOT EXISTS queue (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ date DATETIME NOT NULL,
+ notes varchar(300),
+ fileset INT NOT NULL,
+ ticketid INT NOT NULL,
+ userid INT NOT NULL,
+ commit VARCHAR(64) NOT NULL,
+ FOREIGN KEY (fileset) REFERENCES fileset(id)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'queue' created successfully<br/>";
+}
+else {
+ echo "Error creating 'queue' table: " . $conn->error;
+}
+
+
+$conn->close();
+?>
+
Commit: ef382145ac74343d7e005ad25d0012f9ef41d54d
https://github.com/scummvm/scummvm-sites/commit/ef382145ac74343d7e005ad25d0012f9ef41d54d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-05-31T21:20:57+05:30
Commit Message:
INTEGRITY: Add database seed file
- Add seeds for all tables except queue
- filechecksum table modified; checktype is now VARCHAR
- checksumsize of 0 denotes full file
Changed paths:
A seeds.php
schema.php
diff --git a/schema.php b/schema.php
index 964738e..04d59c1 100644
--- a/schema.php
+++ b/schema.php
@@ -96,7 +96,7 @@ $table = "CREATE TABLE IF NOT EXISTS filechecksum (
id INT AUTO_INCREMENT PRIMARY KEY,
file INT NOT NULL,
checksize INT NOT NULL,
- checktype INT NOT NULL,
+ checktype VARCHAR(10) NOT NULL,
checksum VARCHAR(64) NOT NULL,
FOREIGN KEY (file) REFERENCES file(id)
)";
diff --git a/seeds.php b/seeds.php
new file mode 100644
index 0000000..113524b
--- /dev/null
+++ b/seeds.php
@@ -0,0 +1,74 @@
+<?php
+
+$servername = "localhost";
+$username = "username";
+$password = "password";
+$dbname = "integrity";
+
+// Create connection
+$conn = new mysqli($servername, $username, $password);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+$conn->query("USE " . $dbname);
+
+
+///////////////////////// INSERT VALUES /////////////////////////
+
+$query = "INSERT INTO engine (name, engineid)
+VALUES ('Drascula', '1')";
+$conn->query($query);
+$conn->query("SET @engine_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO game (name, engine, gameid)
+VALUES ('Drascula: The Vampire Strikes Back', @engine_last, '1')";
+$conn->query($query);
+$conn->query("SET @game_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO file (name, size, checksum)
+VALUES ('Packet.001', '32847563', 'fac946707f07d51696a02c00cc182078')";
+$conn->query($query);
+$conn->query("SET @file_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO fileset (game, file, status, `key`)
+VALUES (@game_last, @file_last, 0, 'fac946707f07d51696a02c00cc182078')";
+$conn->query($query);
+$conn->query("SET @fileset_last = LAST_INSERT_ID()");
+
+// Checksize: 0 (full checksum)
+$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum)
+VALUES (@file_last, '0', 'md5', 'fac946707f07d51696a02c00cc182078')";
+$conn->query($query);
+$conn->query("SET @filechecksum_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO fileset_detection (fileset, checksum)
+VALUES (@fileset_last, @filechecksum_last)";
+$conn->query($query);
+
+// Checksize: 5000B
+$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum)
+VALUES (@file_last, '5000', 'md5', 'c6a8697396e213a18472542d5f547cb4')";
+$conn->query($query);
+$conn->query("SET @filechecksum_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO fileset_detection (fileset, checksum)
+VALUES (@fileset_last, @filechecksum_last)";
+$conn->query($query);
+
+// Checksize: 10000B
+$query = "INSERT INTO filechecksum (file, checksize, checktype, checksum)
+VALUES (@file_last, '10000', 'md5', '695f4152f02b8fa4c1374a0ed04cf996')";
+$conn->query($query);
+$conn->query("SET @filechecksum_last = LAST_INSERT_ID()");
+
+$query = "INSERT INTO fileset_detection (fileset, checksum)
+VALUES (@fileset_last, @filechecksum_last)";
+$conn->query($query);
+
+
+$conn->close();
+?>
+
Commit: 50d6c050ef53249cc60125c33a4cd72263b6ac84
https://github.com/scummvm/scummvm-sites/commit/50d6c050ef53249cc60125c33a4cd72263b6ac84
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-05T17:19:44+05:30
Commit Message:
INTEGRITY: Add PHP script for parsing DAT files
Changed paths:
A dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
new file mode 100644
index 0000000..ffe6f94
--- /dev/null
+++ b/dat_parser.php
@@ -0,0 +1,70 @@
+<?php
+
+$dat_filepath = "ngi.dat";
+
+$dat_file = fopen($dat_filepath, "r") or die("Unable to open file!");
+$content = fread($dat_file, filesize($dat_filepath));
+
+if (!$content) {
+ error_log("File not readable");
+}
+
+/**
+ * Convert string as received by regex parsing to associated array
+ */
+function map_key_values($content_string, &$arr) {
+
+ // Split by newline into different pairs
+ $temp = preg_split("/\r\n|\n|\r/", $content_string);
+
+ // Add pairs to the associated array if they are not parantheses
+ foreach ($temp as $pair) {
+ if (trim($pair) == "(" or trim($pair) == ")")
+ continue;
+ $pair = array_map("trim", preg_split("/ +/", $pair, 2));
+
+ // Remove quotes from value if they are present
+ if ($pair[1][0] == "\"")
+ $pair[1] = substr($pair[1], 1, -1);
+
+ // Handle duplicate keys (if the key is rom) and add values to a arary instead
+ if ($pair[0] == "rom") {
+ if ($arr[$pair[0]]) {
+ array_push($arr[$pair[0]], $pair[1]);
+ }
+ else {
+ $arr[$pair[0]] = array($pair[1]);
+ }
+ }
+ else {
+ $arr[$pair[0]] = $pair[1];
+ }
+ }
+}
+
+$header = array();
+$game_data = array();
+$resources = array();
+
+$matches = array();
+$header_exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
+if (preg_match_all($header_exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
+ foreach ($matches[0] as $data_segment) {
+ if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false) {
+ map_key_values($data_segment[0], $header);
+ }
+ elseif (strpos(substr($content, $data_segment[1] - 5, $data_segment[1]), "game") !== false) {
+ $temp = array();
+ map_key_values($data_segment[0], $temp);
+ array_push($game_data, $temp);
+ }
+ elseif (strpos(substr($content, $data_segment[1] - 9, $data_segment[1]), "resource") !== false) {
+ map_key_values($data_segment[0], $resources);
+ }
+ }
+
+}
+
+fclose($dat_file);
+?>
+
Commit: fbbcddb6bd3ce00af7fee32f9363afafa2ed367f
https://github.com/scummvm/scummvm-sites/commit/fbbcddb6bd3ce00af7fee32f9363afafa2ed367f
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-06T08:53:34+05:30
Commit Message:
INTEGRITY: Parse checksum data into key-value pair
- Add debug statements to see the result of the parsing
- Get rid of key not found warning
- Fix variable names
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index ffe6f94..4951de7 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -9,6 +9,23 @@ if (!$content) {
error_log("File not readable");
}
+/**
+ * Convert string of checksum data from rom into associated array
+ * Returns array instead of updating one like map_key_values
+ */
+function map_checksum_data($content_string) {
+ $arr = array();
+ $temp = preg_split("/\s/", $content_string);
+
+ for ($i = 1; $i < count($temp); $i += 2) {
+ if ($temp[$i] == ')')
+ continue;
+ $arr[$temp[$i]] = $temp[$i + 1];
+ }
+
+ return $arr;
+}
+
/**
* Convert string as received by regex parsing to associated array
*/
@@ -29,11 +46,11 @@ function map_key_values($content_string, &$arr) {
// Handle duplicate keys (if the key is rom) and add values to a arary instead
if ($pair[0] == "rom") {
- if ($arr[$pair[0]]) {
- array_push($arr[$pair[0]], $pair[1]);
+ if (array_key_exists($pair[0], $arr)) {
+ array_push($arr[$pair[0]], map_checksum_data($pair[1]));
}
else {
- $arr[$pair[0]] = array($pair[1]);
+ $arr[$pair[0]] = array(map_checksum_data($pair[1]));
}
}
else {
@@ -47,8 +64,8 @@ $game_data = array();
$resources = array();
$matches = array();
-$header_exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
-if (preg_match_all($header_exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
+$exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
+if (preg_match_all($exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
foreach ($matches[0] as $data_segment) {
if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false) {
map_key_values($data_segment[0], $header);
@@ -65,6 +82,15 @@ if (preg_match_all($header_exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
}
+// Print statements for debugging
+// Uncomment to see parsed data
+
+// echo "<pre>";
+// print_r($header);
+// print_r($game_data);
+// print_r($resources);
+// echo "</pre>";
+
fclose($dat_file);
?>
Commit: 7d71eb258c130b0e1c485a245cc72c27179dd1b2
https://github.com/scummvm/scummvm-sites/commit/7d71eb258c130b0e1c485a245cc72c27179dd1b2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-07T11:16:20+05:30
Commit Message:
INTEGRITY: Wrap parsing code into its own function
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 4951de7..6cb041b 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -1,13 +1,7 @@
<?php
-$dat_filepath = "ngi.dat";
+// $dat_filepath = "ngi.dat";
-$dat_file = fopen($dat_filepath, "r") or die("Unable to open file!");
-$content = fread($dat_file, filesize($dat_filepath));
-
-if (!$content) {
- error_log("File not readable");
-}
/**
* Convert string of checksum data from rom into associated array
@@ -59,38 +53,56 @@ function map_key_values($content_string, &$arr) {
}
}
-$header = array();
-$game_data = array();
-$resources = array();
+/**
+ * Take DAT filepath as input and return parsed data in the form of
+ * associated arrays
+ */
+function parse_dat($dat_filepath) {
+ $dat_file = fopen($dat_filepath, "r") or die("Unable to open file!");
+ $content = fread($dat_file, filesize($dat_filepath));
-$matches = array();
-$exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
-if (preg_match_all($exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
- foreach ($matches[0] as $data_segment) {
- if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false) {
- map_key_values($data_segment[0], $header);
- }
- elseif (strpos(substr($content, $data_segment[1] - 5, $data_segment[1]), "game") !== false) {
- $temp = array();
- map_key_values($data_segment[0], $temp);
- array_push($game_data, $temp);
- }
- elseif (strpos(substr($content, $data_segment[1] - 9, $data_segment[1]), "resource") !== false) {
- map_key_values($data_segment[0], $resources);
+ if (!$content) {
+ error_log("File not readable");
+ }
+
+ $header = array();
+ $game_data = array();
+ $resources = array();
+
+ $matches = array();
+ $exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
+ if (preg_match_all($exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
+ foreach ($matches[0] as $data_segment) {
+ if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false) {
+ map_key_values($data_segment[0], $header);
+ }
+ elseif (strpos(substr($content, $data_segment[1] - 5, $data_segment[1]), "game") !== false) {
+ $temp = array();
+ map_key_values($data_segment[0], $temp);
+ array_push($game_data, $temp);
+ }
+ elseif (strpos(substr($content, $data_segment[1] - 9, $data_segment[1]), "resource") !== false) {
+ map_key_values($data_segment[0], $resources);
+ }
}
+
}
-}
+ // Print statements for debugging
+ // Uncomment to see parsed data
+
+ // echo "<pre>";
+ // print_r($header);
+ // print_r($game_data);
+ // print_r($resources);
+ // echo "</pre>";
-// Print statements for debugging
-// Uncomment to see parsed data
+ fclose($dat_file);
+
+ return array($header, $game_data, $resources);
+}
-// echo "<pre>";
-// print_r($header);
-// print_r($game_data);
-// print_r($resources);
-// echo "</pre>";
+parse_dat("ngi.dat");
-fclose($dat_file);
?>
Commit: 83a1f7720e940ce3d3d2c118f72ee639ba9e557c
https://github.com/scummvm/scummvm-sites/commit/83a1f7720e940ce3d3d2c118f72ee639ba9e557c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-07T19:38:19+05:30
Commit Message:
INTEGRITY: Add function to insert data into db
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 6cb041b..c1b4330 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -1,7 +1,6 @@
<?php
-// $dat_filepath = "ngi.dat";
-
+ini_set('memory_limit', '512M');
/**
* Convert string of checksum data from rom into associated array
@@ -102,7 +101,82 @@ function parse_dat($dat_filepath) {
return array($header, $game_data, $resources);
}
-parse_dat("ngi.dat");
+/**
+ * Routine for inserting a file into the database, intserting into all
+ * required tables
+ * $file is an associated array (the contents of 'rom')
+ * If checksum of the given checktype doesn't exists, silently fails
+ */
+function insert_file($file, $key, $conn, $checktype = "md5", $checksize = 0) {
+ if (!array_key_exists($checktype, $file))
+ return;
+
+ $query = sprintf("INSERT INTO file (name, size, checksum)
+ VALUES ('%s', '%s', '%s')", mysqli_real_escape_string($conn, $file["name"]),
+ $file["size"], $file[$checktype]);
+ $conn->query($query);
+ $conn->query("SET @file_last = LAST_INSERT_ID()");
+
+ $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
+ VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $file[$checktype]);
+ $conn->query($query);
+ $conn->query("SET @filechecksum_last = LAST_INSERT_ID()");
+
+ $query = sprintf("INSERT INTO fileset (game, file, status, `key`)
+ VALUES (NULL, @file_last, 0, '%s')", $key);
+ $conn->query($query);
+ $conn->query("SET @fileset_last = LAST_INSERT_ID()");
+
+ $query = "INSERT INTO fileset_detection (fileset, checksum)
+ VALUES (@fileset_last, @filechecksum_last)";
+ $conn->query($query);
+}
+
+/**
+ * Insert values from the associated array into the DB
+ * They will be inserted under gameid NULL as the game itself is unconfirmed
+ */
+function db_insert($data_arr) {
+ $header = $data_arr[0];
+ $game_data = $data_arr[1];
+ $resources = $data_arr[2];
+
+ $servername = "localhost";
+ $username = "username";
+ $password = "password";
+ $dbname = "integrity";
+
+ // Create connection
+ $conn = new mysqli($servername, $username, $password);
+
+ // Check connection
+ if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+ }
+
+ $conn->query("USE " . $dbname);
+
+ // Remove "ScummVM" from engine name
+ $engine_name = preg_split("/\s/", $header["name"]);
+ array_shift($engine_name);
+ $engine_name = implode(" ", $engine_name);
+
+ $query = sprintf("INSERT INTO engine (name, engineid)
+ VALUES ('%s', '1')", $engine_name);
+ $conn->query($query);
+ $conn->query("SET @engine_last = LAST_INSERT_ID()");
+
+ foreach ($game_data as $fileset) {
+ foreach ($fileset["rom"] as $file) {
+ $key = NULL;
+ insert_file($file, $key, $conn);
+ insert_file($file, $key, $conn, "crc");
+ insert_file($file, $key, $conn, "sha1");
+ }
+ }
+}
+
+// db_insert(parse_dat("ngi.dat"));
?>
diff --git a/schema.php b/schema.php
index 04d59c1..5aabd70 100644
--- a/schema.php
+++ b/schema.php
@@ -76,7 +76,7 @@ else {
// Create fileset table
$table = "CREATE TABLE IF NOT EXISTS fileset (
id INT AUTO_INCREMENT PRIMARY KEY,
- game INT NOT NULL,
+ game INT,
file INT NOT NULL,
status INT,
`key` VARCHAR(64),
Commit: fc90472d2ca9a8171cfe3d051c17235bb1777e2e
https://github.com/scummvm/scummvm-sites/commit/fc90472d2ca9a8171cfe3d051c17235bb1777e2e
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-08T12:06:10+05:30
Commit Message:
INTEGRITY: Optimizations for dat_parser.php
- Wrap insertion into a transaction, greatly improving speed
- Close file after reading content to avoid wasting memory
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index c1b4330..66777f8 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -59,6 +59,7 @@ function map_key_values($content_string, &$arr) {
function parse_dat($dat_filepath) {
$dat_file = fopen($dat_filepath, "r") or die("Unable to open file!");
$content = fread($dat_file, filesize($dat_filepath));
+ fclose($dat_file);
if (!$content) {
error_log("File not readable");
@@ -96,8 +97,6 @@ function parse_dat($dat_filepath) {
// print_r($resources);
// echo "</pre>";
- fclose($dat_file);
-
return array($header, $game_data, $resources);
}
@@ -147,7 +146,10 @@ function db_insert($data_arr) {
$dbname = "integrity";
// Create connection
+ mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
$conn = new mysqli($servername, $username, $password);
+ $conn->set_charset('utf8mb4');
+ $conn->autocommit(FALSE);
// Check connection
if ($conn->connect_errno) {
@@ -174,6 +176,8 @@ function db_insert($data_arr) {
insert_file($file, $key, $conn, "sha1");
}
}
+ if (!$conn->commit())
+ echo "Inserting failed<br/>";
}
// db_insert(parse_dat("ngi.dat"));
Commit: 78a8482c2b3e8a2a8530c0485836e4fe4e37cba2
https://github.com/scummvm/scummvm-sites/commit/78a8482c2b3e8a2a8530c0485836e4fe4e37cba2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-09T19:48:13+05:30
Commit Message:
INTEGIRTY: Add compute_hash.py
- Base of the CLI application that calculates
checksums of files in a given dir
- Calculated checksums are stored in a dict
TODO:
- Accept CLI arguments for dir path and engine
name
- Write checksum values into DAT file
Changed paths:
A compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
new file mode 100644
index 0000000..5caa302
--- /dev/null
+++ b/compute_hash.py
@@ -0,0 +1,55 @@
+import hashlib
+import zlib
+import os
+
+
+def filesize(filepath):
+ """ Returns size of file """
+ return os.stat(filepath).st_size
+
+
+def checksum(filepath, alg, size):
+ """ Returns checksum value of file using a specific algoritm """
+ with open(filepath, "rb") as file:
+ if alg == "md5":
+ md5_hash = hashlib.md5()
+ # Read file in 8MB chunks
+ for byte_block in iter(lambda: file.read(8388608), b""):
+ md5_hash.update(byte_block)
+ return md5_hash.hexdigest()
+
+ elif alg == "crc32":
+ crc_hash = 0
+ for byte_block in iter(lambda: file.read(8388608), b""):
+ crc_hash += zlib.crc32(byte_block)
+ crc_hash &= 0xffffffff # Keeping it at 32 bits
+ return crc_hash
+
+ elif alg == "sha1":
+ sha1_hash = hashlib.sha1()
+ for byte_block in iter(lambda: file.read(8388608), b""):
+ sha1_hash.update(byte_block)
+ return sha1_hash.hexdigest()
+
+ elif alg == "sha256":
+ sha256_hash = hashlib.sha256()
+ for byte_block in iter(lambda: file.read(8388608), b""):
+ sha256_hash.update(byte_block)
+ return sha256_hash.hexdigest()
+
+
+def compute_hash_of_dir(directory, alg="md5", size=0):
+ """ Return dictionary containing checksums of all files in directory """
+ res = dict()
+ # Getting contents of directory and filtering only the files
+ files = [f for f in os.listdir(directory) if os.path.isfile(
+ os.path.join(directory, f))]
+
+ for file in files:
+ res[file] = checksum(os.path.join(directory, file), alg=alg, size=size)
+
+ return res
+
+
+path = os.path.expanduser("~/test")
+print(compute_hash_of_dir(path, alg="sha256"))
Commit: b493e8bb31d4c9b5b99e5c5b1e0aa7f59a026e3a
https://github.com/scummvm/scummvm-sites/commit/b493e8bb31d4c9b5b99e5c5b1e0aa7f59a026e3a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-12T11:38:17+05:30
Commit Message:
INTEGRITY: Calculating various sizes of checksum
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 5caa302..b2b7727 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -1,5 +1,4 @@
import hashlib
-import zlib
import os
@@ -11,31 +10,50 @@ def filesize(filepath):
def checksum(filepath, alg, size):
""" Returns checksum value of file using a specific algoritm """
with open(filepath, "rb") as file:
- if alg == "md5":
- md5_hash = hashlib.md5()
- # Read file in 8MB chunks
- for byte_block in iter(lambda: file.read(8388608), b""):
- md5_hash.update(byte_block)
- return md5_hash.hexdigest()
-
- elif alg == "crc32":
- crc_hash = 0
- for byte_block in iter(lambda: file.read(8388608), b""):
- crc_hash += zlib.crc32(byte_block)
- crc_hash &= 0xffffffff # Keeping it at 32 bits
- return crc_hash
+ # Will contain 5 elements:
+ # - Full size checksum
+ # - Checksum of first 5000B
+ # - Checksum of first 1MB
+ # - Checksum of last 5000B (tail)
+ # - Checksum of first *size* bytes
+ hashes = []
+ if alg == "md5":
+ hashes = [hashlib.md5() for _ in range(5)]
elif alg == "sha1":
- sha1_hash = hashlib.sha1()
- for byte_block in iter(lambda: file.read(8388608), b""):
- sha1_hash.update(byte_block)
- return sha1_hash.hexdigest()
-
+ hashes = [hashlib.sha1() for _ in range(5)]
elif alg == "sha256":
- sha256_hash = hashlib.sha256()
- for byte_block in iter(lambda: file.read(8388608), b""):
- sha256_hash.update(byte_block)
- return sha256_hash.hexdigest()
+ hashes = [hashlib.sha256() for _ in range(5)]
+
+ # Read file in 8MB chunks for full checksum
+ for byte_block in iter(lambda: file.read(8 * 1024 * 1024), b""):
+ hashes[0].update(byte_block)
+
+ # First 5000B
+ file.seek(0)
+ hashes[1].update(file.read(5000))
+
+ # First 1MB
+ file.seek(0)
+ hashes[2].update(file.read(1024 * 1024))
+
+ # Last 5000B
+ if filesize(filepath) >= 5000:
+ file.seek(-5000, os.SEEK_END)
+ hashes[3].update(file.read())
+ else:
+ hashes[3] = hashes[0]
+
+ # Custom size; may be None
+ # Size is in bytes
+ # Reads entire required size at once, inefficient for large sizes
+ if size and size <= filesize(filepath):
+ file.seek(0)
+ hashes[4].update(file.read(size))
+ else:
+ hashes[4] = None
+
+ return [hash.hexdigest() for hash in hashes if hash]
def compute_hash_of_dir(directory, alg="md5", size=0):
@@ -51,5 +69,5 @@ def compute_hash_of_dir(directory, alg="md5", size=0):
return res
-path = os.path.expanduser("~/test")
-print(compute_hash_of_dir(path, alg="sha256"))
+path = os.path.expanduser("~/Downloads/drascula-1.0")
+print(compute_hash_of_dir(path, size=4000))
Commit: 1a6843f12be44bd874011e59e9e8fc9fec0720fc
https://github.com/scummvm/scummvm-sites/commit/1a6843f12be44bd874011e59e9e8fc9fec0720fc
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-13T19:37:04+05:30
Commit Message:
INTEGRITY: Add command line args and options
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index b2b7727..08f9a9b 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -1,5 +1,6 @@
import hashlib
import os
+import argparse
def filesize(filepath):
@@ -53,7 +54,7 @@ def checksum(filepath, alg, size):
else:
hashes[4] = None
- return [hash.hexdigest() for hash in hashes if hash]
+ return [h.hexdigest() for h in hashes if h]
def compute_hash_of_dir(directory, alg="md5", size=0):
@@ -69,5 +70,18 @@ def compute_hash_of_dir(directory, alg="md5", size=0):
return res
-path = os.path.expanduser("~/Downloads/drascula-1.0")
+parser = argparse.ArgumentParser()
+parser.add_argument("directory",
+ help="Path of directory with game files")
+parser.add_argument("engine",
+ help="Name of the engine the game is built on")
+parser.add_argument("--size",
+ help="Use first n bytes of file to calculate checksum")
+args = parser.parse_args()
+path = os.path.abspath(args.directory)
+engine_name = args.engine
+checksum_size = args.size
+
+
+# path = os.path.expanduser("~/Downloads/drascula-1.0")
print(compute_hash_of_dir(path, size=4000))
Commit: b6b55571b72951876bdb48a9864ed51f9fefeb63
https://github.com/scummvm/scummvm-sites/commit/b6b55571b72951876bdb48a9864ed51f9fefeb63
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-13T19:55:23+05:30
Commit Message:
INTEGRITY: Add function to create DAT file
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 08f9a9b..766e1ad 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -70,6 +70,24 @@ def compute_hash_of_dir(directory, alg="md5", size=0):
return res
+def create_dat_file(hash_of_dir, engine, path):
+ with open(f"{engine}.dat", "w") as file:
+ # Header
+ file.writelines([
+ "clrmamepro (\n",
+ f"\tname \"ScummVM {engine}\"\n",
+ ")\n\n"
+ ])
+
+ # Game files
+ file.write("game (\n")
+ for filename, hashes in hash_of_dir.items():
+ # Only works for MD5s, ignores optional extra size
+ data = f"name {filename} size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000t {hashes[3]}"
+ file.write(f"\trom ( {data} )\n")
+ file.write(")\n\n")
+
+
parser = argparse.ArgumentParser()
parser.add_argument("directory",
help="Path of directory with game files")
@@ -83,5 +101,5 @@ engine_name = args.engine
checksum_size = args.size
-# path = os.path.expanduser("~/Downloads/drascula-1.0")
-print(compute_hash_of_dir(path, size=4000))
+path = os.path.expanduser("~/Downloads/drascula-1.0")
+create_dat_file(compute_hash_of_dir(path, size=4000), engine_name, path)
Commit: e77641517ecf7ebd31a1d7811a938186b24db591
https://github.com/scummvm/scummvm-sites/commit/e77641517ecf7ebd31a1d7811a938186b24db591
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-14T12:21:47+05:30
Commit Message:
INTEGRITY: Add minor changes to DAT format
- Created file uses headers labelled scummvm
instead of clrmamepro
- Modified the parser to reflect this
Changed paths:
compute_hash.py
dat_parser.php
diff --git a/compute_hash.py b/compute_hash.py
index 766e1ad..41662f0 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -74,7 +74,7 @@ def create_dat_file(hash_of_dir, engine, path):
with open(f"{engine}.dat", "w") as file:
# Header
file.writelines([
- "clrmamepro (\n",
+ "scummvm (\n",
f"\tname \"ScummVM {engine}\"\n",
")\n\n"
])
diff --git a/dat_parser.php b/dat_parser.php
index 66777f8..89e4adf 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -73,7 +73,8 @@ function parse_dat($dat_filepath) {
$exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
if (preg_match_all($exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
foreach ($matches[0] as $data_segment) {
- if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false) {
+ if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false ||
+ strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "scummvm") !== false) {
map_key_values($data_segment[0], $header);
}
elseif (strpos(substr($content, $data_segment[1] - 5, $data_segment[1]), "game") !== false) {
Commit: 8ea71e27a426e3a79001795b99e8c5e7ed6d0545
https://github.com/scummvm/scummvm-sites/commit/8ea71e27a426e3a79001795b99e8c5e7ed6d0545
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-21T18:27:57+05:30
Commit Message:
INTEGRITY: Updates to parser
Parser handles filenames with spaces and brackets,
provided the name is enclosed in double quotes
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 89e4adf..5b2dd5a 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -8,11 +8,14 @@ ini_set('memory_limit', '512M');
*/
function map_checksum_data($content_string) {
$arr = array();
- $temp = preg_split("/\s/", $content_string);
+ $temp = preg_split('/("[^"]*")|\h+/', $content_string, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
for ($i = 1; $i < count($temp); $i += 2) {
if ($temp[$i] == ')')
continue;
+ if ($temp[$i + 1][0] == '"') {
+ $temp[$i + 1] = substr($temp[$i + 1], 1, -1);
+ }
$arr[$temp[$i]] = $temp[$i + 1];
}
@@ -52,6 +55,41 @@ function map_key_values($content_string, &$arr) {
}
}
+/**
+ * Parse DAT file and separate the contents each segment into an array
+ * Segments are of the form `scummvm ( )`, `game ( )` etc.
+ */
+function match_outermost_brackets($input) {
+ $matches = array();
+ $depth = 0;
+ $inside_quotes = false;
+ $cur_index = 0;
+
+ for ($i = 0; $i < strlen($input); $i++) {
+ $char = $input[$i];
+
+ if ($char == '(' && !$inside_quotes) {
+ if ($depth === 0) {
+ $cur_index = $i;
+ }
+ $depth++;
+ }
+ elseif ($char == ')' && !$inside_quotes) {
+ $depth--;
+ if ($depth === 0) {
+ $match = substr($input, $cur_index, $i - $cur_index + 1);
+ array_push($matches, array($match, $cur_index));
+ }
+ }
+ elseif ($char == '"') {
+ $inside_quotes = !$inside_quotes;
+ }
+ }
+
+ return $matches;
+}
+
+
/**
* Take DAT filepath as input and return parsed data in the form of
* associated arrays
@@ -69,12 +107,11 @@ function parse_dat($dat_filepath) {
$game_data = array();
$resources = array();
- $matches = array();
- $exp = '/\((?:[^)(]+|(?R))*+\)/u'; // Get content inside outermost brackets
- if (preg_match_all($exp, $content, $matches, PREG_OFFSET_CAPTURE)) {
- foreach ($matches[0] as $data_segment) {
- if (strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "clrmamepro") !== false ||
- strpos(substr($content, $data_segment[1] - 11, $data_segment[1]), "scummvm") !== false) {
+ $matches = match_outermost_brackets($content);
+ if ($matches) {
+ foreach ($matches as $data_segment) {
+ if (strpos(substr($content, $data_segment[1] - 11, 11), "clrmamepro") !== false ||
+ strpos(substr($content, $data_segment[1] - 8, 8), "scummvm") !== false) {
map_key_values($data_segment[0], $header);
}
elseif (strpos(substr($content, $data_segment[1] - 5, $data_segment[1]), "game") !== false) {
Commit: 6a85968d3efe33bea95a61257bf415e64395af0a
https://github.com/scummvm/scummvm-sites/commit/6a85968d3efe33bea95a61257bf415e64395af0a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-23T12:39:44+05:30
Commit Message:
Update database schema
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 5aabd70..0182916 100644
--- a/schema.php
+++ b/schema.php
@@ -31,7 +31,7 @@ $conn->query("USE " . $dbname);
// Create engine table
$table = "CREATE TABLE IF NOT EXISTS engine (
id INT AUTO_INCREMENT PRIMARY KEY,
- name VARCHAR(200) NOT NULL,
+ name VARCHAR(200),
engineid VARCHAR(100) NOT NULL
)";
@@ -45,9 +45,11 @@ else {
// Create game table
$table = "CREATE TABLE IF NOT EXISTS game (
id INT AUTO_INCREMENT PRIMARY KEY,
- name VARCHAR(200) NOT NULL,
+ name VARCHAR(200),
engine INT NOT NULL,
gameid VARCHAR(100) NOT NULL,
+ platform VARCHAR(30),
+ language VARCHAR(10),
FOREIGN KEY (engine) REFERENCES engine(id)
)";
@@ -58,37 +60,39 @@ else {
echo "Error creating 'game' table: " . $conn->error;
}
-// Create file table
-$table = "CREATE TABLE IF NOT EXISTS file (
+// Create fileset table
+$table = "CREATE TABLE IF NOT EXISTS fileset (
id INT AUTO_INCREMENT PRIMARY KEY,
- name VARCHAR(200) NOT NULL,
- size INT NOT NULL,
- checksum VARCHAR(64) NOT NULL
+ game INT,
+ status VARCHAR(20),
+ src VARCHAR(20),
+ `key` VARCHAR(64),
+ FOREIGN KEY (game) REFERENCES game(id)
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'file' created successfully<br/>";
+ echo "Table 'fileset' created successfully<br/>";
}
else {
- echo "Error creating 'file' table: " . $conn->error;
+ echo "Error creating 'fileset' table: " . $conn->error;
}
-// Create fileset table
-$table = "CREATE TABLE IF NOT EXISTS fileset (
+// Create file table
+$table = "CREATE TABLE IF NOT EXISTS file (
id INT AUTO_INCREMENT PRIMARY KEY,
- game INT,
- file INT NOT NULL,
- status INT,
- `key` VARCHAR(64),
- FOREIGN KEY (game) REFERENCES game(id),
- FOREIGN KEY (file) REFERENCES file(id)
+ name VARCHAR(200) NOT NULL,
+ size INT NOT NULL,
+ checksum VARCHAR(64) NOT NULL,
+ fileset INT NOT NULL,
+ detection BOOLEAN NOT NULL,
+ FOREIGN KEY (fileset) REFERENCES fileset(id)
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'fileset' created successfully<br/>";
+ echo "Table 'file' created successfully<br/>";
}
else {
- echo "Error creating 'fileset' table: " . $conn->error;
+ echo "Error creating 'file' table: " . $conn->error;
}
// Create filechecksum table
@@ -108,28 +112,12 @@ else {
echo "Error creating 'filechecksum' table: " . $conn->error;
}
-// Create fileset_detection table
-$table = "CREATE TABLE IF NOT EXISTS fileset_detection (
- id INT AUTO_INCREMENT PRIMARY KEY,
- fileset INT NOT NULL,
- checksum INT NOT NULL,
- FOREIGN KEY (fileset) REFERENCES fileset(id),
- FOREIGN KEY (checksum) REFERENCES filechecksum(id)
-)";
-
-if ($conn->query($table) === TRUE) {
- echo "Table 'fileset_detection' created successfully<br/>";
-}
-else {
- echo "Error creating 'fileset_detection' table: " . $conn->error;
-}
-
// Create queue table
$table = "CREATE TABLE IF NOT EXISTS queue (
id INT AUTO_INCREMENT PRIMARY KEY,
date DATETIME NOT NULL,
notes varchar(300),
- fileset INT NOT NULL,
+ fileset INT,
ticketid INT NOT NULL,
userid INT NOT NULL,
commit VARCHAR(64) NOT NULL,
Commit: f5488bc65d4fdbd5f9c7f9a8682bccc597b453b2
https://github.com/scummvm/scummvm-sites/commit/f5488bc65d4fdbd5f9c7f9a8682bccc597b453b2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-23T18:12:55+05:30
Commit Message:
Add field `extra` to `game` table
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 0182916..4571fc5 100644
--- a/schema.php
+++ b/schema.php
@@ -48,6 +48,7 @@ $table = "CREATE TABLE IF NOT EXISTS game (
name VARCHAR(200),
engine INT NOT NULL,
gameid VARCHAR(100) NOT NULL,
+ extra VARCHAR(200),
platform VARCHAR(30),
language VARCHAR(10),
FOREIGN KEY (engine) REFERENCES engine(id)
Commit: 5129972cfa9ffa4916fd64e6ae9d808015907c2b
https://github.com/scummvm/scummvm-sites/commit/5129972cfa9ffa4916fd64e6ae9d808015907c2b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-23T20:40:03+05:30
Commit Message:
INTEGRITY: Update type of file.size to BIGINT
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 4571fc5..cdb93fd 100644
--- a/schema.php
+++ b/schema.php
@@ -82,7 +82,7 @@ else {
$table = "CREATE TABLE IF NOT EXISTS file (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(200) NOT NULL,
- size INT NOT NULL,
+ size BIGINT NOT NULL,
checksum VARCHAR(64) NOT NULL,
fileset INT NOT NULL,
detection BOOLEAN NOT NULL,
Commit: f59bcdd6dc99fa653b69360b15c81feb287610a6
https://github.com/scummvm/scummvm-sites/commit/f59bcdd6dc99fa653b69360b15c81feb287610a6
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-23T21:46:50+05:30
Commit Message:
INTEGRITY: Update parser for detection entries
- Update insertion queries to match updated
schema
- Handle different checksizes and checktypes
- Add $src to check the source of the DAT file
being parsed and behave accordingly
- Remove escape characters in certain values with
stripslashes()
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 5b2dd5a..5bef6b3 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -16,7 +16,7 @@ function map_checksum_data($content_string) {
if ($temp[$i + 1][0] == '"') {
$temp[$i + 1] = substr($temp[$i + 1], 1, -1);
}
- $arr[$temp[$i]] = $temp[$i + 1];
+ $arr[$temp[$i]] = stripslashes($temp[$i + 1]);
}
return $arr;
@@ -50,7 +50,7 @@ function map_key_values($content_string, &$arr) {
}
}
else {
- $arr[$pair[0]] = $pair[1];
+ $arr[$pair[0]] = stripslashes($pair[1]);
}
}
}
@@ -81,7 +81,7 @@ function match_outermost_brackets($input) {
array_push($matches, array($match, $cur_index));
}
}
- elseif ($char == '"') {
+ elseif ($char == '"' && $input[$i - 1] != '\\') {
$inside_quotes = !$inside_quotes;
}
}
@@ -89,7 +89,6 @@ function match_outermost_brackets($input) {
return $matches;
}
-
/**
* Take DAT filepath as input and return parsed data in the form of
* associated arrays
@@ -139,33 +138,96 @@ function parse_dat($dat_filepath) {
}
/**
- * Routine for inserting a file into the database, intserting into all
+ * Routine for inserting a game into the database, inserting into engine and
+ * game tables
+ */
+function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn) {
+ // Set @engine_last if engine already present in table
+ $exists = false;
+ if ($res = $conn->query(sprintf("SELECT id FROM engine WHERE engineid = '%s'", $engineid))) {
+ if ($res->num_rows > 0) {
+ $exists = true;
+ $conn->query(sprintf("SET @engine_last = '%d'", $res->fetch_array()[0]));
+ }
+ }
+
+ // Insert into table if not present
+ if (!$exists) {
+ $query = sprintf("INSERT INTO engine (name, engineid)
+ VALUES (NULL, '%s')", $engineid);
+ $conn->query($query);
+ $conn->query("SET @engine_last = LAST_INSERT_ID()");
+ }
+
+ // Insert into game
+ $query = sprintf("INSERT INTO game (name, engine, gameid, extra, platform, language)
+ VALUES ('%s', @engine_last, '%s', '%s', '%s', '%s')", mysqli_real_escape_string($conn, $title),
+ $gameid, mysqli_real_escape_string($conn, $extra), $platform, $lang);
+ $conn->query($query);
+ $conn->query("SET @game_last = LAST_INSERT_ID()");
+}
+
+/**
+ * Inserting new fileset
+ * Called for both detection entries and other forms of DATs
+ */
+function insert_fileset($src, $key, $detection, $conn) {
+ // $status can be: {detection, unconfirmed, confirmed (unused here)}
+ $status = "unconfirmed";
+ $game = "NULL";
+
+ if ($detection) {
+ $status = "detection";
+ $game = "@game_last";
+ }
+
+ // $game should not be parsed as a mysql string, hence no quotes
+ $query = sprintf("INSERT INTO fileset (game, status, src, `key`)
+ VALUES (%s, '%s', '%s', '%s')", $game, $status, $src, $key);
+ $conn->query($query);
+ $conn->query("SET @fileset_last = LAST_INSERT_ID()");
+}
+
+/**
+ * Routine for inserting a file into the database, inserting into all
* required tables
* $file is an associated array (the contents of 'rom')
* If checksum of the given checktype doesn't exists, silently fails
*/
-function insert_file($file, $key, $conn, $checktype = "md5", $checksize = 0) {
- if (!array_key_exists($checktype, $file))
- return;
+function insert_file($file, $detection, $conn, $checktype = "md5") {
+ // Find first checksum value
- $query = sprintf("INSERT INTO file (name, size, checksum)
- VALUES ('%s', '%s', '%s')", mysqli_real_escape_string($conn, $file["name"]),
- $file["size"], $file[$checktype]);
+ $checksum = "";
+ foreach ($file as $key => $value) {
+ if ($key != "name" && $key != "size")
+ $checksum = $value;
+ }
+
+ $query = sprintf("INSERT INTO file (name, size, checksum, fileset, detection)
+ VALUES ('%s', '%s', '%s', @fileset_last, %d)", mysqli_real_escape_string($conn, $file["name"]),
+ $file["size"], $checksum, $detection);
$conn->query($query);
$conn->query("SET @file_last = LAST_INSERT_ID()");
+}
- $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
- VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $file[$checktype]);
- $conn->query($query);
- $conn->query("SET @filechecksum_last = LAST_INSERT_ID()");
+function insert_filechecksum($file, $checktype, $conn) {
+ if (!array_key_exists($checktype, $file))
+ return;
- $query = sprintf("INSERT INTO fileset (game, file, status, `key`)
- VALUES (NULL, @file_last, 0, '%s')", $key);
- $conn->query($query);
- $conn->query("SET @fileset_last = LAST_INSERT_ID()");
+ $checksize = 0;
+ $checksum = $file[$checktype];
+ if (strpos($checktype, '-') !== false) {
+ $checksize = explode('-', $checktype)[1];
+ $checktype = explode('-', $checktype)[0];
+ }
- $query = "INSERT INTO fileset_detection (fileset, checksum)
- VALUES (@fileset_last, @filechecksum_last)";
+ if (strpos($checksum, ':') !== false) {
+ $checktype .= "-" . explode(':', $checksum)[0];
+ $checksum = explode(':', $checksum)[1];
+ }
+
+ $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
+ VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $checksum);
$conn->query($query);
}
@@ -196,22 +258,48 @@ function db_insert($data_arr) {
$conn->query("USE " . $dbname);
- // Remove "ScummVM" from engine name
- $engine_name = preg_split("/\s/", $header["name"]);
- array_shift($engine_name);
- $engine_name = implode(" ", $engine_name);
-
- $query = sprintf("INSERT INTO engine (name, engineid)
- VALUES ('%s', '1')", $engine_name);
- $conn->query($query);
- $conn->query("SET @engine_last = LAST_INSERT_ID()");
+ /**
+ * Author can be:
+ * scummvm -> Detection Entries
+ * scanner -> CLI tool
+ * _anything else_ -> DAT file
+ */
+ $author = $header["author"];
+
+ /**
+ * src can be:
+ * detection -> Detection entries (source of truth)
+ * user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
+ * scan -> Submitted by scanner, unmatched
+ * dat -> Submitted by DAT, unmatched
+ * partialmatch -> Submitted by DAT, matched
+ * fullmatch -> Submitted by scanner, matched
+ */
+ $src = "";
+ if ($author == "scanner")
+ $src = "scan";
+ elseif ($author == "scummvm")
+ $src = "detection";
+ else
+ $src = "dat";
foreach ($game_data as $fileset) {
+ if ($src == "detection") {
+ $engineid = $fileset["sourcefile"];
+ $gameid = $fileset["name"];
+ $title = $fileset["title"];
+ $extra = $fileset["extra"];
+ $platform = $fileset["platform"];
+ $lang = $fileset["language"];
+
+ insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn);
+ }
+
+ $key = NULL;
+ insert_fileset($src, $key, ($src == "detection"), $conn);
foreach ($fileset["rom"] as $file) {
- $key = NULL;
- insert_file($file, $key, $conn);
- insert_file($file, $key, $conn, "crc");
- insert_file($file, $key, $conn, "sha1");
+ insert_file($file, ($src == "detection"), $conn, "md5-5000");
+ insert_filechecksum($file, "md5-5000", $conn);
}
}
if (!$conn->commit())
Commit: 37d26163131cba1bfa545db5da09758da2c7fa88
https://github.com/scummvm/scummvm-sites/commit/37d26163131cba1bfa545db5da09758da2c7fa88
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-26T13:18:25+05:30
Commit Message:
INTEGRITY: Add t: to tail checksums, update header
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 41662f0..cbd8c52 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -54,10 +54,12 @@ def checksum(filepath, alg, size):
else:
hashes[4] = None
- return [h.hexdigest() for h in hashes if h]
+ hashes = [h.hexdigest() for h in hashes if h]
+ hashes[3] = 't:' + hashes[3] # Add tail prefix
+ return hashes
-def compute_hash_of_dir(directory, alg="md5", size=0):
+def compute_hash_of_dir(directory, depth, alg="md5", size=0):
""" Return dictionary containing checksums of all files in directory """
res = dict()
# Getting contents of directory and filtering only the files
@@ -70,12 +72,12 @@ def compute_hash_of_dir(directory, alg="md5", size=0):
return res
-def create_dat_file(hash_of_dir, engine, path):
- with open(f"{engine}.dat", "w") as file:
+def create_dat_file(hash_of_dir, path):
+ with open(f"{os.path.basename(path)}.dat", "w") as file:
# Header
file.writelines([
"scummvm (\n",
- f"\tname \"ScummVM {engine}\"\n",
+ f"\tauthor cli\n",
")\n\n"
])
@@ -83,23 +85,23 @@ def create_dat_file(hash_of_dir, engine, path):
file.write("game (\n")
for filename, hashes in hash_of_dir.items():
# Only works for MD5s, ignores optional extra size
- data = f"name {filename} size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000t {hashes[3]}"
+ data = f"name \"{filename}\" size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
file.write(f"\trom ( {data} )\n")
file.write(")\n\n")
parser = argparse.ArgumentParser()
-parser.add_argument("directory",
+parser.add_argument("--directory",
help="Path of directory with game files")
-parser.add_argument("engine",
- help="Name of the engine the game is built on")
+parser.add_argument("--depth",
+ help="Depth from root to game directories")
parser.add_argument("--size",
help="Use first n bytes of file to calculate checksum")
args = parser.parse_args()
-path = os.path.abspath(args.directory)
-engine_name = args.engine
+path = os.path.abspath(args.directory) if args.directory else os.getcwd()
+depth = args.depth
checksum_size = args.size
path = os.path.expanduser("~/Downloads/drascula-1.0")
-create_dat_file(compute_hash_of_dir(path, size=4000), engine_name, path)
+create_dat_file(compute_hash_of_dir(path, depth, size=4000), path)
Commit: 7465b20844f1a498c01cf8c4f777c31adae3ac16
https://github.com/scummvm/scummvm-sites/commit/7465b20844f1a498c01cf8c4f777c31adae3ac16
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-26T13:20:34+05:30
Commit Message:
INTEGRITY: Create find_matching_game()
Function returns the game id(s) of games that have
filesets that are a subset of parsed game files.
Refactored some of the code into separate functions
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 5bef6b3..fb93e83 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -137,6 +137,91 @@ function parse_dat($dat_filepath) {
return array($header, $game_data, $resources);
}
+/**
+ * Retrieves the checksum and checktype of a given type + checksum
+ * eg: md5-5000 t:12345... -> 5000, md5-t, 12345...
+ */
+function get_checksum_props($checktype, $checksum) {
+ $checksize = 0;
+ if (strpos($checktype, '-') !== false) {
+ $checksize = explode('-', $checktype)[1];
+ $checktype = explode('-', $checktype)[0];
+ }
+
+ if (strpos($checksum, ':') !== false) {
+ $prefix = explode(':', $checksum)[0];
+
+ if (strpos($prefix, 't') !== false)
+ $checktype .= "-" . 't';
+ $checksum = explode(':', $checksum)[1];
+ }
+
+ return array($checksize, $checktype, $checksum);
+}
+
+/**
+ * Detects games based on the file descriptions in $dat_arr
+ * Compares the files with those in the detection entries table
+ * $game_files consists of both the game ( ) and resources ( ) parts
+ */
+function find_matching_game($game_files) {
+ $matching_games = array(); // All matching games
+ $matching_filesets = array(); // All filesets containing one file from $game_files
+ $matches_count = 0; // Number of files with a matching detection entry
+
+ $conn = db_connect();
+
+ foreach ($game_files["rom"] as $file) {
+ foreach ($file as $key => $value) {
+ if (strpos($key, "md5") === false)
+ continue;
+
+ list($checksize, $checktype, $checksum) = get_checksum_props($key, $value);
+
+ $records = $conn->query(sprintf("SELECT file.fileset
+ FROM filechecksum
+ JOIN file ON filechecksum.file = file.id
+ WHERE filechecksum.checksum = '%s' AND file.detection = TRUE", $checksum));
+ $records = $records->fetch_all();
+
+ // If file is not part of detection entries, skip it
+ if (count($records) == 0)
+ continue;
+
+ $matches_count++;
+ foreach ($records as $record) {
+ array_push($matching_filesets, $record[0]);
+ }
+ }
+ }
+
+ // Check if there is a fileset_id that is present in all results
+ foreach (array_count_values($matching_filesets) as $key => $value) {
+ $count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
+ JOIN fileset ON file.fileset = fileset.id
+ WHERE fileset.id = '%s'", $key));
+ $count_files_in_fileset = $count_files_in_fileset->fetch_array()[0];
+
+ // We use < instead of != since one file may have more than one entry in the fileset
+ // We see this in Drascula English version, where one entry is duplicated
+ if ($value < $matches_count || $value < $count_files_in_fileset)
+ continue;
+
+ $records = $conn->query(sprintf("SELECT game.id
+ FROM game
+ JOIN fileset ON fileset.game = game.id
+ WHERE fileset.id = '%s'", $key));
+
+ array_push($matching_games, $records->fetch_all());
+ }
+
+ // If we got a unique match, return the id of the game
+ if (count($matching_games) == 1)
+ return $matching_games[0][0][0];
+
+ return $matching_games;
+}
+
/**
* Routine for inserting a game into the database, inserting into engine and
* game tables
@@ -214,17 +299,8 @@ function insert_filechecksum($file, $checktype, $conn) {
if (!array_key_exists($checktype, $file))
return;
- $checksize = 0;
$checksum = $file[$checktype];
- if (strpos($checktype, '-') !== false) {
- $checksize = explode('-', $checktype)[1];
- $checktype = explode('-', $checktype)[0];
- }
-
- if (strpos($checksum, ':') !== false) {
- $checktype .= "-" . explode(':', $checksum)[0];
- $checksum = explode(':', $checksum)[1];
- }
+ list($checksize, $checktype, $checksum) = get_checksum_props($checktype, $checksum);
$query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $checksum);
@@ -232,14 +308,9 @@ function insert_filechecksum($file, $checktype, $conn) {
}
/**
- * Insert values from the associated array into the DB
- * They will be inserted under gameid NULL as the game itself is unconfirmed
+ * Create and return a mysqli connection
*/
-function db_insert($data_arr) {
- $header = $data_arr[0];
- $game_data = $data_arr[1];
- $resources = $data_arr[2];
-
+function db_connect() {
$servername = "localhost";
$username = "username";
$password = "password";
@@ -258,6 +329,20 @@ function db_insert($data_arr) {
$conn->query("USE " . $dbname);
+ return $conn;
+}
+
+/**
+ * Insert values from the associated array into the DB
+ * They will be inserted under gameid NULL as the game itself is unconfirmed
+ */
+function db_insert($data_arr) {
+ $header = $data_arr[0];
+ $game_data = $data_arr[1];
+ $resources = $data_arr[2];
+
+ $conn = db_connect();
+
/**
* Author can be:
* scummvm -> Detection Entries
@@ -276,8 +361,8 @@ function db_insert($data_arr) {
* fullmatch -> Submitted by scanner, matched
*/
$src = "";
- if ($author == "scanner")
- $src = "scan";
+ if ($author == "cli")
+ $src = "cli";
elseif ($author == "scummvm")
$src = "detection";
else
@@ -306,7 +391,9 @@ function db_insert($data_arr) {
echo "Inserting failed<br/>";
}
-// db_insert(parse_dat("ngi.dat"));
-
+echo "<pre>";
+// print_r(parse_dat("drascula-1.0.dat")[1]);
+print_r(find_matching_game(parse_dat("drascula-1.0.dat")[1][0]));
+echo "</pre>";
?>
Commit: cf2b8c8a1b64cc72e1fc20b211b808c6e43ba7fe
https://github.com/scummvm/scummvm-sites/commit/cf2b8c8a1b64cc72e1fc20b211b808c6e43ba7fe
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-26T17:47:28+05:30
Commit Message:
INTEGRITY: Update schema to contain indices
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index cdb93fd..c6ff9dc 100644
--- a/schema.php
+++ b/schema.php
@@ -132,6 +132,25 @@ else {
echo "Error creating 'queue' table: " . $conn->error;
}
+// Create indices for fast data retrieval
+// PK and FK are automatically indexed in InnoDB, so they are not included
+$index = "CREATE INDEX detection ON file (detection)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'file.detection'<br/>";
+}
+else {
+ echo "Error creating index for 'file.detection': " . $conn->error;
+}
+
+$index = "CREATE INDEX checksum ON filechecksum (checksum)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'filechecksum.checksum'<br/>";
+}
+else {
+ echo "Error creating index for 'filechecksum.checksum': " . $conn->error;
+}
$conn->close();
?>
Commit: 5a1303e486d8a6fdfdeaef0274aa7649547003e0
https://github.com/scummvm/scummvm-sites/commit/5a1303e486d8a6fdfdeaef0274aa7649547003e0
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-26T17:50:00+05:30
Commit Message:
INTEGRITY: Match game with filesets
- Match game with filesets when game data comes
from CLI or DATs
- Match resource files with game files using romof
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index fb93e83..1369361 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -119,7 +119,9 @@ function parse_dat($dat_filepath) {
array_push($game_data, $temp);
}
elseif (strpos(substr($content, $data_segment[1] - 9, $data_segment[1]), "resource") !== false) {
- map_key_values($data_segment[0], $resources);
+ $temp = array();
+ map_key_values($data_segment[0], $temp);
+ $resources[$temp["name"]] = $temp;
}
}
@@ -215,10 +217,6 @@ function find_matching_game($game_files) {
array_push($matching_games, $records->fetch_all());
}
- // If we got a unique match, return the id of the game
- if (count($matching_games) == 1)
- return $matching_games[0][0][0];
-
return $matching_games;
}
@@ -256,10 +254,9 @@ function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn
* Inserting new fileset
* Called for both detection entries and other forms of DATs
*/
-function insert_fileset($src, $key, $detection, $conn) {
+function insert_fileset($src, $key, $detection, $conn, $game = "NULL") {
// $status can be: {detection, unconfirmed, confirmed (unused here)}
$status = "unconfirmed";
- $game = "NULL";
if ($detection) {
$status = "detection";
@@ -279,9 +276,8 @@ function insert_fileset($src, $key, $detection, $conn) {
* $file is an associated array (the contents of 'rom')
* If checksum of the given checktype doesn't exists, silently fails
*/
-function insert_file($file, $detection, $conn, $checktype = "md5") {
+function insert_file($file, $detection, $conn) {
// Find first checksum value
-
$checksum = "";
foreach ($file as $key => $value) {
if ($key != "name" && $key != "size")
@@ -355,10 +351,10 @@ function db_insert($data_arr) {
* src can be:
* detection -> Detection entries (source of truth)
* user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
- * scan -> Submitted by scanner, unmatched
+ * cli -> Submitted by cli/scanner, unmatched
* dat -> Submitted by DAT, unmatched
* partialmatch -> Submitted by DAT, matched
- * fullmatch -> Submitted by scanner, matched
+ * fullmatch -> Submitted by cli/scanner, matched
*/
$src = "";
if ($author == "cli")
@@ -368,8 +364,10 @@ function db_insert($data_arr) {
else
$src = "dat";
+ $detection = ($src == "detection");
+
foreach ($game_data as $fileset) {
- if ($src == "detection") {
+ if ($detection) {
$engineid = $fileset["sourcefile"];
$gameid = $fileset["name"];
$title = $fileset["title"];
@@ -378,12 +376,26 @@ function db_insert($data_arr) {
$lang = $fileset["language"];
insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn);
+ insert_fileset($src, NULL, true, $conn);
+ }
+ elseif ($src == "dat")
+ if (isset($resources[$fileset["romof"]]))
+ $fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
+
+ if (!$detection) {
+ $matching_games = find_matching_game($fileset);
+
+ // If a unique match is found, assign the fileset a gameid
+ if (count($matching_games) == 1) {
+ $key = NULL;
+ insert_fileset($src, $key, $detection, $conn, $matching_games[0][0][0]);
+ }
+ else
+ insert_fileset($src, NULL, $detection, $conn);
}
- $key = NULL;
- insert_fileset($src, $key, ($src == "detection"), $conn);
foreach ($fileset["rom"] as $file) {
- insert_file($file, ($src == "detection"), $conn, "md5-5000");
+ insert_file($file, $detection, $conn);
insert_filechecksum($file, "md5-5000", $conn);
}
}
@@ -391,9 +403,8 @@ function db_insert($data_arr) {
echo "Inserting failed<br/>";
}
-echo "<pre>";
-// print_r(parse_dat("drascula-1.0.dat")[1]);
-print_r(find_matching_game(parse_dat("drascula-1.0.dat")[1][0]));
-echo "</pre>";
+// echo "<pre>";
+// db_insert(parse_dat("scummvm_detection_entries.dat"));
+// echo "</pre>";
?>
Commit: b51b98e3331fcffc8ff853c238280eab664a75bd
https://github.com/scummvm/scummvm-sites/commit/b51b98e3331fcffc8ff853c238280eab664a75bd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-27T11:05:06+05:30
Commit Message:
INTEGRITY: Separate game matching and db insertion
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 1369361..713e81a 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -173,28 +173,22 @@ function find_matching_game($game_files) {
$conn = db_connect();
- foreach ($game_files["rom"] as $file) {
- foreach ($file as $key => $value) {
- if (strpos($key, "md5") === false)
- continue;
-
- list($checksize, $checktype, $checksum) = get_checksum_props($key, $value);
-
- $records = $conn->query(sprintf("SELECT file.fileset
- FROM filechecksum
- JOIN file ON filechecksum.file = file.id
- WHERE filechecksum.checksum = '%s' AND file.detection = TRUE", $checksum));
- $records = $records->fetch_all();
-
- // If file is not part of detection entries, skip it
- if (count($records) == 0)
- continue;
-
- $matches_count++;
- foreach ($records as $record) {
- array_push($matching_filesets, $record[0]);
- }
- }
+ foreach ($game_files as $file) {
+ $checksum = $file[1];
+
+ $records = $conn->query(sprintf("SELECT file.fileset
+ FROM filechecksum
+ JOIN file ON filechecksum.file = file.id
+ WHERE filechecksum.checksum = '%s' AND file.detection = TRUE", $checksum));
+ $records = $records->fetch_all();
+
+ // If file is not part of detection entries, skip it
+ if (count($records) == 0)
+ continue;
+
+ $matches_count++;
+ foreach ($records as $record)
+ array_push($matching_filesets, $record[0]);
}
// Check if there is a fileset_id that is present in all results
@@ -217,7 +211,16 @@ function find_matching_game($game_files) {
array_push($matching_games, $records->fetch_all());
}
- return $matching_games;
+ // Flattening the array
+ $res = array();
+ foreach ($matching_games as $query) {
+ foreach ($query as $selection) {
+ $game_id = $selection[0];
+ array_push($res, $game_id);
+ }
+ }
+
+ return $res;
}
/**
@@ -254,9 +257,10 @@ function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn
* Inserting new fileset
* Called for both detection entries and other forms of DATs
*/
-function insert_fileset($src, $key, $detection, $conn, $game = "NULL") {
+function insert_fileset($src, $detection, $conn) {
// $status can be: {detection, unconfirmed, confirmed (unused here)}
$status = "unconfirmed";
+ $game = "NULL";
if ($detection) {
$status = "detection";
@@ -265,7 +269,7 @@ function insert_fileset($src, $key, $detection, $conn, $game = "NULL") {
// $game should not be parsed as a mysql string, hence no quotes
$query = sprintf("INSERT INTO fileset (game, status, src, `key`)
- VALUES (%s, '%s', '%s', '%s')", $game, $status, $src, $key);
+ VALUES (%s, '%s', '%s', NULL)", $game, $status, $src);
$conn->query($query);
$conn->query("SET @fileset_last = LAST_INSERT_ID()");
}
@@ -280,8 +284,10 @@ function insert_file($file, $detection, $conn) {
// Find first checksum value
$checksum = "";
foreach ($file as $key => $value) {
- if ($key != "name" && $key != "size")
- $checksum = $value;
+ if (strpos($key, "md5") !== false) {
+ list($tmp1, $tmp2, $checksum) = get_checksum_props($key, $value);
+ break;
+ }
}
$query = sprintf("INSERT INTO file (name, size, checksum, fileset, detection)
@@ -376,24 +382,12 @@ function db_insert($data_arr) {
$lang = $fileset["language"];
insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn);
- insert_fileset($src, NULL, true, $conn);
}
elseif ($src == "dat")
if (isset($resources[$fileset["romof"]]))
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
- if (!$detection) {
- $matching_games = find_matching_game($fileset);
-
- // If a unique match is found, assign the fileset a gameid
- if (count($matching_games) == 1) {
- $key = NULL;
- insert_fileset($src, $key, $detection, $conn, $matching_games[0][0][0]);
- }
- else
- insert_fileset($src, NULL, $detection, $conn);
- }
-
+ insert_fileset($src, $detection, $conn);
foreach ($fileset["rom"] as $file) {
insert_file($file, $detection, $conn);
insert_filechecksum($file, "md5-5000", $conn);
@@ -403,8 +397,47 @@ function db_insert($data_arr) {
echo "Inserting failed<br/>";
}
-// echo "<pre>";
+function populate_matching_games($conn) {
+ // Getting unmatched filesets
+ $unmatched_filesets = array();
+
+ $unmatched_files = $conn->query(sprintf("SELECT fileset.id, file.checksum from fileset
+ JOIN file ON file.fileset = fileset.id
+ WHERE fileset.game IS NULL"));
+ $unmatched_files = $unmatched_files->fetch_all();
+
+ // Splitting them into different filesets
+ for ($i = 0; $i < count($unmatched_files); $i++) {
+ $cur_fileset = $unmatched_files[$i][0];
+ $temp = array();
+ while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
+ array_push($temp, $unmatched_files[$i]);
+ $i++;
+ }
+ array_push($unmatched_filesets, $temp);
+ }
+
+ // return;
+ foreach ($unmatched_filesets as $fileset) {
+ $matching_games = find_matching_game($fileset);
+ if (count($matching_games) != 1) // If there is no match/non-unique match
+ continue;
+
+ // Updating the fileset.game value to be $matching_games[0]
+ $query = sprintf("UPDATE fileset
+ SET game = %d
+ WHERE id = %d", $matching_games[0], $fileset[0]);
+ $conn->query($query);
+
+ if (!$conn->commit())
+ echo "Updating matched games failed<br/>";
+ }
+}
+
+echo "<pre>";
// db_insert(parse_dat("scummvm_detection_entries.dat"));
-// echo "</pre>";
+// db_insert(parse_dat("drascula-2.0.dat"));
+populate_matching_games(db_connect());
+echo "</pre>";
?>
Commit: 3e58080581c595ba0031132d136e3f8c7197a5a3
https://github.com/scummvm/scummvm-sites/commit/3e58080581c595ba0031132d136e3f8c7197a5a3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-27T22:04:51+05:30
Commit Message:
INTEGRITY: Create basic website with pagination
Display 25 games on every page, along with 'prev'
and 'next' buttons to navigate between pages
Changed paths:
A index.php
diff --git a/index.php b/index.php
new file mode 100644
index 0000000..8dbc61d
--- /dev/null
+++ b/index.php
@@ -0,0 +1,51 @@
+<?php
+
+$servername = "localhost";
+$username = "username";
+$password = "password";
+$dbname = "integrity";
+
+// Create connection
+mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+$conn = new mysqli($servername, $username, $password);
+$conn->set_charset('utf8mb4');
+$conn->autocommit(FALSE);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+$conn->query("USE " . $dbname);
+
+if (!isset($_GET['page'])) {
+ $page = 1;
+}
+else {
+ $page = $_GET['page'];
+}
+
+$results_per_page = 25;
+$offset = ($page - 1) * $results_per_page;
+$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_column();
+$num_of_pages = ceil($num_of_results / $results_per_page);
+
+$query = sprintf("SELECT * FROM game LIMIT %d OFFSET %d", $results_per_page, $offset);
+$result = $conn->query($query);
+
+echo "<ol start=\"" . $offset + 1 . "\">";
+while ($row = $result->fetch_array()) {
+ echo "<li>";
+ echo sprintf("%s (%s, %s, %s)<br/>", $row["name"], $row["extra"], $row["platform"], $row["language"]);
+ echo "</li>";
+}
+echo "</ol>";
+if ($page != 1)
+ echo '<a href = "index.php?page=' . $page - 1 . '">' . "prev" . ' </a>';
+if ($page != $num_of_pages)
+ echo '<a href = "index.php?page=' . $page + 1 . '">' . "next" . ' </a>';
+
+if (!$conn->commit())
+ echo "Error fetching games";
+?>
+
Commit: 865a3925e7baaae44108cadc54db45d104f262e2
https://github.com/scummvm/scummvm-sites/commit/865a3925e7baaae44108cadc54db45d104f262e2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-28T22:18:29+05:30
Commit Message:
INTEGRITY: Add page for browsing logs
Changed paths:
A logs.php
diff --git a/logs.php b/logs.php
new file mode 100644
index 0000000..6cc00ec
--- /dev/null
+++ b/logs.php
@@ -0,0 +1,52 @@
+<?php
+$filename = "logs.php";
+
+$servername = "localhost";
+$username = "username";
+$password = "password";
+$dbname = "integrity";
+
+// Create connection
+mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+$conn = new mysqli($servername, $username, $password);
+$conn->set_charset('utf8mb4');
+$conn->autocommit(FALSE);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+$conn->query("USE " . $dbname);
+
+if (!isset($_GET['page'])) {
+ $page = 1;
+}
+else {
+ $page = $_GET['page'];
+}
+
+$results_per_page = 25;
+$offset = ($page - 1) * $results_per_page;
+$num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_column();
+$num_of_pages = ceil($num_of_results / $results_per_page);
+
+$query = sprintf("SELECT * FROM log LIMIT %d OFFSET %d", $results_per_page, $offset);
+$result = $conn->query($query);
+
+echo "<ol start=\"" . $offset + 1 . "\">";
+while ($row = $result->fetch_array()) {
+ echo "<li>";
+ echo sprintf("%s (%s, %s, %s)<br/>", $row["timestamp"], $row["category"], $row["user"], $row["text"]);
+ echo "</li>";
+}
+echo "</ol>";
+
+echo '<a href =' . $filename . '"?page=1">first</a>';
+if ($page > 1)
+ echo '<a href =' . $filename . '"?page=' . $page - 1 . '">prev</a>';
+if ($page < $num_of_pages)
+ echo '<a href =' . $filename . '"?page=' . $page + 1 . '">next</a>';
+echo '<a href =' . $filename . '"?page=' . $num_of_pages . '">last</a>';
+?>
+
Commit: c401b6c5b001447809ecc4b3085767f210826423
https://github.com/scummvm/scummvm-sites/commit/c401b6c5b001447809ecc4b3085767f210826423
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-28T22:18:55+05:30
Commit Message:
INTEGRITY: Add first,last page button to index.php
Changed paths:
index.php
diff --git a/index.php b/index.php
index 8dbc61d..376ec71 100644
--- a/index.php
+++ b/index.php
@@ -1,4 +1,5 @@
<?php
+$filename = "index.php";
$servername = "localhost";
$username = "username";
@@ -30,7 +31,8 @@ $offset = ($page - 1) * $results_per_page;
$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_column();
$num_of_pages = ceil($num_of_results / $results_per_page);
-$query = sprintf("SELECT * FROM game LIMIT %d OFFSET %d", $results_per_page, $offset);
+$query = sprintf("SELECT name, extra, platform, language FROM game LIMIT %d OFFSET %d",
+ $results_per_page, $offset);
$result = $conn->query($query);
echo "<ol start=\"" . $offset + 1 . "\">";
@@ -40,12 +42,12 @@ while ($row = $result->fetch_array()) {
echo "</li>";
}
echo "</ol>";
-if ($page != 1)
- echo '<a href = "index.php?page=' . $page - 1 . '">' . "prev" . ' </a>';
-if ($page != $num_of_pages)
- echo '<a href = "index.php?page=' . $page + 1 . '">' . "next" . ' </a>';
-if (!$conn->commit())
- echo "Error fetching games";
+echo '<a href =' . $filename . '"?page=1">first</a>';
+if ($page > 1)
+ echo '<a href =' . $filename . '"?page=' . $page - 1 . '">prev</a>';
+if ($page < $num_of_pages)
+ echo '<a href =' . $filename . '"?page=' . $page + 1 . '">next</a>';
+echo '<a href =' . $filename . '"?page=' . $num_of_pages . '">last</a>';
?>
Commit: af0104cf550c2fdadee734be10172e2e2559c3d9
https://github.com/scummvm/scummvm-sites/commit/af0104cf550c2fdadee734be10172e2e2559c3d9
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-28T22:20:38+05:30
Commit Message:
INTEGRITY: Create log table, index engine.engineid
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index c6ff9dc..4f59932 100644
--- a/schema.php
+++ b/schema.php
@@ -132,6 +132,22 @@ else {
echo "Error creating 'queue' table: " . $conn->error;
}
+// Create log table
+$table = "CREATE TABLE IF NOT EXISTS log (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ `timestamp` TIMESTAMP NOT NULL,
+ category VARCHAR(100) NOT NULL,
+ user VARCHAR(100) NOT NULL,
+ `text` varchar(300)
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'log' created successfully<br/>";
+}
+else {
+ echo "Error creating 'log' table: " . $conn->error;
+}
+
// Create indices for fast data retrieval
// PK and FK are automatically indexed in InnoDB, so they are not included
$index = "CREATE INDEX detection ON file (detection)";
@@ -152,6 +168,15 @@ else {
echo "Error creating index for 'filechecksum.checksum': " . $conn->error;
}
+$index = "CREATE INDEX engineid ON engine (engineid)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'engine.engineid'<br/>";
+}
+else {
+ echo "Error creating index for 'engine.engineid': " . $conn->error;
+}
+
$conn->close();
?>
Commit: 78b307cf9bf79a2582a6e12a370f0663ca11fdba
https://github.com/scummvm/scummvm-sites/commit/78b307cf9bf79a2582a6e12a370f0663ca11fdba
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-28T22:21:30+05:30
Commit Message:
INTEGRITY: Create log on matching game files
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 713e81a..08cf43b 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -195,32 +195,23 @@ function find_matching_game($game_files) {
foreach (array_count_values($matching_filesets) as $key => $value) {
$count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
JOIN fileset ON file.fileset = fileset.id
- WHERE fileset.id = '%s'", $key));
- $count_files_in_fileset = $count_files_in_fileset->fetch_array()[0];
+ WHERE fileset.id = '%s'", $key))->fetch_column();
// We use < instead of != since one file may have more than one entry in the fileset
// We see this in Drascula English version, where one entry is duplicated
if ($value < $matches_count || $value < $count_files_in_fileset)
continue;
- $records = $conn->query(sprintf("SELECT game.id
+ $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform, language, `key`, src
FROM game
JOIN fileset ON fileset.game = game.id
+ JOIN engine ON engine.id = game.engine
WHERE fileset.id = '%s'", $key));
- array_push($matching_games, $records->fetch_all());
+ array_push($matching_games, $records->fetch_array());
}
- // Flattening the array
- $res = array();
- foreach ($matching_games as $query) {
- foreach ($query as $selection) {
- $game_id = $selection[0];
- array_push($res, $game_id);
- }
- }
-
- return $res;
+ return $matching_games;
}
/**
@@ -334,6 +325,18 @@ function db_connect() {
return $conn;
}
+/**
+ * Create an entry to the log table on each call of db_insert() or
+ * populate_matching_games()
+ */
+function create_log($category, $user, $text) {
+ $conn = db_connect();
+ $conn->query(sprintf("INSERT INTO log (`timestamp`, category, user, `text`)
+ VALUES (FROM_UNIXTIME(%d), '%s', '%s', '%s')", time(), $category, $user, $text));
+ if (!$conn->commit())
+ echo "Creating log failed<br/>";
+}
+
/**
* Insert values from the associated array into the DB
* They will be inserted under gameid NULL as the game itself is unconfirmed
@@ -364,7 +367,7 @@ function db_insert($data_arr) {
*/
$src = "";
if ($author == "cli")
- $src = "cli";
+ $src = "scan";
elseif ($author == "scummvm")
$src = "detection";
else
@@ -397,11 +400,13 @@ function db_insert($data_arr) {
echo "Inserting failed<br/>";
}
-function populate_matching_games($conn) {
+function populate_matching_games() {
+ $conn = db_connect();
+
// Getting unmatched filesets
$unmatched_filesets = array();
- $unmatched_files = $conn->query(sprintf("SELECT fileset.id, file.checksum from fileset
+ $unmatched_files = $conn->query(sprintf("SELECT fileset.id, file.checksum, fileset.src from fileset
JOIN file ON file.fileset = fileset.id
WHERE fileset.game IS NULL"));
$unmatched_files = $unmatched_files->fetch_all();
@@ -409,6 +414,7 @@ function populate_matching_games($conn) {
// Splitting them into different filesets
for ($i = 0; $i < count($unmatched_files); $i++) {
$cur_fileset = $unmatched_files[$i][0];
+ $src = $unmatched_files[$i][2];
$temp = array();
while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
array_push($temp, $unmatched_files[$i]);
@@ -417,17 +423,35 @@ function populate_matching_games($conn) {
array_push($unmatched_filesets, $temp);
}
- // return;
foreach ($unmatched_filesets as $fileset) {
$matching_games = find_matching_game($fileset);
+
if (count($matching_games) != 1) // If there is no match/non-unique match
continue;
- // Updating the fileset.game value to be $matching_games[0]
+ $matched_game = $matching_games[0];
+
+ // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
+ $status = "unconfirmed";
+ if ($matched_game["src"] == "dat")
+ $status = "partialmatch";
+ elseif ($matched_game["src"] == "scan")
+ $status = "fullmatch";
+
+ // Convert NULL values to string with value NULL for printing
+ $matched_game = array_map(function ($val) {
+ return (is_null($val)) ? "NULL" : $val;
+ }, $matched_game);
+ $log_text = sprintf("Matched game %s: %s-%s-%s variant %s from %s",
+ $matched_game["engineid"], $matched_game["gameid"], $matched_game["platform"],
+ $matched_game["language"], $matched_game["key"], $matched_game["src"]);
+
+ // Updating the fileset.game value to be $matched_game["id"]
$query = sprintf("UPDATE fileset
- SET game = %d
- WHERE id = %d", $matching_games[0], $fileset[0]);
- $conn->query($query);
+ SET game = %d, status = '%s'
+ WHERE id = %d", $matched_game["id"], $status, $fileset[0]);
+ if ($conn->query($query))
+ create_log("Matched, state '" . $status . "'", "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
if (!$conn->commit())
echo "Updating matched games failed<br/>";
@@ -437,7 +461,7 @@ function populate_matching_games($conn) {
echo "<pre>";
// db_insert(parse_dat("scummvm_detection_entries.dat"));
// db_insert(parse_dat("drascula-2.0.dat"));
-populate_matching_games(db_connect());
+populate_matching_games();
echo "</pre>";
?>
Commit: 0aacb1ce2624932f0773527c2996a7e0ffb909a5
https://github.com/scummvm/scummvm-sites/commit/0aacb1ce2624932f0773527c2996a7e0ffb909a5
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-29T11:15:46+05:30
Commit Message:
INTEGRITY: Add depth option to compute_hash.py
Directories at a given depth (defaults to 0) are
treated as top-level dirs of a game, and hashes of
all files are computed, including those in
subdirectories
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index cbd8c52..b0e171a 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -8,6 +8,17 @@ def filesize(filepath):
return os.stat(filepath).st_size
+def get_dirs_at_depth(directory, depth):
+ directory = directory.rstrip(os.path.sep)
+ assert os.path.isdir(directory)
+ num_sep = directory.count(os.path.sep)
+
+ for root, dirs, contents in os.walk(directory):
+ num_sep_this = root.count(os.path.sep)
+ if depth == num_sep_this - num_sep:
+ yield root
+
+
def checksum(filepath, alg, size):
""" Returns checksum value of file using a specific algoritm """
with open(filepath, "rb") as file:
@@ -59,20 +70,28 @@ def checksum(filepath, alg, size):
return hashes
-def compute_hash_of_dir(directory, depth, alg="md5", size=0):
+def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
""" Return dictionary containing checksums of all files in directory """
- res = dict()
- # Getting contents of directory and filtering only the files
- files = [f for f in os.listdir(directory) if os.path.isfile(
- os.path.join(directory, f))]
+ res = []
+
+ for directory in get_dirs_at_depth(root_directory, depth):
+ hash_of_dir = dict()
+ files = []
- for file in files:
- res[file] = checksum(os.path.join(directory, file), alg=alg, size=size)
+ # Getting only files of directory and subdirectories recursively
+ for root, dirs, contents in os.walk(directory):
+ files.extend([os.path.join(root, f) for f in contents])
+
+ for file in files:
+ hash_of_dir[os.path.relpath(file, root_directory)] = checksum(
+ file, alg, size)
+
+ res.append(hash_of_dir)
return res
-def create_dat_file(hash_of_dir, path):
+def create_dat_file(hash_of_dirs, path):
with open(f"{os.path.basename(path)}.dat", "w") as file:
# Header
file.writelines([
@@ -82,12 +101,13 @@ def create_dat_file(hash_of_dir, path):
])
# Game files
- file.write("game (\n")
- for filename, hashes in hash_of_dir.items():
- # Only works for MD5s, ignores optional extra size
- data = f"name \"{filename}\" size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
- file.write(f"\trom ( {data} )\n")
- file.write(")\n\n")
+ for hash_of_dir in hash_of_dirs:
+ file.write("game (\n")
+ for filename, hashes in hash_of_dir.items():
+ # Only works for MD5s, ignores optional extra size
+ data = f"name \"{filename}\" size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
+ file.write(f"\trom ( {data} )\n")
+ file.write(")\n\n")
parser = argparse.ArgumentParser()
@@ -99,9 +119,7 @@ parser.add_argument("--size",
help="Use first n bytes of file to calculate checksum")
args = parser.parse_args()
path = os.path.abspath(args.directory) if args.directory else os.getcwd()
-depth = args.depth
-checksum_size = args.size
-
+depth = int(args.depth) if args.depth else 0
+checksum_size = int(args.size) if args.size else 0
-path = os.path.expanduser("~/Downloads/drascula-1.0")
-create_dat_file(compute_hash_of_dir(path, depth, size=4000), path)
+create_dat_file(compute_hash_of_dirs(path, depth, checksum_size), path)
Commit: 23791d3177ad38922622bbeac059df2d18c3561c
https://github.com/scummvm/scummvm-sites/commit/23791d3177ad38922622bbeac059df2d18c3561c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-29T18:32:14+05:30
Commit Message:
INTEGRITY: Add mysql_config file for username/pass
Changed paths:
dat_parser.php
index.php
logs.php
schema.php
seeds.php
diff --git a/dat_parser.php b/dat_parser.php
index 08cf43b..279b42c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -304,9 +304,10 @@ function insert_filechecksum($file, $checktype, $conn) {
* Create and return a mysqli connection
*/
function db_connect() {
+ $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
- $username = "username";
- $password = "password";
+ $username = $mysql_cred["username"];
+ $password = $mysql_cred["password"];
$dbname = "integrity";
// Create connection
diff --git a/index.php b/index.php
index 376ec71..10754e4 100644
--- a/index.php
+++ b/index.php
@@ -1,9 +1,10 @@
<?php
$filename = "index.php";
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
-$username = "username";
-$password = "password";
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
$dbname = "integrity";
// Create connection
diff --git a/logs.php b/logs.php
index 6cc00ec..346134e 100644
--- a/logs.php
+++ b/logs.php
@@ -1,9 +1,10 @@
<?php
$filename = "logs.php";
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
-$username = "username";
-$password = "password";
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
$dbname = "integrity";
// Create connection
diff --git a/schema.php b/schema.php
index 4f59932..57a04be 100644
--- a/schema.php
+++ b/schema.php
@@ -1,8 +1,9 @@
<?php
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
-$username = "username";
-$password = "password";
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
$dbname = "integrity";
// Create connection
diff --git a/seeds.php b/seeds.php
index 113524b..c9d62e6 100644
--- a/seeds.php
+++ b/seeds.php
@@ -1,8 +1,9 @@
<?php
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
-$username = "username";
-$password = "password";
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
$dbname = "integrity";
// Create connection
Commit: 7221a59fff2612ae0b0abfc6474e309386c98e91
https://github.com/scummvm/scummvm-sites/commit/7221a59fff2612ae0b0abfc6474e309386c98e91
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-29T19:00:04+05:30
Commit Message:
INTEGRITY: Remove some PHP8 functions
Changed paths:
dat_parser.php
index.php
logs.php
diff --git a/dat_parser.php b/dat_parser.php
index 279b42c..b81f640 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -195,7 +195,7 @@ function find_matching_game($game_files) {
foreach (array_count_values($matching_filesets) as $key => $value) {
$count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
JOIN fileset ON file.fileset = fileset.id
- WHERE fileset.id = '%s'", $key))->fetch_column();
+ WHERE fileset.id = '%s'", $key))->fetch_array()[0];
// We use < instead of != since one file may have more than one entry in the fileset
// We see this in Drascula English version, where one entry is duplicated
diff --git a/index.php b/index.php
index 10754e4..f466fa8 100644
--- a/index.php
+++ b/index.php
@@ -29,7 +29,7 @@ else {
$results_per_page = 25;
$offset = ($page - 1) * $results_per_page;
-$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_column();
+$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_array()[0];
$num_of_pages = ceil($num_of_results / $results_per_page);
$query = sprintf("SELECT name, extra, platform, language FROM game LIMIT %d OFFSET %d",
diff --git a/logs.php b/logs.php
index 346134e..8cb34de 100644
--- a/logs.php
+++ b/logs.php
@@ -29,7 +29,7 @@ else {
$results_per_page = 25;
$offset = ($page - 1) * $results_per_page;
-$num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_column();
+$num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_array()[0];
$num_of_pages = ceil($num_of_results / $results_per_page);
$query = sprintf("SELECT * FROM log LIMIT %d OFFSET %d", $results_per_page, $offset);
Commit: accd4345f20867f6dfd2212efa8e7b79c7ca12cf
https://github.com/scummvm/scummvm-sites/commit/accd4345f20867f6dfd2212efa8e7b79c7ca12cf
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-29T22:10:14+05:30
Commit Message:
INTEGRITY: Calc fileset.key for detection entries
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index b81f640..af387ed 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -2,6 +2,20 @@
ini_set('memory_limit', '512M');
+/**
+ * Calculate `key` value as md5("file1:size:md5:file2:...")
+ */
+function calc_key($files) {
+ $key_string = "";
+ foreach ($files as $file) {
+ foreach ($file as $key => $value) {
+ $key_string .= ':' . $value;
+ }
+ }
+ $key_string = trim($key_string, ':');
+ return md5($key_string);
+}
+
/**
* Convert string of checksum data from rom into associated array
* Returns array instead of updating one like map_key_values
@@ -392,10 +406,17 @@ function db_insert($data_arr) {
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
insert_fileset($src, $detection, $conn);
+ calc_key($fileset["rom"]);
foreach ($fileset["rom"] as $file) {
insert_file($file, $detection, $conn);
insert_filechecksum($file, "md5-5000", $conn);
}
+
+ // Add key if uploaded DAT is of detection entries
+ if ($detection) {
+ $conn->query(sprintf("UPDATE fileset SET `key` = '%s' WHERE id = @fileset_last",
+ calc_key($fileset["rom"])));
+ }
}
if (!$conn->commit())
echo "Inserting failed<br/>";
@@ -460,9 +481,9 @@ function populate_matching_games() {
}
echo "<pre>";
-// db_insert(parse_dat("scummvm_detection_entries.dat"));
-// db_insert(parse_dat("drascula-2.0.dat"));
-populate_matching_games();
+db_insert(parse_dat("scummvm_detection_entries.dat"));
+// db_insert(parse_dat("drascula-1.0.dat"));
+// populate_matching_games();
echo "</pre>";
?>
Commit: d24480df5550ad810c4b686d5bd0cd14ea2e9016
https://github.com/scummvm/scummvm-sites/commit/d24480df5550ad810c4b686d5bd0cd14ea2e9016
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-29T22:57:18+05:30
Commit Message:
INTEGRITY: Create log on inserting into the db
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index af387ed..4a00c53 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -150,7 +150,7 @@ function parse_dat($dat_filepath) {
// print_r($resources);
// echo "</pre>";
- return array($header, $game_data, $resources);
+ return array($header, $game_data, $resources, $dat_filepath);
}
/**
@@ -360,6 +360,7 @@ function db_insert($data_arr) {
$header = $data_arr[0];
$game_data = $data_arr[1];
$resources = $data_arr[2];
+ $filepath = $data_arr[3];
$conn = db_connect();
@@ -370,6 +371,7 @@ function db_insert($data_arr) {
* _anything else_ -> DAT file
*/
$author = $header["author"];
+ $version = $header["version"];
/**
* src can be:
@@ -384,11 +386,12 @@ function db_insert($data_arr) {
if ($author == "cli")
$src = "scan";
elseif ($author == "scummvm")
- $src = "detection";
+ $src = "scummvm";
else
$src = "dat";
- $detection = ($src == "detection");
+ $detection = ($src == "scummvm");
+ $status = $detection ? "detection" : $src;
foreach ($game_data as $fileset) {
if ($detection) {
@@ -418,8 +421,14 @@ function db_insert($data_arr) {
calc_key($fileset["rom"])));
}
}
+ $category_text = "Uploaded from " . $src . ", state '" . $status . "'";
+ $log_text = sprintf("Loaded DAT file, filename \"%s\", size %d, author \"%s\", version %s",
+ $filepath, filesize($filepath), $author, $version);
+
if (!$conn->commit())
echo "Inserting failed<br/>";
+ else
+ create_log(mysqli_real_escape_string($conn, $category_text), "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: User name is "unknown"
}
function populate_matching_games() {
@@ -464,6 +473,8 @@ function populate_matching_games() {
$matched_game = array_map(function ($val) {
return (is_null($val)) ? "NULL" : $val;
}, $matched_game);
+
+ $category_text = "Matched, state '" . $status . "'";
$log_text = sprintf("Matched game %s: %s-%s-%s variant %s from %s",
$matched_game["engineid"], $matched_game["gameid"], $matched_game["platform"],
$matched_game["language"], $matched_game["key"], $matched_game["src"]);
@@ -473,7 +484,7 @@ function populate_matching_games() {
SET game = %d, status = '%s'
WHERE id = %d", $matched_game["id"], $status, $fileset[0]);
if ($conn->query($query))
- create_log("Matched, state '" . $status . "'", "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
+ create_log($category_text, "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
if (!$conn->commit())
echo "Updating matched games failed<br/>";
@@ -481,8 +492,8 @@ function populate_matching_games() {
}
echo "<pre>";
-db_insert(parse_dat("scummvm_detection_entries.dat"));
-// db_insert(parse_dat("drascula-1.0.dat"));
+// db_insert(parse_dat("scummvm_detection_entries.dat"));
+db_insert(parse_dat("drascula-1.0.dat"));
// populate_matching_games();
echo "</pre>";
?>
Commit: 5e7ac07d552a5a10ccb91ee0695ebe1b37f986f3
https://github.com/scummvm/scummvm-sites/commit/5e7ac07d552a5a10ccb91ee0695ebe1b37f986f3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T12:51:14+05:30
Commit Message:
INTEGRITY: Fix name of files when using depth option
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index b0e171a..d19b539 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -83,8 +83,8 @@ def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
files.extend([os.path.join(root, f) for f in contents])
for file in files:
- hash_of_dir[os.path.relpath(file, root_directory)] = checksum(
- file, alg, size)
+ hash_of_dir[os.path.relpath(file, directory)] = (checksum(
+ file, alg, size), filesize(file))
res.append(hash_of_dir)
@@ -103,9 +103,9 @@ def create_dat_file(hash_of_dirs, path):
# Game files
for hash_of_dir in hash_of_dirs:
file.write("game (\n")
- for filename, hashes in hash_of_dir.items():
+ for filename, (hashes, filesize) in hash_of_dir.items():
# Only works for MD5s, ignores optional extra size
- data = f"name \"{filename}\" size {filesize(os.path.join(path, filename))} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
+ data = f"name \"{filename}\" size {filesize} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
file.write(f"\trom ( {data} )\n")
file.write(")\n\n")
Commit: 2d033622b6fffd0f62c0ea56d57a5a14311c107a
https://github.com/scummvm/scummvm-sites/commit/2d033622b6fffd0f62c0ea56d57a5a14311c107a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T12:54:21+05:30
Commit Message:
INTEGRITY: Add version to header of DAT file
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index d19b539..97b6f50 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -2,6 +2,8 @@ import hashlib
import os
import argparse
+script_version = "0.1"
+
def filesize(filepath):
""" Returns size of file """
@@ -97,6 +99,7 @@ def create_dat_file(hash_of_dirs, path):
file.writelines([
"scummvm (\n",
f"\tauthor cli\n",
+ f"\tversion {script_version}\n",
")\n\n"
])
Commit: 96096724b2da3750c205e54516fa51b029cac6a9
https://github.com/scummvm/scummvm-sites/commit/96096724b2da3750c205e54516fa51b029cac6a9
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T16:22:06+05:30
Commit Message:
INTEGRITY: Add ON DELETE CASCADE to file, filechecksum
Changed checksize to VARCHAR to allow '1M' as a value
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 57a04be..a2d6d0b 100644
--- a/schema.php
+++ b/schema.php
@@ -87,7 +87,7 @@ $table = "CREATE TABLE IF NOT EXISTS file (
checksum VARCHAR(64) NOT NULL,
fileset INT NOT NULL,
detection BOOLEAN NOT NULL,
- FOREIGN KEY (fileset) REFERENCES fileset(id)
+ FOREIGN KEY (fileset) REFERENCES fileset(id) ON DELETE CASCADE
)";
if ($conn->query($table) === TRUE) {
@@ -101,10 +101,10 @@ else {
$table = "CREATE TABLE IF NOT EXISTS filechecksum (
id INT AUTO_INCREMENT PRIMARY KEY,
file INT NOT NULL,
- checksize INT NOT NULL,
+ checksize VARCHAR(10) NOT NULL,
checktype VARCHAR(10) NOT NULL,
checksum VARCHAR(64) NOT NULL,
- FOREIGN KEY (file) REFERENCES file(id)
+ FOREIGN KEY (file) REFERENCES file(id) ON DELETE CASCADE
)";
if ($conn->query($table) === TRUE) {
Commit: 0c730a0092ea7bd7773f63397cc50bf722a0e46d
https://github.com/scummvm/scummvm-sites/commit/0c730a0092ea7bd7773f63397cc50bf722a0e46d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T16:30:26+05:30
Commit Message:
INTEGRITY: Merge filesets on detection entry match
- Create merge_filesets()
- Fix file.checksum value for cli DATs
- Fix status based on $src
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 4a00c53..a2d0a8d 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -216,7 +216,8 @@ function find_matching_game($game_files) {
if ($value < $matches_count || $value < $count_files_in_fileset)
continue;
- $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform, language, `key`, src
+ $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform,
+ language, `key`, src, fileset.id as fileset
FROM game
JOIN fileset ON fileset.game = game.id
JOIN engine ON engine.id = game.engine
@@ -285,13 +286,18 @@ function insert_fileset($src, $detection, $conn) {
* $file is an associated array (the contents of 'rom')
* If checksum of the given checktype doesn't exists, silently fails
*/
-function insert_file($file, $detection, $conn) {
- // Find first checksum value
+function insert_file($file, $detection, $src, $conn) {
+ // Find md5-5000, or else use first checksum value
$checksum = "";
- foreach ($file as $key => $value) {
- if (strpos($key, "md5") !== false) {
- list($tmp1, $tmp2, $checksum) = get_checksum_props($key, $value);
- break;
+ if (isset($file["md5-5000"])) {
+ $checksum = $file["md5-5000"];
+ }
+ else {
+ foreach ($file as $key => $value) {
+ if (strpos($key, "md5") !== false) {
+ list($tmp1, $tmp2, $checksum) = get_checksum_props($key, $value);
+ break;
+ }
}
}
@@ -314,6 +320,28 @@ function insert_filechecksum($file, $checktype, $conn) {
$conn->query($query);
}
+/**
+ * Merge two filesets without duplicating files
+ * Used after matching an unconfirmed fileset with a detection entry
+ */
+function merge_filesets($detection_id, $dat_id, $conn) {
+ $detection_files = $conn->query(sprintf("SELECT checksum FROM file
+ WHERE fileset = '%d'", $detection_id))->fetch_all();
+
+ foreach ($detection_files as $file) {
+ $checksum = $file[0];
+
+ // Mark files present in the detection entries
+ $conn->query(sprintf("UPDATE file
+ SET detection = TRUE
+ WHERE id = '%d' AND checksum = '%s'", $dat_id, $checksum));
+
+ // Delete original detection entry so newly matched fileset is the only fileset for game
+ $conn->query(sprintf("DELETE FROM fileset
+ WHERE id = '%d'", $detection_id));
+ }
+}
+
/**
* Create and return a mysqli connection
*/
@@ -411,8 +439,11 @@ function db_insert($data_arr) {
insert_fileset($src, $detection, $conn);
calc_key($fileset["rom"]);
foreach ($fileset["rom"] as $file) {
- insert_file($file, $detection, $conn);
- insert_filechecksum($file, "md5-5000", $conn);
+ insert_file($file, $detection, $src, $conn);
+ foreach ($file as $key => $value) {
+ if ($key != "name" && $key != "size")
+ insert_filechecksum($file, $key, $conn);
+ }
}
// Add key if uploaded DAT is of detection entries
@@ -464,9 +495,9 @@ function populate_matching_games() {
// Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
$status = "unconfirmed";
- if ($matched_game["src"] == "dat")
+ if ($fileset[0][2] == "dat")
$status = "partialmatch";
- elseif ($matched_game["src"] == "scan")
+ elseif ($fileset[0][2] == "scan")
$status = "fullmatch";
// Convert NULL values to string with value NULL for printing
@@ -481,20 +512,24 @@ function populate_matching_games() {
// Updating the fileset.game value to be $matched_game["id"]
$query = sprintf("UPDATE fileset
- SET game = %d, status = '%s'
- WHERE id = %d", $matched_game["id"], $status, $fileset[0]);
+ SET game = %d, status = '%s', `key` = '%s'
+ WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
+ merge_filesets($matched_game["fileset"], $fileset[0][0], $conn);
+
if ($conn->query($query))
- create_log($category_text, "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
+ create_log(mysqli_real_escape_string($conn, $category_text), "unknown",
+ mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
if (!$conn->commit())
echo "Updating matched games failed<br/>";
}
}
-echo "<pre>";
+// echo "<pre>";
// db_insert(parse_dat("scummvm_detection_entries.dat"));
-db_insert(parse_dat("drascula-1.0.dat"));
+// db_insert(parse_dat("drascula-1.0.dat"));
+// db_insert(parse_dat("Downloads.dat"));
// populate_matching_games();
-echo "</pre>";
+// echo "</pre>";
?>
Commit: c75b15ef41774e0157a683ab56396fa63a9d24d8
https://github.com/scummvm/scummvm-sites/commit/c75b15ef41774e0157a683ab56396fa63a9d24d8
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T21:43:37+05:30
Commit Message:
INTEGRITY: Fix formatting and text for webpages
Changed paths:
index.php
logs.php
diff --git a/index.php b/index.php
index f466fa8..e3c6cac 100644
--- a/index.php
+++ b/index.php
@@ -32,23 +32,31 @@ $offset = ($page - 1) * $results_per_page;
$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_array()[0];
$num_of_pages = ceil($num_of_results / $results_per_page);
-$query = sprintf("SELECT name, extra, platform, language FROM game LIMIT %d OFFSET %d",
+$query = sprintf("SELECT engineid, gameid, extra, platform, language, game.name, status
+FROM game
+JOIN engine ON engine.id = game.engine
+JOIN fileset ON game.id = fileset.game
+LIMIT %d OFFSET %d",
$results_per_page, $offset);
$result = $conn->query($query);
echo "<ol start=\"" . $offset + 1 . "\">";
while ($row = $result->fetch_array()) {
echo "<li>";
- echo sprintf("%s (%s, %s, %s)<br/>", $row["name"], $row["extra"], $row["platform"], $row["language"]);
+ echo sprintf("%s:%s-%s-%s-%s %s %s<br/>",
+ $row["engineid"], $row["gameid"], $row["extra"], $row["platform"], $row["language"],
+ $row["name"], $row["status"]);
echo "</li>";
}
echo "</ol>";
-echo '<a href =' . $filename . '"?page=1">first</a>';
-if ($page > 1)
- echo '<a href =' . $filename . '"?page=' . $page - 1 . '">prev</a>';
-if ($page < $num_of_pages)
- echo '<a href =' . $filename . '"?page=' . $page + 1 . '">next</a>';
-echo '<a href =' . $filename . '"?page=' . $num_of_pages . '">last</a>';
+if ($page > 1) {
+ echo sprintf('<a href=%s>first</a>', $filename);
+ echo sprintf('<a href=%s?page=%d>prev</a>', $filename, $page - 1);
+}
+if ($page < $num_of_pages) {
+ echo sprintf('<a href=%s?page=%d>next</a>', $filename, $page + 1);
+ echo sprintf('<a href=%s?page=%d>last</a>', $filename, $num_of_pages);
+}
?>
diff --git a/logs.php b/logs.php
index 8cb34de..ca395b5 100644
--- a/logs.php
+++ b/logs.php
@@ -38,16 +38,18 @@ $result = $conn->query($query);
echo "<ol start=\"" . $offset + 1 . "\">";
while ($row = $result->fetch_array()) {
echo "<li>";
- echo sprintf("%s (%s, %s, %s)<br/>", $row["timestamp"], $row["category"], $row["user"], $row["text"]);
+ echo sprintf("%s (%s, user %s) %s<br/>", $row["timestamp"], $row["category"], $row["user"], $row["text"]);
echo "</li>";
}
echo "</ol>";
-echo '<a href =' . $filename . '"?page=1">first</a>';
-if ($page > 1)
- echo '<a href =' . $filename . '"?page=' . $page - 1 . '">prev</a>';
-if ($page < $num_of_pages)
- echo '<a href =' . $filename . '"?page=' . $page + 1 . '">next</a>';
-echo '<a href =' . $filename . '"?page=' . $num_of_pages . '">last</a>';
+if ($page > 1) {
+ echo sprintf('<a href=%s>first</a>', $filename);
+ echo sprintf('<a href=%s?page=%d>prev</a>', $filename, $page - 1);
+}
+if ($page < $num_of_pages) {
+ echo sprintf('<a href=%s?page=%d>next</a>', $filename, $page + 1);
+ echo sprintf('<a href=%s?page=%d>last</a>', $filename, $num_of_pages);
+}
?>
Commit: 77cc3740205935b1cf96a5afd610f4b0fab01e09
https://github.com/scummvm/scummvm-sites/commit/77cc3740205935b1cf96a5afd610f4b0fab01e09
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-06-30T21:45:26+05:30
Commit Message:
INTEGRITY: Merge filesets based on all checksums
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index a2d0a8d..d7c7ad2 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -468,8 +468,9 @@ function populate_matching_games() {
// Getting unmatched filesets
$unmatched_filesets = array();
- $unmatched_files = $conn->query(sprintf("SELECT fileset.id, file.checksum, fileset.src from fileset
+ $unmatched_files = $conn->query(sprintf("SELECT fileset.id, filechecksum.checksum, fileset.src from fileset
JOIN file ON file.fileset = fileset.id
+ JOIN filechecksum ON file.id = filechecksum.file
WHERE fileset.game IS NULL"));
$unmatched_files = $unmatched_files->fetch_all();
@@ -525,11 +526,11 @@ function populate_matching_games() {
}
}
-// echo "<pre>";
+echo "<pre>";
// db_insert(parse_dat("scummvm_detection_entries.dat"));
// db_insert(parse_dat("drascula-1.0.dat"));
// db_insert(parse_dat("Downloads.dat"));
-// populate_matching_games();
-// echo "</pre>";
+populate_matching_games();
+echo "</pre>";
?>
Commit: 0ad938c7317d323d604c8e57c0140b8bc4788f47
https://github.com/scummvm/scummvm-sites/commit/0ad938c7317d323d604c8e57c0140b8bc4788f47
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-01T18:41:01+05:30
Commit Message:
INTEGRITY: Copy checksize, checktype to new entries
Also handle duplicate detection entries:
- One copy of the file data is deleted
- If any copies remain, they are moved to the new fileset
- New fileset data is updated as normal
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index d7c7ad2..6fa4857 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -325,20 +325,32 @@ function insert_filechecksum($file, $checktype, $conn) {
* Used after matching an unconfirmed fileset with a detection entry
*/
function merge_filesets($detection_id, $dat_id, $conn) {
- $detection_files = $conn->query(sprintf("SELECT checksum FROM file
+ $detection_files = $conn->query(sprintf("SELECT filechecksum.checksum, checksize, checktype
+ FROM filechecksum JOIN file on file.id = filechecksum.file
WHERE fileset = '%d'", $detection_id))->fetch_all();
foreach ($detection_files as $file) {
$checksum = $file[0];
+ $checksize = $file[1];
+ $checktype = $file[2];
- // Mark files present in the detection entries
+ // Delete original detection entry so newly matched fileset is the only fileset for game
+ $conn->query(sprintf("DELETE FROM file
+ WHERE checksum = '%s' LIMIT 1", $checksum));
+
+ // Move any remaining files (duplicates) to the new fileset
$conn->query(sprintf("UPDATE file
- SET detection = TRUE
- WHERE id = '%d' AND checksum = '%s'", $dat_id, $checksum));
+ SET fileset = %d
+ WHERE fileset = %d AND checksum = '%s'"), $dat_id, $detection_id, $checksum);
- // Delete original detection entry so newly matched fileset is the only fileset for game
- $conn->query(sprintf("DELETE FROM fileset
- WHERE id = '%d'", $detection_id));
+ // Mark files present in the detection entries
+ $conn->query(sprintf("UPDATE file
+ JOIN filechecksum ON filechecksum.file = file.id
+ SET detection = TRUE,
+ checksize = %d,
+ checktype = '%s'
+ WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
+ $checksize, $checktype, $dat_id, $checksum));
}
}
@@ -526,11 +538,11 @@ function populate_matching_games() {
}
}
-echo "<pre>";
+// echo "<pre>";
// db_insert(parse_dat("scummvm_detection_entries.dat"));
// db_insert(parse_dat("drascula-1.0.dat"));
// db_insert(parse_dat("Downloads.dat"));
-populate_matching_games();
-echo "</pre>";
+// populate_matching_games();
+// echo "</pre>";
?>
Commit: 8ae58943e65806cc59348a757aa83d3e03c646cd
https://github.com/scummvm/scummvm-sites/commit/8ae58943e65806cc59348a757aa83d3e03c646cd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T13:48:59+05:30
Commit Message:
INTEGRITY: Remove status unconfirmed
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 6fa4857..32ad392 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -264,8 +264,7 @@ function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn
* Called for both detection entries and other forms of DATs
*/
function insert_fileset($src, $detection, $conn) {
- // $status can be: {detection, unconfirmed, confirmed (unused here)}
- $status = "unconfirmed";
+ $status = $detection ? "detection" : $src;
$game = "NULL";
if ($detection) {
@@ -324,7 +323,9 @@ function insert_filechecksum($file, $checktype, $conn) {
* Merge two filesets without duplicating files
* Used after matching an unconfirmed fileset with a detection entry
*/
-function merge_filesets($detection_id, $dat_id, $conn) {
+function merge_filesets($detection_id, $dat_id) {
+ $conn = db_connect();
+
$detection_files = $conn->query(sprintf("SELECT filechecksum.checksum, checksize, checktype
FROM filechecksum JOIN file on file.id = filechecksum.file
WHERE fileset = '%d'", $detection_id))->fetch_all();
@@ -341,7 +342,7 @@ function merge_filesets($detection_id, $dat_id, $conn) {
// Move any remaining files (duplicates) to the new fileset
$conn->query(sprintf("UPDATE file
SET fileset = %d
- WHERE fileset = %d AND checksum = '%s'"), $dat_id, $detection_id, $checksum);
+ WHERE fileset = %d AND checksum = '%s'", $dat_id, $detection_id, $checksum));
// Mark files present in the detection entries
$conn->query(sprintf("UPDATE file
@@ -507,7 +508,7 @@ function populate_matching_games() {
$matched_game = $matching_games[0];
// Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
- $status = "unconfirmed";
+ $status = $fileset[0][2];
if ($fileset[0][2] == "dat")
$status = "partialmatch";
elseif ($fileset[0][2] == "scan")
Commit: e862900d411056c4ec9294ee47651c91b8461982
https://github.com/scummvm/scummvm-sites/commit/e862900d411056c4ec9294ee47651c91b8461982
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T14:00:38+05:30
Commit Message:
INTEGRITY: Rename index.php to games_list.php
Changed paths:
A games_list.php
R index.php
diff --git a/index.php b/games_list.php
similarity index 98%
rename from index.php
rename to games_list.php
index e3c6cac..b2ecfed 100644
--- a/index.php
+++ b/games_list.php
@@ -1,5 +1,5 @@
<?php
-$filename = "index.php";
+$filename = "games_list.php";
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
Commit: bb3507030fb49da6085980fbb5ffd2aa24c7ee1a
https://github.com/scummvm/scummvm-sites/commit/bb3507030fb49da6085980fbb5ffd2aa24c7ee1a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T14:13:11+05:30
Commit Message:
INTEGRITY: Create (new) index.php with page links
Changed paths:
A index.php
diff --git a/index.php b/index.php
new file mode 100644
index 0000000..2bc2461
--- /dev/null
+++ b/index.php
@@ -0,0 +1,5 @@
+<?php
+
+echo '<a href="games_list.php">List of Detection entries</a><br/>';
+echo '<a href="logs.php">Logs of developer actions</a><br/>';
+?>
Commit: dfa9e869dad40f48dcb31ca57e02cd26f8408b5c
https://github.com/scummvm/scummvm-sites/commit/dfa9e869dad40f48dcb31ca57e02cd26f8408b5c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T14:35:41+05:30
Commit Message:
INTEGRITY: Create fileset `history` table
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 32ad392..0c178eb 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -353,6 +353,10 @@ function merge_filesets($detection_id, $dat_id) {
WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
$checksize, $checktype, $dat_id, $checksum));
}
+
+ // Add fileset pair to history ($dat_id is the new fileset for $detection_id)
+ $conn->query(sprintf("INSERT INTO history (fileset, oldfileset)
+ VALUES (%d, %d)", $dat_id, $detection_id));
}
/**
@@ -528,7 +532,7 @@ function populate_matching_games() {
$query = sprintf("UPDATE fileset
SET game = %d, status = '%s', `key` = '%s'
WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
- merge_filesets($matched_game["fileset"], $fileset[0][0], $conn);
+ merge_filesets($matched_game["fileset"], $fileset[0][0]);
if ($conn->query($query))
create_log(mysqli_real_escape_string($conn, $category_text), "unknown",
diff --git a/schema.php b/schema.php
index a2d6d0b..a2e5313 100644
--- a/schema.php
+++ b/schema.php
@@ -149,6 +149,20 @@ else {
echo "Error creating 'log' table: " . $conn->error;
}
+// Create history table
+$table = "CREATE TABLE IF NOT EXISTS history (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ fileset INT NOT NULL,
+ oldfileset INT NOT NULL
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'history' created successfully<br/>";
+}
+else {
+ echo "Error creating 'history' table: " . $conn->error;
+}
+
// Create indices for fast data retrieval
// PK and FK are automatically indexed in InnoDB, so they are not included
$index = "CREATE INDEX detection ON file (detection)";
@@ -178,6 +192,15 @@ else {
echo "Error creating index for 'engine.engineid': " . $conn->error;
}
+$index = "CREATE INDEX fileset ON history (fileset)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'history.fileset'<br/>";
+}
+else {
+ echo "Error creating index for 'history.fileset': " . $conn->error;
+}
+
$conn->close();
?>
Commit: 748a35e341a41c98237b86ba0cc50fe148d55972
https://github.com/scummvm/scummvm-sites/commit/748a35e341a41c98237b86ba0cc50fe148d55972
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T15:10:55+05:30
Commit Message:
INTEGRITY: games_list.php now uses table for data
TODO: Fix numbering for the table on all pages
Changed paths:
A style.css
games_list.php
diff --git a/games_list.php b/games_list.php
index b2ecfed..4e63a6c 100644
--- a/games_list.php
+++ b/games_list.php
@@ -1,5 +1,7 @@
<?php
$filename = "games_list.php";
+$stylesheet = "style.css";
+echo "<link rel='stylesheet' href='{$stylesheet}'>";
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
@@ -40,15 +42,19 @@ LIMIT %d OFFSET %d",
$results_per_page, $offset);
$result = $conn->query($query);
-echo "<ol start=\"" . $offset + 1 . "\">";
+echo "<table>";
while ($row = $result->fetch_array()) {
- echo "<li>";
- echo sprintf("%s:%s-%s-%s-%s %s %s<br/>",
- $row["engineid"], $row["gameid"], $row["extra"], $row["platform"], $row["language"],
- $row["name"], $row["status"]);
- echo "</li>";
+ echo "<tr>";
+ echo "<td>{$row['engineid']}</td>";
+ echo "<td>{$row['gameid']}</td>";
+ echo "<td>{$row['extra']}</td>";
+ echo "<td>{$row['platform']}</td>";
+ echo "<td>{$row['language']}</td>";
+ echo "<td>{$row['name']}</td>";
+ echo "<td>{$row['status']}</td>";
+ echo "</tr>";
}
-echo "</ol>";
+echo "</table>";
if ($page > 1) {
echo sprintf('<a href=%s>first</a>', $filename);
diff --git a/style.css b/style.css
new file mode 100644
index 0000000..4d012b0
--- /dev/null
+++ b/style.css
@@ -0,0 +1,11 @@
+table {
+ counter-reset: rowNumber;
+}
+
+table tr::before {
+ display: table-cell;
+ counter-increment: rowNumber;
+ content: counter(rowNumber) ".";
+ padding-right: 0.3em;
+ text-align: right;
+}
Commit: 3b62839dc2f7e278b56797c639e15f76abac8bc1
https://github.com/scummvm/scummvm-sites/commit/3b62839dc2f7e278b56797c639e15f76abac8bc1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T20:06:10+05:30
Commit Message:
INTEGRITY: Extract duplicated code into remove_quotes()
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 0c178eb..3d40f97 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -16,6 +16,14 @@ function calc_key($files) {
return md5($key_string);
}
+function remove_quotes($string) {
+ // Remove quotes from value if they are present
+ if ($string[0] == "\"")
+ $string = substr($string, 1, -1);
+
+ return $string;
+}
+
/**
* Convert string of checksum data from rom into associated array
* Returns array instead of updating one like map_key_values
@@ -27,9 +35,8 @@ function map_checksum_data($content_string) {
for ($i = 1; $i < count($temp); $i += 2) {
if ($temp[$i] == ')')
continue;
- if ($temp[$i + 1][0] == '"') {
- $temp[$i + 1] = substr($temp[$i + 1], 1, -1);
- }
+ $temp[$i + 1] = remove_quotes($temp[$i + 1]);
+
$arr[$temp[$i]] = stripslashes($temp[$i + 1]);
}
@@ -49,10 +56,7 @@ function map_key_values($content_string, &$arr) {
if (trim($pair) == "(" or trim($pair) == ")")
continue;
$pair = array_map("trim", preg_split("/ +/", $pair, 2));
-
- // Remove quotes from value if they are present
- if ($pair[1][0] == "\"")
- $pair[1] = substr($pair[1], 1, -1);
+ $pair[1] = remove_quotes($pair[1]);
// Handle duplicate keys (if the key is rom) and add values to a arary instead
if ($pair[0] == "rom") {
Commit: f2e799542edee87c18b3bdf0a7df21dcd816d977
https://github.com/scummvm/scummvm-sites/commit/f2e799542edee87c18b3bdf0a7df21dcd816d977
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-03T21:08:28+05:30
Commit Message:
INTEGRITY: Improve styling for games_list.php
- Table now has borders
- Numbering now works with pagination
- Look of page navigation buttons greatly improved
Changed paths:
games_list.php
style.css
diff --git a/games_list.php b/games_list.php
index 4e63a6c..23622ca 100644
--- a/games_list.php
+++ b/games_list.php
@@ -43,8 +43,19 @@ LIMIT %d OFFSET %d",
$result = $conn->query($query);
echo "<table>";
+echo "<th></th>";
+echo "<th>engineid</th>";
+echo "<th>gameid</th>";
+echo "<th>extra</th>";
+echo "<th>platform</th>";
+echo "<th>language</th>";
+echo "<th>name</th>";
+echo "<th>status</th>";
+
+$counter = $offset + 1;
while ($row = $result->fetch_array()) {
echo "<tr>";
+ echo "<td>{$counter}.</td>";
echo "<td>{$row['engineid']}</td>";
echo "<td>{$row['gameid']}</td>";
echo "<td>{$row['extra']}</td>";
@@ -53,16 +64,20 @@ while ($row = $result->fetch_array()) {
echo "<td>{$row['name']}</td>";
echo "<td>{$row['status']}</td>";
echo "</tr>";
+
+ $counter++;
}
echo "</table>";
+echo "<div class=pagination>";
if ($page > 1) {
- echo sprintf('<a href=%s>first</a>', $filename);
- echo sprintf('<a href=%s?page=%d>prev</a>', $filename, $page - 1);
+ echo "<a href={$filename}>â®â®</a>";
+ echo sprintf("<a href=%s?page=%d>â®</a>", $filename, $page - 1);
}
if ($page < $num_of_pages) {
- echo sprintf('<a href=%s?page=%d>next</a>', $filename, $page + 1);
- echo sprintf('<a href=%s?page=%d>last</a>', $filename, $num_of_pages);
+ echo sprintf("<a href=%s?page=%d>â¯</a>", $filename, $page + 1);
+ echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>";
}
+echo "</div>";
?>
diff --git a/style.css b/style.css
index 4d012b0..545c263 100644
--- a/style.css
+++ b/style.css
@@ -1,11 +1,24 @@
-table {
- counter-reset: rowNumber;
+table, th, td {
+ border: 1px solid;
}
-table tr::before {
- display: table-cell;
- counter-increment: rowNumber;
- content: counter(rowNumber) ".";
- padding-right: 0.3em;
- text-align: right;
+.pagination {
+ display: inline-block;
}
+
+.pagination a {
+ color: black;
+ float: left;
+ padding: 8px 16px;
+ text-decoration: none;
+ transition: background-color .3s;
+ border: 1px solid #ddd;
+}
+
+.pagination a.active {
+ background-color: #2d56d0;
+ color: white;
+ border: 1px solid #2d56d0;
+}
+
+.pagination a:hover:not(.active) {background-color: #ddd;}
Commit: b0ce11fa045f2b5b47b3863fbd33aa68e944ed7c
https://github.com/scummvm/scummvm-sites/commit/b0ce11fa045f2b5b47b3863fbd33aa68e944ed7c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T12:29:47+05:30
Commit Message:
INTEGRITY: Improve navigation styling
- Show buttons for first, last, previous 2, next 2 and current page in
the naviagtion div
- Next and previous pages separated from last and first pages by '...'
Changed paths:
games_list.php
style.css
diff --git a/games_list.php b/games_list.php
index 23622ca..f113c08 100644
--- a/games_list.php
+++ b/games_list.php
@@ -69,15 +69,26 @@ while ($row = $result->fetch_array()) {
}
echo "</table>";
+// Navigation elements
echo "<div class=pagination>";
-if ($page > 1) {
+if ($page > 1)
echo "<a href={$filename}>â®â®</a>";
- echo sprintf("<a href=%s?page=%d>â®</a>", $filename, $page - 1);
+if ($page - 2 > 1)
+ echo "<div class=more>...</div>";
+
+for ($i = $page - 2; $i <= $page + 2; $i++) {
+ if ($i >= 1 && $i <= $num_of_pages)
+ if ($i == $page)
+ echo sprintf("<a class=active href=%s?page=%d>%d</a>", $filename, $i, $i);
+ else
+ echo sprintf("<a href=%s?page=%d>%d</a>", $filename, $i, $i);
}
-if ($page < $num_of_pages) {
- echo sprintf("<a href=%s?page=%d>â¯</a>", $filename, $page + 1);
+
+if ($page + 2 < $num_of_pages)
+ echo "<div class=more>...</div>";
+if ($page < $num_of_pages)
echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>";
-}
+
echo "</div>";
?>
diff --git a/style.css b/style.css
index 545c263..98861a0 100644
--- a/style.css
+++ b/style.css
@@ -2,8 +2,19 @@ table, th, td {
border: 1px solid;
}
+th, td {
+ padding-inline: 5px;
+}
+
.pagination {
display: inline-block;
+ align-self: center;
+}
+
+.pagination .more {
+ color: black;
+ float: left;
+ padding: 15px 10px;
}
.pagination a {
@@ -16,9 +27,9 @@ table, th, td {
}
.pagination a.active {
- background-color: #2d56d0;
+ background-color: #27b5e8;
color: white;
- border: 1px solid #2d56d0;
+ border: 1px solid #27b5e8;
}
.pagination a:hover:not(.active) {background-color: #ddd;}
Commit: f65e67fecf8e01f740343b3ec4921d80a909d895
https://github.com/scummvm/scummvm-sites/commit/f65e67fecf8e01f740343b3ec4921d80a909d895
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T15:13:41+05:30
Commit Message:
INTEGRITY: Create pagination module
All code that is not specific to a certain page moved to a function
in an external file, which is called from games_list.php and logs.php
Changed paths:
A pagination.php
games_list.php
logs.php
diff --git a/games_list.php b/games_list.php
index f113c08..d074483 100644
--- a/games_list.php
+++ b/games_list.php
@@ -1,94 +1,13 @@
<?php
-$filename = "games_list.php";
-$stylesheet = "style.css";
-echo "<link rel='stylesheet' href='{$stylesheet}'>";
-
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
-$servername = "localhost";
-$username = $mysql_cred["username"];
-$password = $mysql_cred["password"];
-$dbname = "integrity";
-
-// Create connection
-mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
-$conn = new mysqli($servername, $username, $password);
-$conn->set_charset('utf8mb4');
-$conn->autocommit(FALSE);
-
-// Check connection
-if ($conn->connect_errno) {
- die("Connect failed: " . $conn->connect_error);
-}
-
-$conn->query("USE " . $dbname);
-
-if (!isset($_GET['page'])) {
- $page = 1;
-}
-else {
- $page = $_GET['page'];
-}
-
-$results_per_page = 25;
-$offset = ($page - 1) * $results_per_page;
-$num_of_results = $conn->query("SELECT COUNT(id) FROM game")->fetch_array()[0];
-$num_of_pages = ceil($num_of_results / $results_per_page);
+require "pagination.php";
-$query = sprintf("SELECT engineid, gameid, extra, platform, language, game.name, status
+$filename = "games_list.php";
+$count_query = "SELECT COUNT(id) FROM game";
+$select_query = "SELECT engineid, gameid, extra, platform, language, game.name, status
FROM game
JOIN engine ON engine.id = game.engine
-JOIN fileset ON game.id = fileset.game
-LIMIT %d OFFSET %d",
- $results_per_page, $offset);
-$result = $conn->query($query);
-
-echo "<table>";
-echo "<th></th>";
-echo "<th>engineid</th>";
-echo "<th>gameid</th>";
-echo "<th>extra</th>";
-echo "<th>platform</th>";
-echo "<th>language</th>";
-echo "<th>name</th>";
-echo "<th>status</th>";
-
-$counter = $offset + 1;
-while ($row = $result->fetch_array()) {
- echo "<tr>";
- echo "<td>{$counter}.</td>";
- echo "<td>{$row['engineid']}</td>";
- echo "<td>{$row['gameid']}</td>";
- echo "<td>{$row['extra']}</td>";
- echo "<td>{$row['platform']}</td>";
- echo "<td>{$row['language']}</td>";
- echo "<td>{$row['name']}</td>";
- echo "<td>{$row['status']}</td>";
- echo "</tr>";
-
- $counter++;
-}
-echo "</table>";
-
-// Navigation elements
-echo "<div class=pagination>";
-if ($page > 1)
- echo "<a href={$filename}>â®â®</a>";
-if ($page - 2 > 1)
- echo "<div class=more>...</div>";
-
-for ($i = $page - 2; $i <= $page + 2; $i++) {
- if ($i >= 1 && $i <= $num_of_pages)
- if ($i == $page)
- echo sprintf("<a class=active href=%s?page=%d>%d</a>", $filename, $i, $i);
- else
- echo sprintf("<a href=%s?page=%d>%d</a>", $filename, $i, $i);
-}
-
-if ($page + 2 < $num_of_pages)
- echo "<div class=more>...</div>";
-if ($page < $num_of_pages)
- echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>";
+JOIN fileset ON game.id = fileset.game";
-echo "</div>";
+create_page($filename, 25, $count_query, $select_query);
?>
diff --git a/logs.php b/logs.php
index ca395b5..b28bb18 100644
--- a/logs.php
+++ b/logs.php
@@ -1,55 +1,11 @@
<?php
-$filename = "logs.php";
-
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
-$servername = "localhost";
-$username = $mysql_cred["username"];
-$password = $mysql_cred["password"];
-$dbname = "integrity";
-
-// Create connection
-mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
-$conn = new mysqli($servername, $username, $password);
-$conn->set_charset('utf8mb4');
-$conn->autocommit(FALSE);
-
-// Check connection
-if ($conn->connect_errno) {
- die("Connect failed: " . $conn->connect_error);
-}
-
-$conn->query("USE " . $dbname);
+require "pagination.php";
-if (!isset($_GET['page'])) {
- $page = 1;
-}
-else {
- $page = $_GET['page'];
-}
-
-$results_per_page = 25;
-$offset = ($page - 1) * $results_per_page;
-$num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_array()[0];
-$num_of_pages = ceil($num_of_results / $results_per_page);
-
-$query = sprintf("SELECT * FROM log LIMIT %d OFFSET %d", $results_per_page, $offset);
-$result = $conn->query($query);
-
-echo "<ol start=\"" . $offset + 1 . "\">";
-while ($row = $result->fetch_array()) {
- echo "<li>";
- echo sprintf("%s (%s, user %s) %s<br/>", $row["timestamp"], $row["category"], $row["user"], $row["text"]);
- echo "</li>";
-}
-echo "</ol>";
+$filename = "logs.php";
+$count_query = "SELECT COUNT(id) FROM log";
+$select_query = "SELECT `timestamp`, category, user, `text`
+FROM log";
-if ($page > 1) {
- echo sprintf('<a href=%s>first</a>', $filename);
- echo sprintf('<a href=%s?page=%d>prev</a>', $filename, $page - 1);
-}
-if ($page < $num_of_pages) {
- echo sprintf('<a href=%s?page=%d>next</a>', $filename, $page + 1);
- echo sprintf('<a href=%s?page=%d>last</a>', $filename, $num_of_pages);
-}
+create_page($filename, 25, $count_query, $select_query);
?>
diff --git a/pagination.php b/pagination.php
new file mode 100644
index 0000000..1a97327
--- /dev/null
+++ b/pagination.php
@@ -0,0 +1,86 @@
+<?php
+$stylesheet = "style.css";
+echo "<link rel='stylesheet' href='{$stylesheet}'>";
+
+function create_page($filename, $results_per_page, $count_query, $select_query) {
+ $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+ $servername = "localhost";
+ $username = $mysql_cred["username"];
+ $password = $mysql_cred["password"];
+ $dbname = "integrity";
+
+ // Create connection
+ mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+ $conn = new mysqli($servername, $username, $password);
+ $conn->set_charset('utf8mb4');
+ $conn->autocommit(FALSE);
+
+ // Check connection
+ if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+ }
+
+ $conn->query("USE " . $dbname);
+
+ if (!isset($_GET['page'])) {
+ $page = 1;
+ }
+ else {
+ $page = $_GET['page'];
+ }
+
+ $offset = ($page - 1) * $results_per_page;
+ $num_of_results = $conn->query($count_query)->fetch_array()[0];
+ $num_of_pages = ceil($num_of_results / $results_per_page);
+
+ $query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
+ $result = $conn->query($query);
+
+ echo "<table>";
+ echo "<th/>"; // Numbering column
+
+ $counter = $offset + 1;
+ while ($row = $result->fetch_assoc()) {
+ if ($counter == $offset + 1) { // If it is the first run of the loop
+ foreach ($row as $key => $value) {
+ echo "<th>{$key}</th>";
+ }
+ }
+
+ echo "<tr>";
+ echo "<td>{$counter}.</td>";
+ foreach ($row as $key => $value) {
+ echo "<td>{$value}</td>";
+ }
+ echo "</tr>";
+
+ $counter++;
+ }
+
+ echo "</table>";
+
+ // Navigation elements
+ echo "<div class=pagination>";
+ if ($page > 1)
+ echo "<a href={$filename}>â®â®</a>";
+ if ($page - 2 > 1)
+ echo "<div class=more>...</div>";
+
+ for ($i = $page - 2; $i <= $page + 2; $i++) {
+ if ($i >= 1 && $i <= $num_of_pages)
+ if ($i == $page)
+ echo sprintf("<a class=active href=%s?page=%d>%d</a>", $filename, $i, $i);
+ else
+ echo sprintf("<a href=%s?page=%d>%d</a>", $filename, $i, $i);
+ }
+
+ if ($page + 2 < $num_of_pages)
+ echo "<div class=more>...</div>";
+ if ($page < $num_of_pages)
+ echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>";
+
+ echo "</div>";
+
+}
+?>
+
Commit: ef25db858ec2e068ef95e370efd988c1fa34df5d
https://github.com/scummvm/scummvm-sites/commit/ef25db858ec2e068ef95e370efd988c1fa34df5d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T17:09:54+05:30
Commit Message:
INTEGRITY: Add .gitignore, README and server config
Changed paths:
A .gitignore
A apache2-config/gamesdb.sev.zone.conf
README.md
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..016f5c8
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.dat
+mysql_config.json
diff --git a/README.md b/README.md
index ac56195..4eb2581 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,12 @@
# ScummVM File Integrity Check (GSoC 2023)
-This repository contains the server-side code for the upcoming file integrity check for game datafiles. This repository is part of the Google Summer of Code 2023 program.
\ No newline at end of file
+This repository contains the server-side code for the upcoming file integrity check for game datafiles. This repository is part of the Google Summer of Code 2023 program.
+
+This website needs a `mysql_config.json` in the root to run, in the form:
+
+ {
+ "username": "<your username>",
+ "password": "<your password>"
+ }
+
+The apache2 .conf file is located under `apache2-conf/`.
diff --git a/apache2-config/gamesdb.sev.zone.conf b/apache2-config/gamesdb.sev.zone.conf
new file mode 100644
index 0000000..b6178f5
--- /dev/null
+++ b/apache2-config/gamesdb.sev.zone.conf
@@ -0,0 +1,15 @@
+<VirtualHost *:80>
+ ServerName gamesdb.sev.zone
+ ServerAlias www.gamesdb.sev.zone
+ ServerAdmin webmaster at localhost
+ DocumentRoot /var/www/vhosts.d/gamesdb.sev.zone/htdocs/
+ ErrorLog /var/www/vhosts.d/gamesdb.sev.zone/logs/error.log
+ CustomLog /var/www/vhosts.d/gamesdb.sev.zone/logs/access.log combined
+ <Directory /var/www/vhosts.d/gamesdb.sev.zone/htdocs>
+ php_admin_value open_basedir "/var/www/vhosts.d/gamesdb.sev.zone/"
+ </Directory>
+ <Files "mysql_config.json">
+ Order allow,deny
+ Deny from all
+ </Files>
+</VirtualHost>
Commit: 4ea3eb034775965098a66e6900f60221891ab5cd
https://github.com/scummvm/scummvm-sites/commit/4ea3eb034775965098a66e6900f60221891ab5cd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T17:34:34+05:30
Commit Message:
INTEGRITY: Updated table styling
Changed paths:
style.css
diff --git a/style.css b/style.css
index 98861a0..67c078e 100644
--- a/style.css
+++ b/style.css
@@ -1,11 +1,25 @@
-table, th, td {
- border: 1px solid;
+:root {
+ --primary-color: #27b5e8;
}
-th, td {
+td, th {
+ border: 1px solid #ddd;
padding-inline: 5px;
}
+tr:nth-child(even) {background-color: #f2f2f2;}
+tr {background-color: white;}
+
+tr:hover {background-color: #ddd;}
+
+th {
+ padding-top: 5px;
+ padding-bottom: 5px;
+ text-align: left;
+ background-color: var(--primary-color);
+ color: white;
+}
+
.pagination {
display: inline-block;
align-self: center;
@@ -22,14 +36,16 @@ th, td {
float: left;
padding: 8px 16px;
text-decoration: none;
- transition: background-color .3s;
+ transition: background-color 0.3s;
border: 1px solid #ddd;
}
.pagination a.active {
- background-color: #27b5e8;
color: white;
- border: 1px solid #27b5e8;
+ background-color: var(--primary-color);
+ border: 1px solid var(--primary-color);
}
-.pagination a:hover:not(.active) {background-color: #ddd;}
+.pagination a:hover:not(.active) {
+ background-color: #ddd;
+}
Commit: 203138d1befba5b39b693fb40ba76f3dfdf18927
https://github.com/scummvm/scummvm-sites/commit/203138d1befba5b39b693fb40ba76f3dfdf18927
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T19:10:32+05:30
Commit Message:
INTEGRITY: Add newlines to HTML tags
This is so the source sent to the browser is a bit
more readable
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 1a97327..0e4840c 100644
--- a/pagination.php
+++ b/pagination.php
@@ -1,6 +1,6 @@
<?php
$stylesheet = "style.css";
-echo "<link rel='stylesheet' href='{$stylesheet}'>";
+echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
function create_page($filename, $results_per_page, $count_query, $select_query) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
@@ -36,50 +36,50 @@ function create_page($filename, $results_per_page, $count_query, $select_query)
$query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
$result = $conn->query($query);
- echo "<table>";
- echo "<th/>"; // Numbering column
+ echo "<table>\n";
+ echo "<th/>\n"; // Numbering column
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
foreach ($row as $key => $value) {
- echo "<th>{$key}</th>";
+ echo "<th>{$key}</th>\n";
}
}
- echo "<tr>";
- echo "<td>{$counter}.</td>";
+ echo "<tr>\n";
+ echo "<td>{$counter}.</td>\n";
foreach ($row as $key => $value) {
- echo "<td>{$value}</td>";
+ echo "<td>{$value}</td>\n";
}
- echo "</tr>";
+ echo "</tr>\n";
$counter++;
}
- echo "</table>";
+ echo "</table>\n";
// Navigation elements
- echo "<div class=pagination>";
+ echo "<div class=pagination>\n";
if ($page > 1)
- echo "<a href={$filename}>â®â®</a>";
+ echo "<a href={$filename}>â®â®</a>\n";
if ($page - 2 > 1)
- echo "<div class=more>...</div>";
+ echo "<div class=more>...</div>\n";
for ($i = $page - 2; $i <= $page + 2; $i++) {
if ($i >= 1 && $i <= $num_of_pages)
if ($i == $page)
- echo sprintf("<a class=active href=%s?page=%d>%d</a>", $filename, $i, $i);
+ echo sprintf("<a class=active href=%s?page=%d>%d</a>\n", $filename, $i, $i);
else
- echo sprintf("<a href=%s?page=%d>%d</a>", $filename, $i, $i);
+ echo sprintf("<a href=%s?page=%d>%d</a>\n", $filename, $i, $i);
}
if ($page + 2 < $num_of_pages)
- echo "<div class=more>...</div>";
+ echo "<div class=more>...</div>\n";
if ($page < $num_of_pages)
- echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>";
+ echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>\n";
- echo "</div>";
+ echo "</div>\n";
}
?>
Commit: 0b8a823c7323fd32c147a074d1cfca435f8db27a
https://github.com/scummvm/scummvm-sites/commit/0b8a823c7323fd32c147a074d1cfca435f8db27a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-04T21:40:13+05:30
Commit Message:
INTEGRITY: Add filters to games_list.php
Add dropdown form to select the column and value to filter by. Currently
only one filter can be selected at a time, and only columns from the
game table can be filtered.
Changed paths:
games_list.php
pagination.php
diff --git a/games_list.php b/games_list.php
index d074483..46012c1 100644
--- a/games_list.php
+++ b/games_list.php
@@ -3,11 +3,21 @@ require "pagination.php";
$filename = "games_list.php";
$count_query = "SELECT COUNT(id) FROM game";
-$select_query = "SELECT engineid, gameid, extra, platform, language, game.name, status
+$select_query = "SELECT engineid, gameid, extra, platform, language, game.name as 'game name', status
FROM game
JOIN engine ON engine.id = game.engine
JOIN fileset ON game.id = fileset.game";
-create_page($filename, 25, $count_query, $select_query);
+$filters = array(
+ "engineid" => "engine",
+ "gameid" => "game",
+ "extra" => "game",
+ "platform" => "game",
+ "language" => "game",
+ "name" => "game",
+ "status" => "fileset"
+);
+
+create_page($filename, 25, $count_query, $select_query, $filters);
?>
diff --git a/pagination.php b/pagination.php
index 0e4840c..2a1ad61 100644
--- a/pagination.php
+++ b/pagination.php
@@ -2,7 +2,7 @@
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
-function create_page($filename, $results_per_page, $count_query, $select_query) {
+function create_page($filename, $results_per_page, $count_query, $select_query, $filters) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
$username = $mysql_cred["username"];
@@ -30,26 +30,60 @@ function create_page($filename, $results_per_page, $count_query, $select_query)
}
$offset = ($page - 1) * $results_per_page;
- $num_of_results = $conn->query($count_query)->fetch_array()[0];
+ if (isset($_GET['column']) && isset($_GET['value'])) {
+ $column = $_GET['column'];
+ $value = $_GET['value'];
+ $num_of_results = $conn->query("{$count_query} WHERE {$column} = '{$value}'")->fetch_array()[0];
+ $query = "{$select_query} WHERE {$column} = '{$value}' LIMIT {$results_per_page} OFFSET {$offset}";
+ }
+ else {
+ $num_of_results = $conn->query($count_query)->fetch_array()[0];
+ $query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
+ }
$num_of_pages = ceil($num_of_results / $results_per_page);
-
- $query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
$result = $conn->query($query);
+
+ // Create filter dropdown
+ echo "<div class='filter'>\n";
+ echo "<form name='filter' method='GET'>\n";
+ echo "Filter: ";
+
+ echo "<select name='column'>\n";
+ foreach (array_keys($filters) as $key) {
+ echo "<option>{$key}</option>\n";
+ }
+ echo "</select>\n";
+
+ if (isset($_GET['column'])) {
+ echo "<select name='value'>\n";
+ $res = $conn->query("SELECT DISTINCT({$_GET['column']}) FROM {$filters[$_GET['column']]}");
+ while ($value = $res->fetch_array()[0]) {
+ echo "<option>{$value}</option>\n";
+ }
+ echo "</select>\n";
+ }
+
+ echo "<input type='submit' name='submit' value='Select' />\n";
+ echo "</form>\n";
+ echo "</div>\n";
+
+
+ // Table
echo "<table>\n";
echo "<th/>\n"; // Numbering column
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
- foreach ($row as $key => $value) {
+ foreach (array_keys($row) as $key) {
echo "<th>{$key}</th>\n";
}
}
echo "<tr>\n";
echo "<td>{$counter}.</td>\n";
- foreach ($row as $key => $value) {
+ foreach (array_values($row) as $value) {
echo "<td>{$value}</td>\n";
}
echo "</tr>\n";
@@ -59,6 +93,7 @@ function create_page($filename, $results_per_page, $count_query, $select_query)
echo "</table>\n";
+
// Navigation elements
echo "<div class=pagination>\n";
if ($page > 1)
Commit: ad8c9f2379104781e1a7b8aa72483e07b8563607
https://github.com/scummvm/scummvm-sites/commit/ad8c9f2379104781e1a7b8aa72483e07b8563607
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T09:49:59+05:30
Commit Message:
INTEGRITY: Filter now works for all columns
Changed paths:
games_list.php
pagination.php
diff --git a/games_list.php b/games_list.php
index 46012c1..dba3f44 100644
--- a/games_list.php
+++ b/games_list.php
@@ -2,12 +2,13 @@
require "pagination.php";
$filename = "games_list.php";
-$count_query = "SELECT COUNT(id) FROM game";
+$records_table = "game";
$select_query = "SELECT engineid, gameid, extra, platform, language, game.name as 'game name', status
FROM game
JOIN engine ON engine.id = game.engine
JOIN fileset ON game.id = fileset.game";
+// Filter column => table
$filters = array(
"engineid" => "engine",
"gameid" => "game",
@@ -18,6 +19,6 @@ $filters = array(
"status" => "fileset"
);
-create_page($filename, 25, $count_query, $select_query, $filters);
+create_page($filename, 25, $records_table, $select_query, $filters);
?>
diff --git a/pagination.php b/pagination.php
index 2a1ad61..c0f2da2 100644
--- a/pagination.php
+++ b/pagination.php
@@ -2,7 +2,7 @@
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
-function create_page($filename, $results_per_page, $count_query, $select_query, $filters) {
+function create_page($filename, $results_per_page, $records_table, $select_query, $filters) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = "localhost";
$username = $mysql_cred["username"];
@@ -32,12 +32,13 @@ function create_page($filename, $results_per_page, $count_query, $select_query,
$offset = ($page - 1) * $results_per_page;
if (isset($_GET['column']) && isset($_GET['value'])) {
$column = $_GET['column'];
- $value = $_GET['value'];
- $num_of_results = $conn->query("{$count_query} WHERE {$column} = '{$value}'")->fetch_array()[0];
+ $value = mysqli_real_escape_string($conn, $_GET['value']);
+ $num_of_results = $conn->query(
+ "SELECT COUNT(id) FROM {$filters[$column]} WHERE {$column} = '{$value}'")->fetch_array()[0];
$query = "{$select_query} WHERE {$column} = '{$value}' LIMIT {$results_per_page} OFFSET {$offset}";
}
else {
- $num_of_results = $conn->query($count_query)->fetch_array()[0];
+ $num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
$query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
}
$num_of_pages = ceil($num_of_results / $results_per_page);
Commit: aac2dd095c6530f400b9904a58156e4780e5572b
https://github.com/scummvm/scummvm-sites/commit/aac2dd095c6530f400b9904a58156e4780e5572b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T09:53:57+05:30
Commit Message:
INTEGRITY: Add filters to logs.php
Changed paths:
logs.php
diff --git a/logs.php b/logs.php
index b28bb18..e618191 100644
--- a/logs.php
+++ b/logs.php
@@ -2,10 +2,15 @@
require "pagination.php";
$filename = "logs.php";
-$count_query = "SELECT COUNT(id) FROM log";
+$records_table = "log";
$select_query = "SELECT `timestamp`, category, user, `text`
FROM log";
-create_page($filename, 25, $count_query, $select_query);
+$filters = array(
+ "category" => "log",
+ "user" => "log"
+);
+
+create_page($filename, 25, $records_table, $select_query, $filters);
?>
Commit: 8b0acd255c6920b07c31aada2fd12a16acfa49ce
https://github.com/scummvm/scummvm-sites/commit/8b0acd255c6920b07c31aada2fd12a16acfa49ce
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T10:08:35+05:30
Commit Message:
INTEGRITY: Replace value dropdown with textbox
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index c0f2da2..075890a 100644
--- a/pagination.php
+++ b/pagination.php
@@ -55,15 +55,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<option>{$key}</option>\n";
}
echo "</select>\n";
-
- if (isset($_GET['column'])) {
- echo "<select name='value'>\n";
- $res = $conn->query("SELECT DISTINCT({$_GET['column']}) FROM {$filters[$_GET['column']]}");
- while ($value = $res->fetch_array()[0]) {
- echo "<option>{$value}</option>\n";
- }
- echo "</select>\n";
- }
+ echo "<input type='text' name='value' placeholder='Value'>\n";
echo "<input type='submit' name='submit' value='Select' />\n";
echo "</form>\n";
Commit: b18651a2842503c472bbb3f62106266bbfbd4ba4
https://github.com/scummvm/scummvm-sites/commit/b18651a2842503c472bbb3f62106266bbfbd4ba4
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T15:55:37+05:30
Commit Message:
INTEGRITY: Update apache to deny access to more files
Changed paths:
apache2-config/gamesdb.sev.zone.conf
diff --git a/apache2-config/gamesdb.sev.zone.conf b/apache2-config/gamesdb.sev.zone.conf
index b6178f5..9578a31 100644
--- a/apache2-config/gamesdb.sev.zone.conf
+++ b/apache2-config/gamesdb.sev.zone.conf
@@ -12,4 +12,12 @@
Order allow,deny
Deny from all
</Files>
+ <Files "schema.php">
+ Order allow,deny
+ Deny from all
+ </Files>
+ <Files "dat_parser.php">
+ Order allow,deny
+ Deny from all
+ </Files>
</VirtualHost>
Commit: 24ebd9953f6e9c21ec77df752c0bb3fc3ce74ad6
https://github.com/scummvm/scummvm-sites/commit/24ebd9953f6e9c21ec77df752c0bb3fc3ce74ad6
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T15:58:54+05:30
Commit Message:
INTEGRITY: Update references of CLI to scan
Changed paths:
compute_hash.py
dat_parser.php
diff --git a/compute_hash.py b/compute_hash.py
index 97b6f50..c12cd6e 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -98,7 +98,7 @@ def create_dat_file(hash_of_dirs, path):
# Header
file.writelines([
"scummvm (\n",
- f"\tauthor cli\n",
+ f"\tauthor scan\n",
f"\tversion {script_version}\n",
")\n\n"
])
diff --git a/dat_parser.php b/dat_parser.php
index 3d40f97..1709558 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -416,7 +416,7 @@ function db_insert($data_arr) {
/**
* Author can be:
* scummvm -> Detection Entries
- * scanner -> CLI tool
+ * scanner -> CLI scanner tool in python
* _anything else_ -> DAT file
*/
$author = $header["author"];
@@ -426,16 +426,14 @@ function db_insert($data_arr) {
* src can be:
* detection -> Detection entries (source of truth)
* user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
- * cli -> Submitted by cli/scanner, unmatched
+ * scan -> Submitted by cli/scanner, unmatched
* dat -> Submitted by DAT, unmatched
* partialmatch -> Submitted by DAT, matched
* fullmatch -> Submitted by cli/scanner, matched
*/
$src = "";
- if ($author == "cli")
- $src = "scan";
- elseif ($author == "scummvm")
- $src = "scummvm";
+ if ($author == "scan" || $author == "scummvm")
+ $src = $author;
else
$src = "dat";
@@ -530,7 +528,7 @@ function populate_matching_games() {
$category_text = "Matched, state '" . $status . "'";
$log_text = sprintf("Matched game %s: %s-%s-%s variant %s from %s",
$matched_game["engineid"], $matched_game["gameid"], $matched_game["platform"],
- $matched_game["language"], $matched_game["key"], $matched_game["src"]);
+ $matched_game["language"], $matched_game["key"], $fileset[0][2]);
// Updating the fileset.game value to be $matched_game["id"]
$query = sprintf("UPDATE fileset
Commit: 13ce72e697e132499f1cf52b10e102346dafbecd
https://github.com/scummvm/scummvm-sites/commit/13ce72e697e132499f1cf52b10e102346dafbecd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T16:16:25+05:30
Commit Message:
INTEGRITY: Add CLI options to dat_parser.php
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 1709558..b5d6db9 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -545,11 +545,15 @@ function populate_matching_games() {
}
}
-// echo "<pre>";
-// db_insert(parse_dat("scummvm_detection_entries.dat"));
-// db_insert(parse_dat("drascula-1.0.dat"));
-// db_insert(parse_dat("Downloads.dat"));
-// populate_matching_games();
-// echo "</pre>";
+// Process command line args
+if ($index = array_search("--upload", $argv)) {
+ $filepath = $argv[$index + 1];
+ print($filepath);
+ db_insert(parse_dat($filepath));
+}
+
+if (in_array("--match", $argv)) {
+ populate_matching_games();
+}
?>
Commit: 097236bd6f998e7963290250c482ba16a5b23a53
https://github.com/scummvm/scummvm-sites/commit/097236bd6f998e7963290250c482ba16a5b23a53
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T17:16:55+05:30
Commit Message:
INTEGRITY: Add support for uploading multiple DATs
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index b5d6db9..d7fff54 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -547,13 +547,17 @@ function populate_matching_games() {
// Process command line args
if ($index = array_search("--upload", $argv)) {
- $filepath = $argv[$index + 1];
- print($filepath);
- db_insert(parse_dat($filepath));
+ foreach (array_slice($argv, $index + 1) as $filepath) {
+ if ($filepath == "--match")
+ continue;
+
+ db_insert(parse_dat($filepath));
+ }
}
if (in_array("--match", $argv)) {
populate_matching_games();
}
+
?>
Commit: 1995d95f932f097c653f122cf9c638d5418ea20b
https://github.com/scummvm/scummvm-sites/commit/1995d95f932f097c653f122cf9c638d5418ea20b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T18:32:52+05:30
Commit Message:
INTEGRITY: Clamp $page value to (1, $num_of_pages)
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 075890a..5c28ae9 100644
--- a/pagination.php
+++ b/pagination.php
@@ -22,26 +22,32 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$conn->query("USE " . $dbname);
+ if (isset($_GET['column']) && isset($_GET['value'])) {
+ $column = $_GET['column'];
+ $value = mysqli_real_escape_string($conn, $_GET['value']);
+ $num_of_results = $conn->query(
+ "SELECT COUNT(id) FROM {$filters[$column]} WHERE {$column} = '{$value}'")->fetch_array()[0];
+ }
+ else {
+ $num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
+ }
+ $num_of_pages = ceil($num_of_results / $results_per_page);
+
if (!isset($_GET['page'])) {
$page = 1;
}
else {
- $page = $_GET['page'];
+ $page = max(1, min($_GET['page'], $num_of_pages));
}
$offset = ($page - 1) * $results_per_page;
if (isset($_GET['column']) && isset($_GET['value'])) {
- $column = $_GET['column'];
- $value = mysqli_real_escape_string($conn, $_GET['value']);
- $num_of_results = $conn->query(
- "SELECT COUNT(id) FROM {$filters[$column]} WHERE {$column} = '{$value}'")->fetch_array()[0];
$query = "{$select_query} WHERE {$column} = '{$value}' LIMIT {$results_per_page} OFFSET {$offset}";
}
else {
- $num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
$query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
}
- $num_of_pages = ceil($num_of_results / $results_per_page);
+
$result = $conn->query($query);
Commit: f734004b41fbd231ff02370a706c131df89ad019
https://github.com/scummvm/scummvm-sites/commit/f734004b41fbd231ff02370a706c131df89ad019
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T19:00:18+05:30
Commit Message:
INTEGRITY: Preserve query variables across pages
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 5c28ae9..b2c583a 100644
--- a/pagination.php
+++ b/pagination.php
@@ -100,18 +100,28 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($page - 2 > 1)
echo "<div class=more>...</div>\n";
+ // Preserve GET variables
+ $vars = "";
+ foreach ($_GET as $key => $value) {
+ if ($key == 'page')
+ continue;
+ $vars .= "&{$key}={$value}";
+ }
+
for ($i = $page - 2; $i <= $page + 2; $i++) {
- if ($i >= 1 && $i <= $num_of_pages)
+ if ($i >= 1 && $i <= $num_of_pages) {
+
if ($i == $page)
- echo sprintf("<a class=active href=%s?page=%d>%d</a>\n", $filename, $i, $i);
+ echo sprintf("<a class=active href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
else
- echo sprintf("<a href=%s?page=%d>%d</a>\n", $filename, $i, $i);
+ echo sprintf("<a href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
+ }
}
if ($page + 2 < $num_of_pages)
echo "<div class=more>...</div>\n";
if ($page < $num_of_pages)
- echo "<a href={$filename}?page={$num_of_pages}>â¯â¯</a>\n";
+ echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
echo "</div>\n";
Commit: 89e371f0a091a072e80c7f94f6d3658c7020e74a
https://github.com/scummvm/scummvm-sites/commit/89e371f0a091a072e80c7f94f6d3658c7020e74a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T19:16:23+05:30
Commit Message:
INTEGRITY: Add text field to manually type page no
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index b2c583a..f1c08d5 100644
--- a/pagination.php
+++ b/pagination.php
@@ -94,6 +94,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// Navigation elements
+ echo "<form method='GET'>\n";
echo "<div class=pagination>\n";
if ($page > 1)
echo "<a href={$filename}>â®â®</a>\n";
@@ -123,6 +124,10 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($page < $num_of_pages)
echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
+ echo "<input type='text' name='page'>\n";
+ echo "<input type='submit' name='submit' value='Submit'>\n";
+ echo "</form>\n";
+
echo "</div>\n";
}
Commit: fbe6c1f29c58129337171fb77c04606576b1884a
https://github.com/scummvm/scummvm-sites/commit/fbe6c1f29c58129337171fb77c04606576b1884a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T22:18:25+05:30
Commit Message:
INTEGRITY: Fetch db, servername from mysql config
Changed paths:
README.md
dat_parser.php
pagination.php
schema.php
seeds.php
diff --git a/README.md b/README.md
index 4eb2581..3fd7df8 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,10 @@ This repository contains the server-side code for the upcoming file integrity ch
This website needs a `mysql_config.json` in the root to run, in the form:
{
+ "servername": "<your servername>",
"username": "<your username>",
- "password": "<your password>"
+ "password": "<your password>",
+ "dbname": "<your db name>"
}
The apache2 .conf file is located under `apache2-conf/`.
diff --git a/dat_parser.php b/dat_parser.php
index d7fff54..1c4ff0d 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -368,10 +368,10 @@ function merge_filesets($detection_id, $dat_id) {
*/
function db_connect() {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
- $servername = "localhost";
+ $servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
- $dbname = "integrity";
+ $dbname = $mysql_cred["dbname"];
// Create connection
mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
diff --git a/pagination.php b/pagination.php
index f1c08d5..86b2862 100644
--- a/pagination.php
+++ b/pagination.php
@@ -4,10 +4,10 @@ echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
function create_page($filename, $results_per_page, $records_table, $select_query, $filters) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
- $servername = "localhost";
+ $servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
- $dbname = "integrity";
+ $dbname = $mysql_cred["dbname"];
// Create connection
mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
diff --git a/schema.php b/schema.php
index a2e5313..33f0b03 100644
--- a/schema.php
+++ b/schema.php
@@ -1,10 +1,10 @@
<?php
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
-$servername = "localhost";
+$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
-$dbname = "integrity";
+$dbname = $mysql_cred["dbname"];
// Create connection
$conn = new mysqli($servername, $username, $password);
diff --git a/seeds.php b/seeds.php
index c9d62e6..858580b 100644
--- a/seeds.php
+++ b/seeds.php
@@ -1,10 +1,10 @@
<?php
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
-$servername = "localhost";
+$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
-$dbname = "integrity";
+$dbname = $mysql_cred["dbname"];
// Create connection
$conn = new mysqli($servername, $username, $password);
Commit: 090028d3c5e3df5ca80bb1253da22cd972afc750
https://github.com/scummvm/scummvm-sites/commit/090028d3c5e3df5ca80bb1253da22cd972afc750
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T22:31:25+05:30
Commit Message:
INTEGRITY: Update matching
Detection id is now retained, newer id is deleted.
Update fileset history on match
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 1c4ff0d..112af7c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -343,11 +343,6 @@ function merge_filesets($detection_id, $dat_id) {
$conn->query(sprintf("DELETE FROM file
WHERE checksum = '%s' LIMIT 1", $checksum));
- // Move any remaining files (duplicates) to the new fileset
- $conn->query(sprintf("UPDATE file
- SET fileset = %d
- WHERE fileset = %d AND checksum = '%s'", $dat_id, $detection_id, $checksum));
-
// Mark files present in the detection entries
$conn->query(sprintf("UPDATE file
JOIN filechecksum ON filechecksum.file = file.id
@@ -356,11 +351,19 @@ function merge_filesets($detection_id, $dat_id) {
checktype = '%s'
WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
$checksize, $checktype, $dat_id, $checksum));
+
+ // Move files from the new fileset to the original fileset
+ $conn->query(sprintf("UPDATE file
+ SET fileset = %d
+ WHERE fileset = %d AND checksum = '%s'", $detection_id, $dat_id, $checksum));
}
// Add fileset pair to history ($dat_id is the new fileset for $detection_id)
- $conn->query(sprintf("INSERT INTO history (fileset, oldfileset)
- VALUES (%d, %d)", $dat_id, $detection_id));
+ $conn->query("INSERT INTO history (fileset, oldfileset) VALUES ({$detection_id}, {$dat_id})");
+ $conn->query("UPDATE history SET fileset = {$detection_id} WHERE fileset = {$dat_id}");
+
+ if (!$conn->commit())
+ echo "Error matching filesets";
}
/**
Commit: 8c22dac99ab4f360eacbae2416898eb459f268d2
https://github.com/scummvm/scummvm-sites/commit/8c22dac99ab4f360eacbae2416898eb459f268d2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-05T23:24:30+05:30
Commit Message:
INTEGRITY: Create fileset webpage
- One line to display the history of the filset as part of the URL
- Add timestamp column to history table
Changed paths:
A fileset.php
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 112af7c..2bef639 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -359,7 +359,8 @@ function merge_filesets($detection_id, $dat_id) {
}
// Add fileset pair to history ($dat_id is the new fileset for $detection_id)
- $conn->query("INSERT INTO history (fileset, oldfileset) VALUES ({$detection_id}, {$dat_id})");
+ $conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
+ VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $detection_id, $dat_id));
$conn->query("UPDATE history SET fileset = {$detection_id} WHERE fileset = {$dat_id}");
if (!$conn->commit())
@@ -529,9 +530,9 @@ function populate_matching_games() {
}, $matched_game);
$category_text = "Matched, state '" . $status . "'";
- $log_text = sprintf("Matched game %s: %s-%s-%s variant %s from %s",
- $matched_game["engineid"], $matched_game["gameid"], $matched_game["platform"],
- $matched_game["language"], $matched_game["key"], $fileset[0][2]);
+ $log_text = "Matched game {$matched_game['engineid']}:
+ {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']}
+ variant {$matched_game['key']} from {$fileset[0][2]}.";
// Updating the fileset.game value to be $matched_game["id"]
$query = sprintf("UPDATE fileset
diff --git a/fileset.php b/fileset.php
new file mode 100644
index 0000000..b7266ef
--- /dev/null
+++ b/fileset.php
@@ -0,0 +1,46 @@
+<?php
+
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+$servername = $mysql_cred["servername"];
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
+$dbname = $mysql_cred["dbname"];
+
+// Create connection
+mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+$conn = new mysqli($servername, $username, $password);
+$conn->set_charset('utf8mb4');
+$conn->autocommit(FALSE);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+$conn->query("USE " . $dbname);
+
+if (!isset($_GET['id'])) {
+ $id = 1;
+}
+else {
+ $max_id = $conn->query("SELECT MAX(id) FROM fileset")->fetch_array()[0];
+ $id = max(1, min($_GET['id'], $max_id));
+}
+
+// Display history
+$res = $conn->query("SELECT `timestamp`, oldfileset
+FROM history WHERE fileset = {$id}
+ORDER BY `timestamp`")->fetch_all();
+
+if (count($res) == 0) {
+ echo "Fileset has no history.";
+}
+else {
+ echo "This fileset was merged with the following filesets in chronological order: ";
+ foreach ($res as $history) {
+ echo "{$history[1]}, ";
+ }
+}
+
+?>
+
diff --git a/schema.php b/schema.php
index 33f0b03..98fec57 100644
--- a/schema.php
+++ b/schema.php
@@ -152,6 +152,7 @@ else {
// Create history table
$table = "CREATE TABLE IF NOT EXISTS history (
id INT AUTO_INCREMENT PRIMARY KEY,
+ `timestamp` TIMESTAMP NOT NULL,
fileset INT NOT NULL,
oldfileset INT NOT NULL
)";
Commit: 6de87602e83584dbaf1ed4580874d42b12d964e1
https://github.com/scummvm/scummvm-sites/commit/6de87602e83584dbaf1ed4580874d42b12d964e1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T12:22:39+05:30
Commit Message:
INTEGRITY: Fix - Duplicates deleted while matching
Also delete original fileset on successfully merging filesets
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 2bef639..d180f78 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -330,7 +330,7 @@ function insert_filechecksum($file, $checktype, $conn) {
function merge_filesets($detection_id, $dat_id) {
$conn = db_connect();
- $detection_files = $conn->query(sprintf("SELECT filechecksum.checksum, checksize, checktype
+ $detection_files = $conn->query(sprintf("SELECT DISTINCT(filechecksum.checksum), checksize, checktype
FROM filechecksum JOIN file on file.id = filechecksum.file
WHERE fileset = '%d'", $detection_id))->fetch_all();
@@ -341,7 +341,7 @@ function merge_filesets($detection_id, $dat_id) {
// Delete original detection entry so newly matched fileset is the only fileset for game
$conn->query(sprintf("DELETE FROM file
- WHERE checksum = '%s' LIMIT 1", $checksum));
+ WHERE checksum = '%s' AND fileset = %d LIMIT 1", $checksum, $detection_id));
// Mark files present in the detection entries
$conn->query(sprintf("UPDATE file
@@ -351,20 +351,23 @@ function merge_filesets($detection_id, $dat_id) {
checktype = '%s'
WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
$checksize, $checktype, $dat_id, $checksum));
-
- // Move files from the new fileset to the original fileset
- $conn->query(sprintf("UPDATE file
- SET fileset = %d
- WHERE fileset = %d AND checksum = '%s'", $detection_id, $dat_id, $checksum));
}
+ // Move files from the original fileset to the new fileset
+ $conn->query(sprintf("UPDATE file
+ SET fileset = %d
+ WHERE fileset = %d", $dat_id, $detection_id));
+
// Add fileset pair to history ($dat_id is the new fileset for $detection_id)
$conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
- VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $detection_id, $dat_id));
- $conn->query("UPDATE history SET fileset = {$detection_id} WHERE fileset = {$dat_id}");
+ VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id));
+ $conn->query("UPDATE history SET fileset = {$dat_id} WHERE fileset = {$detection_id}");
+
+ // Delete original fileset
+ $conn->query("DELETE FROM fileset WHERE id = {$detection_id}");
if (!$conn->commit())
- echo "Error matching filesets";
+ echo "Error merging filesets";
}
/**
Commit: 91a947a0e6e8c9f28b2c6509bbde9edacd15b300
https://github.com/scummvm/scummvm-sites/commit/91a947a0e6e8c9f28b2c6509bbde9edacd15b300
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T13:26:33+05:30
Commit Message:
INTEGRITY: Remove borders from table
Changed paths:
style.css
diff --git a/style.css b/style.css
index 67c078e..eddf942 100644
--- a/style.css
+++ b/style.css
@@ -3,7 +3,6 @@
}
td, th {
- border: 1px solid #ddd;
padding-inline: 5px;
}
Commit: 74c204f63bd4b9d25a057bc6b6a8668d9c497731
https://github.com/scummvm/scummvm-sites/commit/74c204f63bd4b9d25a057bc6b6a8668d9c497731
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T13:44:54+05:30
Commit Message:
INTEGRITY: Add fileset data to fileset.php
- Display data of the fileset in question
- If the fileset was matched, display info of the matched game
- Display a list of all files in the given fileset
- Display history of the fileset id in chronological order
- Redirect old id's to the page of the new id
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index b7266ef..a44b46a 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,4 +1,6 @@
<?php
+$stylesheet = "style.css";
+echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
@@ -25,21 +27,89 @@ if (!isset($_GET['id'])) {
else {
$max_id = $conn->query("SELECT MAX(id) FROM fileset")->fetch_array()[0];
$id = max(1, min($_GET['id'], $max_id));
+ if ($conn->query("SELECT id FROM fileset WHERE id = {$id}")->num_rows == 0)
+ $id = $conn->query("SELECT fileset FROM history WHERE oldfileset = {$id}")->fetch_array()[0];
}
// Display history
-$res = $conn->query("SELECT `timestamp`, oldfileset
+$history = $conn->query("SELECT `timestamp`, oldfileset
FROM history WHERE fileset = {$id}
-ORDER BY `timestamp`")->fetch_all();
+ORDER BY `timestamp`");
-if (count($res) == 0) {
+// Display fileset details
+$result = $conn->query("SELECT * FROM fileset WHERE id = {$id}")->fetch_assoc();
+
+echo "<h3>Fileset details</h3>";
+echo "<table>\n";
+if ($result['game']) {
+ $temp = $conn->query("SELECT game.name as 'game name', engineid, gameid, extra, platform, language
+FROM fileset JOIN game ON game.id = fileset.game JOIN engine ON engine.id = game.engine
+WHERE fileset.id = {$id}");
+ $result = array_merge($result, $temp->fetch_assoc());
+}
+else {
+ unset($result['key']);
+ unset($result['status']);
+}
+
+foreach (array_keys($result) as $column) {
+ if ($column == 'id' || $column == 'game')
+ continue;
+
+ echo "<th>{$column}</th>\n";
+}
+
+echo "<tr>\n";
+foreach ($result as $column => $value) {
+ if ($column == 'id' || $column == 'game')
+ continue;
+
+ echo "<td>{$value}</td>";
+}
+echo "</tr>\n";
+echo "</table>\n";
+
+echo "<h3>Files in the fileset</h3>";
+echo "<table>\n";
+$res = $conn->query("SELECT * from file WHERE fileset = {$id}");
+$first_row = true;
+while ($row = $res->fetch_assoc()) {
+ if ($first_row) { // If it is the first run
+ foreach (array_keys($row) as $column) {
+ if ($column == 'id')
+ continue;
+
+ echo "<th>{$column}</th>\n";
+ }
+ $first_row = false;
+ }
+
+ echo "<tr>\n";
+ foreach ($row as $column => $value) {
+ if ($column == 'id')
+ continue;
+
+ echo "<td>{$value}</td>";
+ }
+ echo "</tr>\n";
+}
+echo "</table>\n";
+
+if ($history->num_rows == 0) {
echo "Fileset has no history.";
}
else {
- echo "This fileset was merged with the following filesets in chronological order: ";
- foreach ($res as $history) {
- echo "{$history[1]}, ";
+ echo "<h3>Fileset history</h3>";
+ echo "<table>\n";
+ echo "<th>Old ID</th>";
+ echo "<th>Changed on</th>";
+ while ($row = $history->fetch_assoc()) {
+ echo "<tr>\n";
+ echo "<td>{$row['oldfileset']}</td>\n";
+ echo "<td>{$row['timestamp']}</td>\n";
+ echo "</tr>\n";
}
+ echo "</table>\n";
}
?>
Commit: f5410f5a45044645657c5d7968b0c24973f5ee8a
https://github.com/scummvm/scummvm-sites/commit/f5410f5a45044645657c5d7968b0c24973f5ee8a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T17:13:18+05:30
Commit Message:
INTEGRITY: Fileset files displayed with pagination
Changed paths:
fileset.php
pagination.php
diff --git a/fileset.php b/fileset.php
index a44b46a..d438b78 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,4 +1,7 @@
<?php
+require 'pagination.php';
+
+$filename = 'fileset.php';
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
@@ -31,11 +34,11 @@ else {
$id = $conn->query("SELECT fileset FROM history WHERE oldfileset = {$id}")->fetch_array()[0];
}
-// Display history
$history = $conn->query("SELECT `timestamp`, oldfileset
FROM history WHERE fileset = {$id}
ORDER BY `timestamp`");
+
// Display fileset details
$result = $conn->query("SELECT * FROM fileset WHERE id = {$id}")->fetch_assoc();
@@ -70,31 +73,11 @@ echo "</tr>\n";
echo "</table>\n";
echo "<h3>Files in the fileset</h3>";
-echo "<table>\n";
-$res = $conn->query("SELECT * from file WHERE fileset = {$id}");
-$first_row = true;
-while ($row = $res->fetch_assoc()) {
- if ($first_row) { // If it is the first run
- foreach (array_keys($row) as $column) {
- if ($column == 'id')
- continue;
-
- echo "<th>{$column}</th>\n";
- }
- $first_row = false;
- }
+create_page($filename, 15, "file WHERE fileset = {$id}",
+ "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}");
- echo "<tr>\n";
- foreach ($row as $column => $value) {
- if ($column == 'id')
- continue;
-
- echo "<td>{$value}</td>";
- }
- echo "</tr>\n";
-}
-echo "</table>\n";
+// Display history
if ($history->num_rows == 0) {
echo "Fileset has no history.";
}
diff --git a/pagination.php b/pagination.php
index 86b2862..8e926ae 100644
--- a/pagination.php
+++ b/pagination.php
@@ -2,7 +2,7 @@
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
-function create_page($filename, $results_per_page, $records_table, $select_query, $filters) {
+function create_page($filename, $results_per_page, $records_table, $select_query, $filters = array()) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -52,20 +52,22 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// Create filter dropdown
- echo "<div class='filter'>\n";
- echo "<form name='filter' method='GET'>\n";
- echo "Filter: ";
+ if ($filters) {
+ echo "<div class='filter'>\n";
+ echo "<form name='filter' method='GET'>\n";
+ echo "Filter: ";
+
+ echo "<select name='column'>\n";
+ foreach (array_keys($filters) as $key) {
+ echo "<option>{$key}</option>\n";
+ }
+ echo "</select>\n";
+ echo "<input type='text' name='value' placeholder='Value'>\n";
- echo "<select name='column'>\n";
- foreach (array_keys($filters) as $key) {
- echo "<option>{$key}</option>\n";
+ echo "<input type='submit' name='submit' value='Select' />\n";
+ echo "</form>\n";
+ echo "</div>\n";
}
- echo "</select>\n";
- echo "<input type='text' name='value' placeholder='Value'>\n";
-
- echo "<input type='submit' name='submit' value='Select' />\n";
- echo "</form>\n";
- echo "</div>\n";
// Table
@@ -92,22 +94,22 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "</table>\n";
+ // Preserve GET variables
+ $vars = "";
+ foreach ($_GET as $key => $value) {
+ if ($key == 'page')
+ continue;
+ $vars .= "&{$key}={$value}";
+ }
// Navigation elements
echo "<form method='GET'>\n";
echo "<div class=pagination>\n";
if ($page > 1)
- echo "<a href={$filename}>â®â®</a>\n";
+ echo "<a href={$filename}?{$vars}>â®â®</a>\n";
if ($page - 2 > 1)
echo "<div class=more>...</div>\n";
- // Preserve GET variables
- $vars = "";
- foreach ($_GET as $key => $value) {
- if ($key == 'page')
- continue;
- $vars .= "&{$key}={$value}";
- }
for ($i = $page - 2; $i <= $page + 2; $i++) {
if ($i >= 1 && $i <= $num_of_pages) {
@@ -124,7 +126,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($page < $num_of_pages)
echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
- echo "<input type='text' name='page'>\n";
+ echo "<input type='text' name='page' placeholder='Page Number'>\n";
echo "<input type='submit' name='submit' value='Submit'>\n";
echo "</form>\n";
Commit: 7dc630bdc276f351d95c9d923d8ef4f6f49c9a0d
https://github.com/scummvm/scummvm-sites/commit/7dc630bdc276f351d95c9d923d8ef4f6f49c9a0d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T18:46:22+05:30
Commit Message:
INTEGRITY: Add fileset to log text
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index d180f78..49fe79c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -478,9 +478,10 @@ function db_insert($data_arr) {
calc_key($fileset["rom"])));
}
}
- $category_text = "Uploaded from " . $src . ", state '" . $status . "'";
- $log_text = sprintf("Loaded DAT file, filename \"%s\", size %d, author \"%s\", version %s",
- $filepath, filesize($filepath), $author, $version);
+ $category_text = "Uploaded from {$src}";
+ $log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
+ State '%s'. Fileset:@fileset_last.",
+ $filepath, filesize($filepath), $author, $version, $status);
if (!$conn->commit())
echo "Inserting failed<br/>";
@@ -532,15 +533,16 @@ function populate_matching_games() {
return (is_null($val)) ? "NULL" : $val;
}, $matched_game);
- $category_text = "Matched, state '" . $status . "'";
+ $category_text = "Matched from {$fileset[0][2]}";
$log_text = "Matched game {$matched_game['engineid']}:
{$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']}
- variant {$matched_game['key']} from {$fileset[0][2]}.";
+ variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}.";
// Updating the fileset.game value to be $matched_game["id"]
$query = sprintf("UPDATE fileset
SET game = %d, status = '%s', `key` = '%s'
WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
+
merge_filesets($matched_game["fileset"], $fileset[0][0]);
if ($conn->query($query))
Commit: 2e39b3a53844fd048b93cf93e237b35d87a08b20
https://github.com/scummvm/scummvm-sites/commit/2e39b3a53844fd048b93cf93e237b35d87a08b20
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T18:49:26+05:30
Commit Message:
INTEGRITY: Add hyperlinks to fileset in logs.php
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 8e926ae..0c73529 100644
--- a/pagination.php
+++ b/pagination.php
@@ -85,6 +85,12 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<tr>\n";
echo "<td>{$counter}.</td>\n";
foreach (array_values($row) as $value) {
+ // Add hyperlink to filesets
+ $matches = array();
+ if (preg_match("/Fileset:(\d+)/", $value, $matches, PREG_OFFSET_CAPTURE)) {
+ $value = substr($value, 0, $matches[0][1]) . "<a href='fileset.php?id={$matches[1][0]}'>{$matches[0][0]}</a>";
+ }
+
echo "<td>{$value}</td>\n";
}
echo "</tr>\n";
Commit: 3a4f001e6fcfcee7f670262312f2e63f35287585
https://github.com/scummvm/scummvm-sites/commit/3a4f001e6fcfcee7f670262312f2e63f35287585
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T19:36:18+05:30
Commit Message:
INTEGRITY: Add username to logs
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 49fe79c..9e91f64 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -485,8 +485,10 @@ function db_insert($data_arr) {
if (!$conn->commit())
echo "Inserting failed<br/>";
- else
- create_log(mysqli_real_escape_string($conn, $category_text), "unknown", mysqli_real_escape_string($conn, $log_text)); // FIXME: User name is "unknown"
+ else {
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
+ }
}
function populate_matching_games() {
@@ -545,9 +547,11 @@ function populate_matching_games() {
merge_filesets($matched_game["fileset"], $fileset[0][0]);
- if ($conn->query($query))
- create_log(mysqli_real_escape_string($conn, $category_text), "unknown",
- mysqli_real_escape_string($conn, $log_text)); // FIXME: user name is unknown
+ if ($conn->query($query)) {
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user,
+ mysqli_real_escape_string($conn, $log_text));
+ }
if (!$conn->commit())
echo "Updating matched games failed<br/>";
Commit: 9b0623370b612ba816686755afec8e864a20fbd6
https://github.com/scummvm/scummvm-sites/commit/9b0623370b612ba816686755afec8e864a20fbd6
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T20:59:16+05:30
Commit Message:
INTEGRITY: Add logs to fileset history
- Log ID hyperlink is displayed in fileset.php history table
- Hyperlink points to logs.php page containing the relevant log
Changed paths:
dat_parser.php
fileset.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 9e91f64..c41aceb 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -361,6 +361,8 @@ function merge_filesets($detection_id, $dat_id) {
// Add fileset pair to history ($dat_id is the new fileset for $detection_id)
$conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id));
+ $history_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
+
$conn->query("UPDATE history SET fileset = {$dat_id} WHERE fileset = {$detection_id}");
// Delete original fileset
@@ -368,6 +370,8 @@ function merge_filesets($detection_id, $dat_id) {
if (!$conn->commit())
echo "Error merging filesets";
+
+ return $history_last;
}
/**
@@ -404,8 +408,12 @@ function create_log($category, $user, $text) {
$conn = db_connect();
$conn->query(sprintf("INSERT INTO log (`timestamp`, category, user, `text`)
VALUES (FROM_UNIXTIME(%d), '%s', '%s', '%s')", time(), $category, $user, $text));
+ $log_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
+
if (!$conn->commit())
echo "Creating log failed<br/>";
+
+ return $log_last;
}
/**
@@ -480,8 +488,9 @@ function db_insert($data_arr) {
}
$category_text = "Uploaded from {$src}";
$log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
- State '%s'. Fileset:@fileset_last.",
- $filepath, filesize($filepath), $author, $version, $status);
+ State '%s'. Fileset:%d.",
+ $filepath, filesize($filepath), $author, $version, $status,
+ $conn->query("SELECT @fileset_last")->fetch_array()[0]);
if (!$conn->commit())
echo "Inserting failed<br/>";
@@ -545,12 +554,15 @@ function populate_matching_games() {
SET game = %d, status = '%s', `key` = '%s'
WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
- merge_filesets($matched_game["fileset"], $fileset[0][0]);
+ $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
if ($conn->query($query)) {
$user = 'cli:' . get_current_user();
- create_log(mysqli_real_escape_string($conn, $category_text), $user,
+ $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user,
mysqli_real_escape_string($conn, $log_text));
+
+ // Add log id to the history table
+ $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
}
if (!$conn->commit())
diff --git a/fileset.php b/fileset.php
index d438b78..43c398b 100644
--- a/fileset.php
+++ b/fileset.php
@@ -5,6 +5,11 @@ $filename = 'fileset.php';
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
+function get_log_page($log_id) {
+ $records_per_page = 25; // FIXME: Fetch this directly from logs.php
+ return intdiv($log_id, $records_per_page) + 1;
+}
+
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -34,7 +39,7 @@ else {
$id = $conn->query("SELECT fileset FROM history WHERE oldfileset = {$id}")->fetch_array()[0];
}
-$history = $conn->query("SELECT `timestamp`, oldfileset
+$history = $conn->query("SELECT `timestamp`, oldfileset, log
FROM history WHERE fileset = {$id}
ORDER BY `timestamp`");
@@ -86,10 +91,13 @@ else {
echo "<table>\n";
echo "<th>Old ID</th>";
echo "<th>Changed on</th>";
+ echo "<th>Log ID</th>";
while ($row = $history->fetch_assoc()) {
+ $log_page = get_log_page($row['log']);
echo "<tr>\n";
echo "<td>{$row['oldfileset']}</td>\n";
echo "<td>{$row['timestamp']}</td>\n";
+ echo "<td><a href='logs.php?page={$log_page}'>{$row['log']}</a></td>\n";
echo "</tr>\n";
}
echo "</table>\n";
diff --git a/schema.php b/schema.php
index 98fec57..10a1ee3 100644
--- a/schema.php
+++ b/schema.php
@@ -154,7 +154,8 @@ $table = "CREATE TABLE IF NOT EXISTS history (
id INT AUTO_INCREMENT PRIMARY KEY,
`timestamp` TIMESTAMP NOT NULL,
fileset INT NOT NULL,
- oldfileset INT NOT NULL
+ oldfileset INT NOT NULL,
+ log INT
)";
if ($conn->query($table) === TRUE) {
Commit: 8294ef1a6ce43556ef50316f161911bbee767b7c
https://github.com/scummvm/scummvm-sites/commit/8294ef1a6ce43556ef50316f161911bbee767b7c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-06T22:24:58+05:30
Commit Message:
INTEGRITY: Add Developer Actions to fileset page
- Add button to fileset.php to mark fileset for deletion
- Button click calls JS function that sends AJAX request, which runs
PHP code when the request is received
- JS code lives in js_functions.js
- jQuery file called by URL
- Add delete column to filset to store if the fileset was marked for
deletion
Changed paths:
A js_functions.js
fileset.php
schema.php
diff --git a/fileset.php b/fileset.php
index 43c398b..b3eb93a 100644
--- a/fileset.php
+++ b/fileset.php
@@ -2,8 +2,12 @@
require 'pagination.php';
$filename = 'fileset.php';
-$stylesheet = "style.css";
+$stylesheet = 'style.css';
+$jquery_file = 'https://code.jquery.com/jquery-3.7.0.min.js';
+$js_file = 'js_functions.js';
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
+echo "<script type='text/javascript' src='{$jquery_file}'></script>\n";
+echo "<script type='text/javascript' src='{$js_file}'></script>\n";
function get_log_page($log_id) {
$records_per_page = 25; // FIXME: Fetch this directly from logs.php
@@ -45,6 +49,8 @@ ORDER BY `timestamp`");
// Display fileset details
+echo "<h2><u>Fileset: {$id}</u></h2>";
+
$result = $conn->query("SELECT * FROM fileset WHERE id = {$id}")->fetch_assoc();
echo "<h3>Fileset details</h3>";
@@ -58,6 +64,7 @@ WHERE fileset.id = {$id}");
else {
unset($result['key']);
unset($result['status']);
+ unset($result['delete']);
}
foreach (array_keys($result) as $column) {
@@ -82,12 +89,24 @@ create_page($filename, 15, "file WHERE fileset = {$id}",
"SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}");
+// Dev Actions
+echo "<h3>Developer Actions</h3>";
+echo "<button id='delete-button' type='button' onclick='delete_id({$id})'>Mark Fileset for Deletion</button>";
+
+if (isset($_POST['delete'])) {
+ $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$_POST['delete']}");
+ $conn->commit();
+}
+
+echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
+
+
// Display history
+echo "<h3>Fileset history</h3>";
if ($history->num_rows == 0) {
- echo "Fileset has no history.";
+ echo "<p>Fileset has no history.</p>";
}
else {
- echo "<h3>Fileset history</h3>";
echo "<table>\n";
echo "<th>Old ID</th>";
echo "<th>Changed on</th>";
diff --git a/js_functions.js b/js_functions.js
new file mode 100644
index 0000000..8bfb7ca
--- /dev/null
+++ b/js_functions.js
@@ -0,0 +1,17 @@
+function delete_id(value) {
+ $("#delete-confirm").slideDown();
+
+ $.ajax({
+ url: "fileset.php",
+ type: "post",
+ dataType: "json",
+ data: {
+ delete: value,
+ },
+ });
+}
+
+$(document).ready(function () {
+ $(".hidden").hide();
+ $("#delete-button").one("click", delete_id);
+});
diff --git a/schema.php b/schema.php
index 10a1ee3..8abebc8 100644
--- a/schema.php
+++ b/schema.php
@@ -69,6 +69,7 @@ $table = "CREATE TABLE IF NOT EXISTS fileset (
status VARCHAR(20),
src VARCHAR(20),
`key` VARCHAR(64),
+ `delete` BOOLEAN DEFAULT FALSE NOT NULL,
FOREIGN KEY (game) REFERENCES game(id)
)";
Commit: 62ff29f34f0ad6a05509d316166022fa6aab9431
https://github.com/scummvm/scummvm-sites/commit/62ff29f34f0ad6a05509d316166022fa6aab9431
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-07T08:49:11+05:30
Commit Message:
INTEGRITY: Add explicit ordering to pagination.php
Changed paths:
fileset.php
games_list.php
logs.php
pagination.php
diff --git a/fileset.php b/fileset.php
index b3eb93a..e95c92a 100644
--- a/fileset.php
+++ b/fileset.php
@@ -86,7 +86,7 @@ echo "</table>\n";
echo "<h3>Files in the fileset</h3>";
create_page($filename, 15, "file WHERE fileset = {$id}",
- "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}");
+ "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}", "ORDER BY name");
// Dev Actions
diff --git a/games_list.php b/games_list.php
index dba3f44..e69ebbd 100644
--- a/games_list.php
+++ b/games_list.php
@@ -7,6 +7,7 @@ $select_query = "SELECT engineid, gameid, extra, platform, language, game.name a
FROM game
JOIN engine ON engine.id = game.engine
JOIN fileset ON game.id = fileset.game";
+$order = "ORDER BY gameid";
// Filter column => table
$filters = array(
@@ -19,6 +20,6 @@ $filters = array(
"status" => "fileset"
);
-create_page($filename, 25, $records_table, $select_query, $filters);
+create_page($filename, 25, $records_table, $select_query, $order, $filters);
?>
diff --git a/logs.php b/logs.php
index e618191..3649fbc 100644
--- a/logs.php
+++ b/logs.php
@@ -3,14 +3,15 @@ require "pagination.php";
$filename = "logs.php";
$records_table = "log";
-$select_query = "SELECT `timestamp`, category, user, `text`
+$select_query = "SELECT id, `timestamp`, category, user, `text`
FROM log";
+$order = "ORDER BY `timestamp` DESC";
$filters = array(
"category" => "log",
"user" => "log"
);
-create_page($filename, 25, $records_table, $select_query, $filters);
+create_page($filename, 25, $records_table, $select_query, $order, $filters);
?>
diff --git a/pagination.php b/pagination.php
index 0c73529..ba0b43b 100644
--- a/pagination.php
+++ b/pagination.php
@@ -2,7 +2,7 @@
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
-function create_page($filename, $results_per_page, $records_table, $select_query, $filters = array()) {
+function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array()) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -42,10 +42,10 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$offset = ($page - 1) * $results_per_page;
if (isset($_GET['column']) && isset($_GET['value'])) {
- $query = "{$select_query} WHERE {$column} = '{$value}' LIMIT {$results_per_page} OFFSET {$offset}";
+ $query = "{$select_query} WHERE {$column} = '{$value}' {$order} LIMIT {$results_per_page} OFFSET {$offset}";
}
else {
- $query = "{$select_query} LIMIT {$results_per_page} OFFSET {$offset}";
+ $query = "{$select_query} {$order} LIMIT {$results_per_page} OFFSET {$offset}";
}
$result = $conn->query($query);
Commit: af096c5646226893101805048eb6c92ac9931a74
https://github.com/scummvm/scummvm-sites/commit/af096c5646226893101805048eb6c92ac9931a74
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-07T09:58:03+05:30
Commit Message:
INTEGRITY: Clamp min fileset id
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index e95c92a..0bc4290 100644
--- a/fileset.php
+++ b/fileset.php
@@ -33,12 +33,13 @@ if ($conn->connect_errno) {
$conn->query("USE " . $dbname);
+$min_id = $conn->query("SELECT MIN(id) FROM fileset")->fetch_array()[0];
if (!isset($_GET['id'])) {
- $id = 1;
+ $id = $min_id;
}
else {
$max_id = $conn->query("SELECT MAX(id) FROM fileset")->fetch_array()[0];
- $id = max(1, min($_GET['id'], $max_id));
+ $id = max($min_id, min($_GET['id'], $max_id));
if ($conn->query("SELECT id FROM fileset WHERE id = {$id}")->num_rows == 0)
$id = $conn->query("SELECT fileset FROM history WHERE oldfileset = {$id}")->fetch_array()[0];
}
Commit: 1d6eb1d74cf29f91ca26c9c44776233c4cd03183
https://github.com/scummvm/scummvm-sites/commit/1d6eb1d74cf29f91ca26c9c44776233c4cd03183
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-07T10:03:19+05:30
Commit Message:
INTEGRITY: Update styling
- Set default font families
- Set styling for buttons, submit button, text input
- Page number input placeholder text changed to 'Page No'
Changed paths:
pagination.php
style.css
diff --git a/pagination.php b/pagination.php
index ba0b43b..b42692a 100644
--- a/pagination.php
+++ b/pagination.php
@@ -132,7 +132,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($page < $num_of_pages)
echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
- echo "<input type='text' name='page' placeholder='Page Number'>\n";
+ echo "<input type='text' name='page' placeholder='Page No'>\n";
echo "<input type='submit' name='submit' value='Submit'>\n";
echo "</form>\n";
diff --git a/style.css b/style.css
index eddf942..5bc4af9 100644
--- a/style.css
+++ b/style.css
@@ -1,5 +1,6 @@
:root {
--primary-color: #27b5e8;
+ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
}
td, th {
@@ -19,6 +20,49 @@ th {
color: white;
}
+button {
+ color: white;
+ padding: 6px 12px;
+ border-radius: 10px;
+ transition: background-color 0.1s;
+ background-color: var(--primary-color);
+ border: 1px solid var(--primary-color);
+}
+
+button:hover {
+ background-color: #29afe0;
+}
+button:active {
+ background-color: #1a95c2;
+}
+
+input[type=submit] {
+ color: white;
+ padding: 6px 12px;
+ border-radius: 10px;
+ transition: background-color 0.1s;
+ background-color: var(--primary-color);
+ border: 1px solid var(--primary-color);
+}
+
+input[type=submit]:hover {
+ background-color: #29afe0;
+}
+input[type=submit]:active {
+ background-color: #1a95c2;
+}
+
+input[type=text], select {
+ width: 25%;
+ height: 38px;
+ padding: 6px 12px;
+ margin: 0px 8px;
+ display: inline-block;
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ box-sizing: border-box;
+}
+
.pagination {
display: inline-block;
align-self: center;
Commit: e66c454f2e4924d957c98e83867298bfe3cff600
https://github.com/scummvm/scummvm-sites/commit/e66c454f2e4924d957c98e83867298bfe3cff600
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-07T11:45:32+05:30
Commit Message:
INTEGRITY: Display page navigation only on >1 page
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index b42692a..4d2e823 100644
--- a/pagination.php
+++ b/pagination.php
@@ -109,34 +109,36 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
// Navigation elements
- echo "<form method='GET'>\n";
- echo "<div class=pagination>\n";
- if ($page > 1)
- echo "<a href={$filename}?{$vars}>â®â®</a>\n";
- if ($page - 2 > 1)
- echo "<div class=more>...</div>\n";
-
-
- for ($i = $page - 2; $i <= $page + 2; $i++) {
- if ($i >= 1 && $i <= $num_of_pages) {
-
- if ($i == $page)
- echo sprintf("<a class=active href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
- else
- echo sprintf("<a href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
+ if ($num_of_pages > 1) {
+ echo "<form method='GET'>\n";
+ echo "<div class=pagination>\n";
+ if ($page > 1)
+ echo "<a href={$filename}?{$vars}>â®â®</a>\n";
+ if ($page - 2 > 1)
+ echo "<div class=more>...</div>\n";
+
+
+ for ($i = $page - 2; $i <= $page + 2; $i++) {
+ if ($i >= 1 && $i <= $num_of_pages) {
+
+ if ($i == $page)
+ echo sprintf("<a class=active href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
+ else
+ echo sprintf("<a href=%s?page=%d%s>%d</a>\n", $filename, $i, $vars, $i);
+ }
}
- }
- if ($page + 2 < $num_of_pages)
- echo "<div class=more>...</div>\n";
- if ($page < $num_of_pages)
- echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
+ if ($page + 2 < $num_of_pages)
+ echo "<div class=more>...</div>\n";
+ if ($page < $num_of_pages)
+ echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
- echo "<input type='text' name='page' placeholder='Page No'>\n";
- echo "<input type='submit' name='submit' value='Submit'>\n";
- echo "</form>\n";
-
- echo "</div>\n";
+ echo "<input type='text' name='page' placeholder='Page No'>\n";
+ echo "<input type='submit' name='submit' value='Submit'>\n";
+ echo "</div>\n";
+
+ echo "</form>\n";
+ }
}
?>
Commit: 4e7b47efb01b01be2a3c2329848418e075b4d16f
https://github.com/scummvm/scummvm-sites/commit/4e7b47efb01b01be2a3c2329848418e075b4d16f
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-07T14:36:07+05:30
Commit Message:
INTEGRITY: Add priority order for matching
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index c41aceb..f546ce8 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -179,12 +179,22 @@ function get_checksum_props($checktype, $checksum) {
return array($checksize, $checktype, $checksum);
}
+/**
+ * Return fileset statuses that can be merged with set of given status
+ * eg: scan and dat -> detection
+ * fullmatch -> partialmatch, detection
+ */
+function status_to_match($status) {
+ $order = array("detection", "dat", "scan", "partialmatch", "fullmatch");
+ return array_slice($order, 0, array_search($status, $order));
+}
+
/**
* Detects games based on the file descriptions in $dat_arr
* Compares the files with those in the detection entries table
* $game_files consists of both the game ( ) and resources ( ) parts
*/
-function find_matching_game($game_files) {
+function find_matching_game($game_files, $status) {
$matching_games = array(); // All matching games
$matching_filesets = array(); // All filesets containing one file from $game_files
$matches_count = 0; // Number of files with a matching detection entry
@@ -197,7 +207,9 @@ function find_matching_game($game_files) {
$records = $conn->query(sprintf("SELECT file.fileset
FROM filechecksum
JOIN file ON filechecksum.file = file.id
- WHERE filechecksum.checksum = '%s' AND file.detection = TRUE", $checksum));
+ JOIN fileset ON fileset.id = file.fileset
+ WHERE filechecksum.checksum = '%s' AND file.detection = TRUE AND
+ fileset.status IN ('%s')", $checksum, implode("', '", status_to_match($status))));
$records = $records->fetch_all();
// If file is not part of detection entries, skip it
@@ -506,7 +518,8 @@ function populate_matching_games() {
// Getting unmatched filesets
$unmatched_filesets = array();
- $unmatched_files = $conn->query(sprintf("SELECT fileset.id, filechecksum.checksum, fileset.src from fileset
+ $unmatched_files = $conn->query(sprintf("SELECT fileset.id, filechecksum.checksum, src, status
+ FROM fileset
JOIN file ON file.fileset = fileset.id
JOIN filechecksum ON file.id = filechecksum.file
WHERE fileset.game IS NULL"));
@@ -525,7 +538,7 @@ function populate_matching_games() {
}
foreach ($unmatched_filesets as $fileset) {
- $matching_games = find_matching_game($fileset);
+ $matching_games = find_matching_game($fileset, $fileset[0][3]);
if (count($matching_games) != 1) // If there is no match/non-unique match
continue;
Commit: 314da9ce624289be50956155263cf4a681177f5c
https://github.com/scummvm/scummvm-sites/commit/314da9ce624289be50956155263cf4a681177f5c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-11T12:02:39+05:30
Commit Message:
INTEGRITY: Add support for MacBinary checksums
Calculate checksums for resource and data forks and attach appropriate
prefixes to the checksums
TODO:
Implement check for whether a file is actually a MacBinary, and run
the new code only check returns True
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index c12cd6e..16317b2 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -1,6 +1,7 @@
import hashlib
import os
import argparse
+import struct
script_version = "0.1"
@@ -21,55 +22,99 @@ def get_dirs_at_depth(directory, depth):
yield root
-def checksum(filepath, alg, size):
- """ Returns checksum value of file using a specific algoritm """
- with open(filepath, "rb") as file:
- # Will contain 5 elements:
- # - Full size checksum
- # - Checksum of first 5000B
- # - Checksum of first 1MB
- # - Checksum of last 5000B (tail)
- # - Checksum of first *size* bytes
- hashes = []
-
- if alg == "md5":
- hashes = [hashlib.md5() for _ in range(5)]
- elif alg == "sha1":
- hashes = [hashlib.sha1() for _ in range(5)]
- elif alg == "sha256":
- hashes = [hashlib.sha256() for _ in range(5)]
-
- # Read file in 8MB chunks for full checksum
- for byte_block in iter(lambda: file.read(8 * 1024 * 1024), b""):
- hashes[0].update(byte_block)
-
- # First 5000B
- file.seek(0)
- hashes[1].update(file.read(5000))
+def is_macbin(filepath):
+ return True
+
+
+def macbin_get_resfork(file_byte_stream):
+ (datalen,) = struct.unpack(">I", file_byte_stream[0x53:0x57])
+ return file_byte_stream[0x80 + datalen:]
- # First 1MB
- file.seek(0)
- hashes[2].update(file.read(1024 * 1024))
- # Last 5000B
- if filesize(filepath) >= 5000:
- file.seek(-5000, os.SEEK_END)
- hashes[3].update(file.read())
- else:
- hashes[3] = hashes[0]
+def macbin_get_datafork(file_byte_stream):
+ (datalen,) = struct.unpack(">I", file_byte_stream[0x53:0x57])
+ return file_byte_stream[0x80: 0x80 + datalen]
- # Custom size; may be None
- # Size is in bytes
- # Reads entire required size at once, inefficient for large sizes
- if size and size <= filesize(filepath):
- file.seek(0)
- hashes[4].update(file.read(size))
- else:
- hashes[4] = None
- hashes = [h.hexdigest() for h in hashes if h]
- hashes[3] = 't:' + hashes[3] # Add tail prefix
- return hashes
+def file_checksum(filepath, alg, size):
+ if not is_macbin(filepath):
+ with open(filepath, "rb") as file:
+ return checksum(file, alg, size, filepath)
+
+ # If the file is a MacBinary
+ with open(filepath, "rb") as file:
+ res = []
+
+ file = macbin_get_resfork(file)
+ prefix = 'r'
+
+ if len(file):
+ for h in checksum(file, alg, size, filepath):
+ if ':' not in h:
+ res.append(f"{prefix}:{h}")
+ else:
+ res.append(f"{prefix}{h}") # If the checksum is like "t:..."
+
+ file = macbin_get_datafork(file)
+ prefix = 'd'
+
+ for h in checksum(file, alg, size, filepath):
+ if ':' not in h:
+ res.append(f"{prefix}:{h}")
+ else:
+ res.append(f"{prefix}{h}") # If the checksum is like "t:..."
+
+ return res
+
+
+def checksum(file, alg, size, filepath):
+ """ Returns checksum value of file buffer using a specific algoritm """
+ # Will contain 5 elements:
+ # - Full size checksum
+ # - Checksum of first 5000B
+ # - Checksum of first 1MB
+ # - Checksum of last 5000B (tail)
+ # - Checksum of first *size* bytes
+ hashes = []
+
+ if alg == "md5":
+ hashes = [hashlib.md5() for _ in range(5)]
+ elif alg == "sha1":
+ hashes = [hashlib.sha1() for _ in range(5)]
+ elif alg == "sha256":
+ hashes = [hashlib.sha256() for _ in range(5)]
+
+ # Read file in 8MB chunks for full checksum
+ for byte_block in iter(lambda: file.read(8 * 1024 * 1024), b""):
+ hashes[0].update(byte_block)
+
+ # First 5000B
+ file.seek(0)
+ hashes[1].update(file.read(5000))
+
+ # First 1MB
+ file.seek(0)
+ hashes[2].update(file.read(1024 * 1024))
+
+ # Last 5000B
+ if filesize(filepath) >= 5000:
+ file.seek(-5000, os.SEEK_END)
+ hashes[3].update(file.read())
+ else:
+ hashes[3] = hashes[0]
+
+ # Custom size; may be None
+ # Size is in bytes
+ # Reads entire required size at once, inefficient for large sizes
+ if size and size <= filesize(filepath):
+ file.seek(0)
+ hashes[4].update(file.read(size))
+ else:
+ hashes[4] = None
+
+ hashes = [h.hexdigest() for h in hashes if h]
+ hashes[3] = 't:' + hashes[3] # Add tail prefix
+ return hashes
def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
@@ -85,7 +130,7 @@ def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
files.extend([os.path.join(root, f) for f in contents])
for file in files:
- hash_of_dir[os.path.relpath(file, directory)] = (checksum(
+ hash_of_dir[os.path.relpath(file, directory)] = (file_checksum(
file, alg, size), filesize(file))
res.append(hash_of_dir)
Commit: bf8fd580326d3e29ad73db95e836bcbe7098846c
https://github.com/scummvm/scummvm-sites/commit/bf8fd580326d3e29ad73db95e836bcbe7098846c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-11T12:07:34+05:30
Commit Message:
INTEGRITY: Add support for custom checksum size
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 16317b2..3096147 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -131,14 +131,14 @@ def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
for file in files:
hash_of_dir[os.path.relpath(file, directory)] = (file_checksum(
- file, alg, size), filesize(file))
+ file, alg, size), alg, filesize(file))
res.append(hash_of_dir)
return res
-def create_dat_file(hash_of_dirs, path):
+def create_dat_file(hash_of_dirs, path, checksum_size=0):
with open(f"{os.path.basename(path)}.dat", "w") as file:
# Header
file.writelines([
@@ -151,9 +151,12 @@ def create_dat_file(hash_of_dirs, path):
# Game files
for hash_of_dir in hash_of_dirs:
file.write("game (\n")
- for filename, (hashes, filesize) in hash_of_dir.items():
+ for filename, (hashes, alg, filesize) in hash_of_dir.items():
# Only works for MD5s, ignores optional extra size
- data = f"name \"{filename}\" size {filesize} md5 {hashes[0]} md5-5000 {hashes[1]} md5-1M {hashes[2]} md5-5000-t {hashes[3]}"
+ data = f"name \"{filename}\" size {filesize} {alg} {hashes[0]} {alg}-5000 {hashes[1]} {alg}-1M {hashes[2]} {alg}-5000-t {hashes[3]}"
+ if checksum_size:
+ data += f" {alg}-{checksum_size} {hashes[4]}"
+
file.write(f"\trom ( {data} )\n")
file.write(")\n\n")
@@ -170,4 +173,5 @@ path = os.path.abspath(args.directory) if args.directory else os.getcwd()
depth = int(args.depth) if args.depth else 0
checksum_size = int(args.size) if args.size else 0
-create_dat_file(compute_hash_of_dirs(path, depth, checksum_size), path)
+create_dat_file(compute_hash_of_dirs(
+ path, depth, checksum_size), path, checksum_size)
Commit: 1075289c4554973ddab7684ccd19e9f37c9201b6
https://github.com/scummvm/scummvm-sites/commit/1075289c4554973ddab7684ccd19e9f37c9201b6
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-11T18:37:10+05:30
Commit Message:
INTEGRITY: Extend MacBinary support
- Create separate control flow for MacBinaries
- Skip checksums if forks are empty
- Extract checksum formatting to its own function
- Update dat_parser to handle cases where checksize is not int
Changed paths:
compute_hash.py
dat_parser.php
diff --git a/compute_hash.py b/compute_hash.py
index 3096147..c60fd05 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -27,25 +27,46 @@ def is_macbin(filepath):
def macbin_get_resfork(file_byte_stream):
+ if not file_byte_stream:
+ return file_byte_stream
+
(datalen,) = struct.unpack(">I", file_byte_stream[0x53:0x57])
return file_byte_stream[0x80 + datalen:]
def macbin_get_datafork(file_byte_stream):
+ if not file_byte_stream:
+ return file_byte_stream
+
(datalen,) = struct.unpack(">I", file_byte_stream[0x53:0x57])
return file_byte_stream[0x80: 0x80 + datalen]
+def create_checksum_pairs(hashes, alg, size, prefix=None):
+ res = []
+
+ keys = [f"{alg}", f"{alg}-5000", f"{alg}-1M", f"{alg}-5000-t"]
+ if size:
+ keys.append(f"{alg}-{size}")
+ if prefix:
+ keys = [key+f'-{prefix}' for key in keys]
+
+ for i, h in enumerate(hashes):
+ res.append((keys[i], h))
+
+ return res
+
+
def file_checksum(filepath, alg, size):
if not is_macbin(filepath):
with open(filepath, "rb") as file:
- return checksum(file, alg, size, filepath)
+ return create_checksum_pairs(checksum(file, alg, size, filepath), alg, size)
- # If the file is a MacBinary
+ # If the file is a MacBinary
with open(filepath, "rb") as file:
res = []
- file = macbin_get_resfork(file)
+ file = macbin_get_resfork(file.read())
prefix = 'r'
if len(file):
@@ -53,7 +74,8 @@ def file_checksum(filepath, alg, size):
if ':' not in h:
res.append(f"{prefix}:{h}")
else:
- res.append(f"{prefix}{h}") # If the checksum is like "t:..."
+ # If the checksum is like "t:..."
+ res.append(f"{prefix}{h}")
file = macbin_get_datafork(file)
prefix = 'd'
@@ -64,7 +86,7 @@ def file_checksum(filepath, alg, size):
else:
res.append(f"{prefix}{h}") # If the checksum is like "t:..."
- return res
+ return create_checksum_pairs(res, alg, size, prefix)
def checksum(file, alg, size, filepath):
@@ -84,33 +106,51 @@ def checksum(file, alg, size, filepath):
elif alg == "sha256":
hashes = [hashlib.sha256() for _ in range(5)]
- # Read file in 8MB chunks for full checksum
- for byte_block in iter(lambda: file.read(8 * 1024 * 1024), b""):
- hashes[0].update(byte_block)
-
- # First 5000B
- file.seek(0)
- hashes[1].update(file.read(5000))
+ # If file is not a MacBinary
+ if not isinstance(file, bytes):
+ # Read file in 8MB chunks for full checksum
+ for byte_block in iter(lambda: file.read(8 * 1024 * 1024), b""):
+ hashes[0].update(byte_block)
- # First 1MB
- file.seek(0)
- hashes[2].update(file.read(1024 * 1024))
-
- # Last 5000B
- if filesize(filepath) >= 5000:
- file.seek(-5000, os.SEEK_END)
- hashes[3].update(file.read())
- else:
- hashes[3] = hashes[0]
+ # First 5000B
+ file.seek(0)
+ hashes[1].update(file.read(5000))
- # Custom size; may be None
- # Size is in bytes
- # Reads entire required size at once, inefficient for large sizes
- if size and size <= filesize(filepath):
+ # First 1MB
file.seek(0)
- hashes[4].update(file.read(size))
+ hashes[2].update(file.read(1024 * 1024))
+
+ # Last 5000B
+ if filesize(filepath) >= 5000:
+ file.seek(-5000, os.SEEK_END)
+ hashes[3].update(file.read())
+ else:
+ hashes[3] = hashes[0]
+
+ # Custom size; may be None
+ # Size is in bytes
+ # Reads entire required size at once, inefficient for large sizes
+ if size and size <= filesize(filepath):
+ file.seek(0)
+ hashes[4].update(file.read(size))
+ else:
+ hashes[4] = None
else:
- hashes[4] = None
+ bytes_stream = file
+
+ hashes[0].update(bytes_stream)
+ hashes[1].update(bytes_stream[:5000])
+ hashes[2].update(bytes_stream[:1024 * 1024])
+ if filesize(filepath) >= 5000:
+ hashes[3].update(bytes_stream[-5000:])
+ else:
+ hashes[3] = hashes[0]
+
+ # Custom size
+ if size and size <= filesize(filepath):
+ hashes[4].update(bytes_stream[:size])
+ else:
+ hashes[4] = None
hashes = [h.hexdigest() for h in hashes if h]
hashes[3] = 't:' + hashes[3] # Add tail prefix
@@ -131,7 +171,7 @@ def compute_hash_of_dirs(root_directory, depth, size=0, alg="md5"):
for file in files:
hash_of_dir[os.path.relpath(file, directory)] = (file_checksum(
- file, alg, size), alg, filesize(file))
+ file, alg, size), filesize(file))
res.append(hash_of_dir)
@@ -151,11 +191,10 @@ def create_dat_file(hash_of_dirs, path, checksum_size=0):
# Game files
for hash_of_dir in hash_of_dirs:
file.write("game (\n")
- for filename, (hashes, alg, filesize) in hash_of_dir.items():
- # Only works for MD5s, ignores optional extra size
- data = f"name \"{filename}\" size {filesize} {alg} {hashes[0]} {alg}-5000 {hashes[1]} {alg}-1M {hashes[2]} {alg}-5000-t {hashes[3]}"
- if checksum_size:
- data += f" {alg}-{checksum_size} {hashes[4]}"
+ for filename, (hashes, filesize) in hash_of_dir.items():
+ data = f"name \"{filename}\" size {filesize}"
+ for key, value in hashes:
+ data += f" {key} {value}"
file.write(f"\trom ( {data} )\n")
file.write(")\n\n")
diff --git a/dat_parser.php b/dat_parser.php
index f546ce8..e72b1e2 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -164,7 +164,8 @@ function parse_dat($dat_filepath) {
function get_checksum_props($checktype, $checksum) {
$checksize = 0;
if (strpos($checktype, '-') !== false) {
- $checksize = explode('-', $checktype)[1];
+ if (gettype(explode('-', $checktype)[1]) == 'integer')
+ $checksize = explode('-', $checktype)[1];
$checktype = explode('-', $checktype)[0];
}
Commit: ec59d324471af6baea6fb36361194253111ff1d9
https://github.com/scummvm/scummvm-sites/commit/ec59d324471af6baea6fb36361194253111ff1d9
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-12T14:50:04+05:30
Commit Message:
INTEGRITY: Add check for MacBinary
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index c60fd05..98e166b 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -2,6 +2,7 @@ import hashlib
import os
import argparse
import struct
+import zlib
script_version = "0.1"
@@ -22,8 +23,42 @@ def get_dirs_at_depth(directory, depth):
yield root
+def read_be(byte_stream, size_in_bits):
+ """ Return unsigned integer of size_in_bits, assuming the data is big-endian """
+ (uint,) = struct.unpack(">I", byte_stream[:size_in_bits//8])
+ return uint
+
+
def is_macbin(filepath):
- return True
+ with open(filepath, "rb") as file:
+ header = file.read(128)
+ if len(header) != 128:
+ return False
+
+ res_fork_offset = -1
+
+ # Preliminary check
+ # Exclude files that have zero name len, zero data fork, zero name fork and zero type_creator.
+ if not header[1] and not read_be(header[83:], 32) and not read_be(header[87:], 32) and not read_be(header[69:], 32):
+ return False
+
+ checksum = zlib.crc32(header)
+ if checksum != read_be(header[124:], 32):
+ return False
+
+ if not header[0] and not header[74] and not header[82] and header[1] <= 63:
+ # Get fork lengths
+ datalen = read_be(header[83:], 32)
+ rsrclen = read_be(header[87:], 32)
+
+ # Files produced by ISOBuster are not padded, thus, compare with the actual size
+ datalen_pad = (((datalen + 127) >> 7) << 7)
+
+ # Length check
+ if (128 + datalen_pad + rsrclen >= len(header)):
+ return False
+
+ return True
def macbin_get_resfork(file_byte_stream):
Commit: 403572ae853e10ff42b60ee235368be25ae989a8
https://github.com/scummvm/scummvm-sites/commit/403572ae853e10ff42b60ee235368be25ae989a8
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-12T14:51:25+05:30
Commit Message:
INTEGRITY: Punyencode filenames if required
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 98e166b..67e6f19 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -23,6 +23,59 @@ def get_dirs_at_depth(directory, depth):
yield root
+def escape_string(s: str) -> str:
+ """
+ Escape strings
+
+ Escape the following:
+ - escape char: \x81
+ - unallowed filename chars: https://en.wikipedia.org/wiki/Filename#Reserved_characters_and_words
+ - control chars < 0x20
+ """
+ new_name = ""
+ for char in s:
+ if char == "\x81":
+ new_name += "\x81\x79"
+ elif char in '/":*|\\?%<>\x7f' or ord(char) < 0x20:
+ new_name += "\x81" + chr(0x80 + ord(char))
+ else:
+ new_name += char
+ return new_name
+
+
+def needs_punyencoding(orig: str) -> bool:
+ """
+ A filename needs to be punyencoded when it:
+
+ - contains a char that should be escaped or
+ - ends with a dot or a space.
+ """
+ if orig != escape_string(orig):
+ return True
+ if orig[-1] in " .":
+ return True
+ return False
+
+
+def punyencode(orig: str) -> str:
+ """
+ Punyencode strings
+
+ - escape special characters and
+ - ensure filenames can't end in a space or dot
+ """
+ s = escape_string(orig)
+ encoded = s.encode("punycode").decode("ascii")
+ # punyencoding adds an '-' at the end when there are no special chars
+ # don't use it for comparing
+ compare = encoded
+ if encoded.endswith("-"):
+ compare = encoded[:-1]
+ if orig != compare or compare[-1] in " .":
+ return "xn--" + encoded
+ return orig
+
+
def read_be(byte_stream, size_in_bits):
""" Return unsigned integer of size_in_bits, assuming the data is big-endian """
(uint,) = struct.unpack(">I", byte_stream[:size_in_bits//8])
@@ -227,6 +280,8 @@ def create_dat_file(hash_of_dirs, path, checksum_size=0):
for hash_of_dir in hash_of_dirs:
file.write("game (\n")
for filename, (hashes, filesize) in hash_of_dir.items():
+ filename = (punyencode(filename)
+ if needs_punyencoding(filename) else filename)
data = f"name \"{filename}\" size {filesize}"
for key, value in hashes:
data += f" {key} {value}"
Commit: eff74a1813f4de3efc966d5636bb24766d9fa965
https://github.com/scummvm/scummvm-sites/commit/eff74a1813f4de3efc966d5636bb24766d9fa965
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-13T11:10:39+05:30
Commit Message:
INTEGRITY: Filters now present on each column
- Supports multiple filters at once
- GET variables are preserved on form submits
- Remove alias called "game name" so filters will work
- Create special styling for filter columns, and disable form spacing
Changed paths:
games_list.php
pagination.php
style.css
diff --git a/games_list.php b/games_list.php
index e69ebbd..01a75c2 100644
--- a/games_list.php
+++ b/games_list.php
@@ -3,7 +3,7 @@ require "pagination.php";
$filename = "games_list.php";
$records_table = "game";
-$select_query = "SELECT engineid, gameid, extra, platform, language, game.name as 'game name', status
+$select_query = "SELECT engineid, gameid, extra, platform, language, game.name, status
FROM game
JOIN engine ON engine.id = game.engine
JOIN fileset ON game.id = fileset.game";
diff --git a/pagination.php b/pagination.php
index 4d2e823..a26ef26 100644
--- a/pagination.php
+++ b/pagination.php
@@ -22,11 +22,22 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$conn->query("USE " . $dbname);
- if (isset($_GET['column']) && isset($_GET['value'])) {
- $column = $_GET['column'];
- $value = mysqli_real_escape_string($conn, $_GET['value']);
+ // If there exist get variables that are for filtering
+ $_GET = array_filter($_GET);
+ if (array_diff(array_keys($_GET), array('page'))) {
+ $condition = "WHERE ";
+ foreach ($_GET as $key => $value) {
+ if ($key == "page" || $value == "")
+ continue;
+
+ $last_table = $key;
+ $condition .= $condition != "WHERE " ? " AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'";
+ }
+ if ($condition == "WHERE ")
+ $condition = "";
+
$num_of_results = $conn->query(
- "SELECT COUNT(id) FROM {$filters[$column]} WHERE {$column} = '{$value}'")->fetch_array()[0];
+ "SELECT COUNT(id) FROM {$filters[$last_table]} {$condition}")->fetch_array()[0];
}
else {
$num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
@@ -41,8 +52,20 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
$offset = ($page - 1) * $results_per_page;
- if (isset($_GET['column']) && isset($_GET['value'])) {
- $query = "{$select_query} WHERE {$column} = '{$value}' {$order} LIMIT {$results_per_page} OFFSET {$offset}";
+
+ // If there exist get variables that are for filtering
+ if (array_diff(array_keys($_GET), array('page'))) {
+ $condition = "WHERE ";
+ foreach ($_GET as $key => $value) {
+ if ($key == "page" || $value == "")
+ continue;
+
+ $condition .= $condition != "WHERE " ? "AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'";
+ }
+ if ($condition == "WHERE ")
+ $condition = "";
+
+ $query = "{$select_query} {$condition} {$order} LIMIT {$results_per_page} OFFSET {$offset}";
}
else {
$query = "{$select_query} {$order} LIMIT {$results_per_page} OFFSET {$offset}";
@@ -51,32 +74,35 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$result = $conn->query($query);
- // Create filter dropdown
- if ($filters) {
- echo "<div class='filter'>\n";
- echo "<form name='filter' method='GET'>\n";
- echo "Filter: ";
+ // Table
+ echo "<form method='GET'>";
+ echo "<table>\n";
- echo "<select name='column'>\n";
- foreach (array_keys($filters) as $key) {
- echo "<option>{$key}</option>\n";
- }
- echo "</select>\n";
- echo "<input type='text' name='value' placeholder='Value'>\n";
+ // Preserve GET variables on form submit
+ foreach ($_GET as $k => $v) {
+ if ($k == 'page')
+ continue;
- echo "<input type='submit' name='submit' value='Select' />\n";
- echo "</form>\n";
- echo "</div>\n";
+ $k = htmlspecialchars($k);
+ $v = htmlspecialchars($v);
+ echo "<input type='hidden' name='{$k}' value='{$v}'>";
}
-
- // Table
- echo "<table>\n";
- echo "<th/>\n"; // Numbering column
-
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
+ echo "<tr class=filter><td></td>";
+ foreach (array_keys($row) as $key) {
+ // Filter textbox
+ $filter_value = isset($_GET[$key]) ? $_GET[$key] : "";
+
+
+ echo "<td class=filter><input type=text class=filter placeholder='{$key}' name='{$key}' value='{$filter_value}'/></td>\n";
+ }
+ echo "</tr>";
+ echo "<tr class=filter><td></td><td class=filter><input type=submit value='Submit'></td></tr>";
+
+ echo "<th/>\n"; // Numbering column
foreach (array_keys($row) as $key) {
echo "<th>{$key}</th>\n";
}
@@ -99,6 +125,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
echo "</table>\n";
+ echo "</form>\n";
// Preserve GET variables
$vars = "";
@@ -111,6 +138,18 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// Navigation elements
if ($num_of_pages > 1) {
echo "<form method='GET'>\n";
+
+ // Preserve GET variables on form submit
+ foreach ($_GET as $key => $value) {
+ if ($key == 'page')
+ continue;
+
+ $key = htmlspecialchars($key);
+ $value = htmlspecialchars($value);
+ if ($v != "")
+ echo "<input type='hidden' name='{$key}' value='{$value}'>";
+ }
+
echo "<div class=pagination>\n";
if ($page > 1)
echo "<a href={$filename}?{$vars}>â®â®</a>\n";
@@ -134,9 +173,9 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
echo "<input type='text' name='page' placeholder='Page No'>\n";
- echo "<input type='submit' name='submit' value='Submit'>\n";
+ echo "<input type='submit' value='Submit'>\n";
echo "</div>\n";
-
+
echo "</form>\n";
}
diff --git a/style.css b/style.css
index 5bc4af9..519aa25 100644
--- a/style.css
+++ b/style.css
@@ -12,10 +12,13 @@ tr {background-color: white;}
tr:hover {background-color: #ddd;}
+tr.filter:hover {background-color:inherit;}
+td.filter {text-align: center;}
+
th {
padding-top: 5px;
padding-bottom: 5px;
- text-align: left;
+ text-align: center;
background-color: var(--primary-color);
color: white;
}
@@ -63,6 +66,10 @@ input[type=text], select {
box-sizing: border-box;
}
+input[type=text].filter {
+ width: 80%;
+}
+
.pagination {
display: inline-block;
align-self: center;
@@ -92,3 +99,9 @@ input[type=text], select {
.pagination a:hover:not(.active) {
background-color: #ddd;
}
+
+form {
+ padding: 0px;
+ margin: 0px;
+ display: inline;
+}
Commit: 798fb543db3e096880466cec43399dab890c1c71
https://github.com/scummvm/scummvm-sites/commit/798fb543db3e096880466cec43399dab890c1c71
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-13T15:59:58+05:30
Commit Message:
INTEGRITY: Fix filtering over multiple tables
- Also fix only one page opening for non-game columns in pagination.php
Changed paths:
games_list.php
pagination.php
diff --git a/games_list.php b/games_list.php
index 01a75c2..c082fa2 100644
--- a/games_list.php
+++ b/games_list.php
@@ -20,6 +20,11 @@ $filters = array(
"status" => "fileset"
);
-create_page($filename, 25, $records_table, $select_query, $order, $filters);
+$mapping = array(
+ 'engine.id' => 'game.engine',
+ 'game.id' => 'fileset.game',
+);
+
+create_page($filename, 25, $records_table, $select_query, $order, $filters, $mapping);
?>
diff --git a/pagination.php b/pagination.php
index a26ef26..dbb26d5 100644
--- a/pagination.php
+++ b/pagination.php
@@ -2,7 +2,22 @@
$stylesheet = "style.css";
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
-function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array()) {
+/**
+ * Return a string denoting which two columns link two tables
+ */
+function get_join_columns($table1, $table2, $mapping) {
+ foreach ($mapping as $primary => $foreign) {
+ $primary = explode('.', $primary);
+ $foreign = explode('.', $foreign);
+ if (($primary[0] == $table1 && $foreign[0] == $table2) ||
+ ($primary[0] == $table2 && $foreign[0] == $table1))
+ return "{$primary[0]}.{$primary[1]} = {$foreign[0]}.{$foreign[1]}";
+ }
+
+ echo "No primary-foreign key mapping provided";
+}
+
+function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -26,18 +41,29 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$_GET = array_filter($_GET);
if (array_diff(array_keys($_GET), array('page'))) {
$condition = "WHERE ";
+ $tables = array();
foreach ($_GET as $key => $value) {
if ($key == "page" || $value == "")
continue;
- $last_table = $key;
+ array_push($tables, $filters[$key]);
$condition .= $condition != "WHERE " ? " AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'";
}
if ($condition == "WHERE ")
$condition = "";
+ // If more than one table is to be searched
+ $from_query = "$records_table";
+ if (count($tables) > 1 || $tables[0] != $records_table)
+ for ($i = 0; $i < count($tables); $i++) {
+ if ($tables[$i] == $records_table)
+ continue;
+
+ $from_query .= sprintf(" JOIN %s ON %s", $tables[$i], get_join_columns($records_table, $tables[$i], $mapping));
+ }
+
$num_of_results = $conn->query(
- "SELECT COUNT(id) FROM {$filters[$last_table]} {$condition}")->fetch_array()[0];
+ "SELECT COUNT({$records_table}.id) FROM {$from_query} {$condition}")->fetch_array()[0];
}
else {
$num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
Commit: f16881e4348434766f14d18b4b44e1c7eb0c20a9
https://github.com/scummvm/scummvm-sites/commit/f16881e4348434766f14d18b4b44e1c7eb0c20a9
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-13T23:14:28+05:30
Commit Message:
INTEGRITY: Add support for engine names/desc
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index e72b1e2..baaf911 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -250,7 +250,7 @@ function find_matching_game($game_files, $status) {
* Routine for inserting a game into the database, inserting into engine and
* game tables
*/
-function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn) {
+function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn) {
// Set @engine_last if engine already present in table
$exists = false;
if ($res = $conn->query(sprintf("SELECT id FROM engine WHERE engineid = '%s'", $engineid))) {
@@ -263,7 +263,7 @@ function insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn
// Insert into table if not present
if (!$exists) {
$query = sprintf("INSERT INTO engine (name, engineid)
- VALUES (NULL, '%s')", $engineid);
+ VALUES ('%s', '%s')", mysqli_real_escape_string($conn, $engine_name), $engineid);
$conn->query($query);
$conn->query("SET @engine_last = LAST_INSERT_ID()");
}
@@ -470,6 +470,7 @@ function db_insert($data_arr) {
foreach ($game_data as $fileset) {
if ($detection) {
+ $engine_name = $fileset["engine"];
$engineid = $fileset["sourcefile"];
$gameid = $fileset["name"];
$title = $fileset["title"];
@@ -477,7 +478,7 @@ function db_insert($data_arr) {
$platform = $fileset["platform"];
$lang = $fileset["language"];
- insert_game($engineid, $title, $gameid, $extra, $platform, $lang, $conn);
+ insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn);
}
elseif ($src == "dat")
if (isset($resources[$fileset["romof"]]))
Commit: 507b15c5e2cb1cc9b898cb5ec0cad0b5beed2633
https://github.com/scummvm/scummvm-sites/commit/507b15c5e2cb1cc9b898cb5ec0cad0b5beed2633
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-13T23:14:33+05:30
Commit Message:
INTEGRITY: Skip duplicate detection entries
Also add an index on fileset.key for faster searching
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index baaf911..38db7c7 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -485,7 +485,6 @@ function db_insert($data_arr) {
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
insert_fileset($src, $detection, $conn);
- calc_key($fileset["rom"]);
foreach ($fileset["rom"] as $file) {
insert_file($file, $detection, $src, $conn);
foreach ($file as $key => $value) {
@@ -496,8 +495,11 @@ function db_insert($data_arr) {
// Add key if uploaded DAT is of detection entries
if ($detection) {
- $conn->query(sprintf("UPDATE fileset SET `key` = '%s' WHERE id = @fileset_last",
- calc_key($fileset["rom"])));
+ $fileset_key = calc_key($fileset["rom"]);
+ if ($conn->query("SELECT id FROM fileset WHERE `key` = '{$fileset_key}'")->num_rows == 0)
+ $conn->query("UPDATE fileset SET `key` = '{$fileset_key}' WHERE id = @fileset_last");
+ else
+ $conn->query("DELETE FROM fileset WHERE id = @fileset_last");
}
}
$category_text = "Uploaded from {$src}";
diff --git a/schema.php b/schema.php
index 8abebc8..642ffc9 100644
--- a/schema.php
+++ b/schema.php
@@ -195,6 +195,15 @@ else {
echo "Error creating index for 'engine.engineid': " . $conn->error;
}
+$index = "CREATE INDEX fileset_key ON fileset (`key`)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'fileset.key'<br/>";
+}
+else {
+ echo "Error creating index for 'fileset.key': " . $conn->error;
+}
+
$index = "CREATE INDEX fileset ON history (fileset)";
if ($conn->query($index) === TRUE) {
Commit: c39736329bb4774de40ae33934d14cc272d07cca
https://github.com/scummvm/scummvm-sites/commit/c39736329bb4774de40ae33934d14cc272d07cca
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T13:46:41+05:30
Commit Message:
INTEGRITY: Add timestamp to fileset table
When detection entries are not updated (not present in the deteciton
DAT), they are marked as obsolete
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 38db7c7..46fffce 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -290,8 +290,8 @@ function insert_fileset($src, $detection, $conn) {
}
// $game should not be parsed as a mysql string, hence no quotes
- $query = sprintf("INSERT INTO fileset (game, status, src, `key`)
- VALUES (%s, '%s', '%s', NULL)", $game, $status, $src);
+ $query = sprintf("INSERT INTO fileset (game, status, src, `key`, `timestamp`)
+ VALUES (%s, '%s', '%s', NULL, FROM_UNIXTIME(@fileset_time_last))", $game, $status, $src);
$conn->query($query);
$conn->query("SET @fileset_last = LAST_INSERT_ID()");
}
@@ -451,13 +451,14 @@ function db_insert($data_arr) {
$version = $header["version"];
/**
- * src can be:
+ * status can be:
* detection -> Detection entries (source of truth)
* user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
* scan -> Submitted by cli/scanner, unmatched
* dat -> Submitted by DAT, unmatched
* partialmatch -> Submitted by DAT, matched
* fullmatch -> Submitted by cli/scanner, matched
+ * obsolete -> Detection entries that are no longer part of the detection set
*/
$src = "";
if ($author == "scan" || $author == "scummvm")
@@ -468,6 +469,9 @@ function db_insert($data_arr) {
$detection = ($src == "scummvm");
$status = $detection ? "detection" : $src;
+ // Set timestamp of fileset insertion
+ $conn->query(sprintf("SET @fileset_time_last = %d", time()));
+
foreach ($game_data as $fileset) {
if ($detection) {
$engine_name = $fileset["engine"];
@@ -496,12 +500,25 @@ function db_insert($data_arr) {
// Add key if uploaded DAT is of detection entries
if ($detection) {
$fileset_key = calc_key($fileset["rom"]);
- if ($conn->query("SELECT id FROM fileset WHERE `key` = '{$fileset_key}'")->num_rows == 0)
+ $existing_entries = $conn->query("SELECT id FROM fileset WHERE `key` = '{$fileset_key}'");
+ if ($existing_entries->num_rows == 0)
$conn->query("UPDATE fileset SET `key` = '{$fileset_key}' WHERE id = @fileset_last");
- else
+ else {
+ $existing_entry = $existing_entries->fetch_array()[0];
+ $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
+ WHERE id = {$existing_entry}");
+ $conn->query("UPDATE fileset SET status = 'detection'
+ WHERE id = {$existing_entry} AND status = 'obsolete'");
$conn->query("DELETE FROM fileset WHERE id = @fileset_last");
+ }
}
}
+
+ if ($detection)
+ $conn->query("UPDATE fileset SET status = 'obsolete'
+ WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last)
+ AND status = 'detection'");
+
$category_text = "Uploaded from {$src}";
$log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
State '%s'. Fileset:%d.",
diff --git a/schema.php b/schema.php
index 642ffc9..f0975a3 100644
--- a/schema.php
+++ b/schema.php
@@ -70,6 +70,7 @@ $table = "CREATE TABLE IF NOT EXISTS fileset (
src VARCHAR(20),
`key` VARCHAR(64),
`delete` BOOLEAN DEFAULT FALSE NOT NULL,
+ `timestamp` TIMESTAMP NOT NULL,
FOREIGN KEY (game) REFERENCES game(id)
)";
Commit: a9c32f5d5a735ff7d3fe088fec17064545961625
https://github.com/scummvm/scummvm-sites/commit/a9c32f5d5a735ff7d3fe088fec17064545961625
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T15:56:29+05:30
Commit Message:
INTEGRITY: Existing filesets are marked for deletion
Triggered with --match
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 46fffce..5a2924c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -16,6 +16,35 @@ function calc_key($files) {
return md5($key_string);
}
+/**
+ * Get an array containing files in a fileset, to be passed into calc_key()
+ */
+function compare_filesets($id1, $id2, $conn) {
+ $fileset1 = $conn->query("SELECT name, size, checksum
+ FROM file WHERE fileset = '{$id1}'")->fetch_all();
+ $fileset2 = $conn->query("SELECT name, size, checksum
+ FROM file WHERE fileset = '{$id2}'")->fetch_all();
+
+ // Sort filesets on checksum
+ usort($fileset1, function ($a, $b) {
+ return $a[2] <=> $b[2];
+ });
+ usort($fileset2, function ($a, $b) {
+ return $a[2] <=> $b[2];
+ });
+
+ if (count($fileset1) != count($fileset2))
+ return false;
+
+ for ($i = 0; $i < count($fileset1); $i++) {
+ // If checksums do not match
+ if ($fileset1[2] != $fileset2[2])
+ return false;
+ }
+
+ return True;
+}
+
function remove_quotes($string) {
// Remove quotes from value if they are present
if ($string[0] == "\"")
@@ -195,7 +224,7 @@ function status_to_match($status) {
* Compares the files with those in the detection entries table
* $game_files consists of both the game ( ) and resources ( ) parts
*/
-function find_matching_game($game_files, $status) {
+function find_matching_game($game_files) {
$matching_games = array(); // All matching games
$matching_filesets = array(); // All filesets containing one file from $game_files
$matches_count = 0; // Number of files with a matching detection entry
@@ -205,13 +234,11 @@ function find_matching_game($game_files, $status) {
foreach ($game_files as $file) {
$checksum = $file[1];
- $records = $conn->query(sprintf("SELECT file.fileset
+ $query = "SELECT file.fileset as file_fileset
FROM filechecksum
JOIN file ON filechecksum.file = file.id
- JOIN fileset ON fileset.id = file.fileset
- WHERE filechecksum.checksum = '%s' AND file.detection = TRUE AND
- fileset.status IN ('%s')", $checksum, implode("', '", status_to_match($status))));
- $records = $records->fetch_all();
+ WHERE filechecksum.checksum = '{$checksum}' AND file.detection = TRUE";
+ $records = $conn->query($query)->fetch_all();
// If file is not part of detection entries, skip it
if (count($records) == 0)
@@ -220,9 +247,7 @@ function find_matching_game($game_files, $status) {
$matches_count++;
foreach ($records as $record)
array_push($matching_filesets, $record[0]);
- }
-
- // Check if there is a fileset_id that is present in all results
+ } // Check if there is a fileset_id that is present in all results
foreach (array_count_values($matching_filesets) as $key => $value) {
$count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
JOIN fileset ON file.fileset = fileset.id
@@ -243,6 +268,23 @@ function find_matching_game($game_files, $status) {
array_push($matching_games, $records->fetch_array());
}
+ if (count($matching_games) != 1)
+ return $matching_games;
+
+ // Check the current fileset priority with that of the match
+ $records = $conn->query(sprintf("SELECT id FROM fileset, ({$query}) AS res
+ WHERE id = file_fileset AND
+ status IN ('%s')", implode("', '", status_to_match($game_files[3]))));
+
+ // If priority order is correct
+ if ($records->num_rows != 0)
+ return $matching_games;
+
+ if (compare_filesets($matching_games[0]['fileset'], $game_files[0][0], $conn)) {
+ $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0]}");
+ return array();
+ }
+
return $matching_games;
}
@@ -539,17 +581,16 @@ function populate_matching_games() {
// Getting unmatched filesets
$unmatched_filesets = array();
- $unmatched_files = $conn->query(sprintf("SELECT fileset.id, filechecksum.checksum, src, status
+ $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status
FROM fileset
JOIN file ON file.fileset = fileset.id
JOIN filechecksum ON file.id = filechecksum.file
- WHERE fileset.game IS NULL"));
+ WHERE fileset.game IS NULL");
$unmatched_files = $unmatched_files->fetch_all();
// Splitting them into different filesets
for ($i = 0; $i < count($unmatched_files); $i++) {
$cur_fileset = $unmatched_files[$i][0];
- $src = $unmatched_files[$i][2];
$temp = array();
while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
array_push($temp, $unmatched_files[$i]);
@@ -559,7 +600,9 @@ function populate_matching_games() {
}
foreach ($unmatched_filesets as $fileset) {
- $matching_games = find_matching_game($fileset, $fileset[0][3]);
+ // If another fileset with the same key exists, mark current fileset for deletion
+
+ $matching_games = find_matching_game($fileset);
if (count($matching_games) != 1) // If there is no match/non-unique match
continue;
Commit: 22356ae98372c670a3e1cf09d8efab3b46320d85
https://github.com/scummvm/scummvm-sites/commit/22356ae98372c670a3e1cf09d8efab3b46320d85
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T16:24:27+05:30
Commit Message:
INTEGRITY: Delete games without detection entries
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 5a2924c..00ac11c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -552,6 +552,7 @@ function db_insert($data_arr) {
$conn->query("UPDATE fileset SET status = 'detection'
WHERE id = {$existing_entry} AND status = 'obsolete'");
$conn->query("DELETE FROM fileset WHERE id = @fileset_last");
+ $conn->query("DELETE FROM game WHERE id = @game_last");
}
}
}
Commit: 3c0b75c563bbfa399a99dfc4c182f32d2844fce5
https://github.com/scummvm/scummvm-sites/commit/3c0b75c563bbfa399a99dfc4c182f32d2844fce5
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T19:24:02+05:30
Commit Message:
INTEGRITY: Skip insertion of existing fileset
Instead of deleting fileset after insertion
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 00ac11c..bbcd9e6 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -322,20 +322,35 @@ function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform
* Inserting new fileset
* Called for both detection entries and other forms of DATs
*/
-function insert_fileset($src, $detection, $conn) {
+function insert_fileset($src, $detection, $key, $conn) {
$status = $detection ? "detection" : $src;
$game = "NULL";
+ $key = $key == "" ? "NULL" : "'{$key}'";
if ($detection) {
$status = "detection";
$game = "@game_last";
+
+ // Check if key already exists, if so, skip insertion
+ $existing_entry = $conn->query("SELECT id FROM fileset WHERE `key` = {$key}");
+ if ($existing_entry->num_rows > 0) {
+ $existing_entry = $existing_entry->fetch_array()[0];
+ $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
+ WHERE id = {$existing_entry}");
+ $conn->query("UPDATE fileset SET status = 'detection'
+ WHERE id = {$existing_entry} AND status = 'obsolete'");
+ $conn->query("DELETE FROM game WHERE id = @game_last");
+ return false;
+ }
}
- // $game should not be parsed as a mysql string, hence no quotes
- $query = sprintf("INSERT INTO fileset (game, status, src, `key`, `timestamp`)
- VALUES (%s, '%s', '%s', NULL, FROM_UNIXTIME(@fileset_time_last))", $game, $status, $src);
+ // $game and $key should not be parsed as a mysql string, hence no quotes
+ $query = "INSERT INTO fileset (game, status, src, `key`, `timestamp`)
+ VALUES ({$game}, '{$status}', '{$src}', {$key}, FROM_UNIXTIME(@fileset_time_last))";
$conn->query($query);
$conn->query("SET @fileset_last = LAST_INSERT_ID()");
+
+ return true;
}
/**
@@ -530,29 +545,14 @@ function db_insert($data_arr) {
if (isset($resources[$fileset["romof"]]))
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
- insert_fileset($src, $detection, $conn);
- foreach ($fileset["rom"] as $file) {
- insert_file($file, $detection, $src, $conn);
- foreach ($file as $key => $value) {
- if ($key != "name" && $key != "size")
- insert_filechecksum($file, $key, $conn);
- }
- }
-
- // Add key if uploaded DAT is of detection entries
- if ($detection) {
- $fileset_key = calc_key($fileset["rom"]);
- $existing_entries = $conn->query("SELECT id FROM fileset WHERE `key` = '{$fileset_key}'");
- if ($existing_entries->num_rows == 0)
- $conn->query("UPDATE fileset SET `key` = '{$fileset_key}' WHERE id = @fileset_last");
- else {
- $existing_entry = $existing_entries->fetch_array()[0];
- $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
- WHERE id = {$existing_entry}");
- $conn->query("UPDATE fileset SET status = 'detection'
- WHERE id = {$existing_entry} AND status = 'obsolete'");
- $conn->query("DELETE FROM fileset WHERE id = @fileset_last");
- $conn->query("DELETE FROM game WHERE id = @game_last");
+ $key = $detection ? calc_key($fileset['rom']) : "";
+ if (insert_fileset($src, $detection, $key, $conn)) {
+ foreach ($fileset["rom"] as $file) {
+ insert_file($file, $detection, $src, $conn);
+ foreach ($file as $key => $value) {
+ if ($key != "name" && $key != "size")
+ insert_filechecksum($file, $key, $conn);
+ }
}
}
}
Commit: 12c978776d7f0310ce691eee0580a335cb20b9eb
https://github.com/scummvm/scummvm-sites/commit/12c978776d7f0310ce691eee0580a335cb20b9eb
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T22:35:23+05:30
Commit Message:
INTEGRITY: Fix MacBinary check, add new CRC16 func
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 67e6f19..a604eed 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -2,10 +2,52 @@ import hashlib
import os
import argparse
import struct
-import zlib
script_version = "0.1"
+# CRC table
+CRC16_XMODEM_TABLE = [
+ 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
+ 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
+ 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
+ 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
+ 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
+ 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
+ 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
+ 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
+ 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
+ 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
+ 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
+ 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
+ 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
+ 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
+ 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
+ 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
+ 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
+ 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
+ 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
+ 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
+ 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
+ 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
+ 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
+ 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
+ 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
+ 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
+ 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
+ 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
+ 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
+ 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
+ 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
+ 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
+]
+
+
+def crc16xmodem(data, crc=0):
+ for byte in data:
+ crc = ((crc << 8) & 0xff00) ^ CRC16_XMODEM_TABLE[(
+ (crc >> 8) & 0xff) ^ byte]
+ return crc & 0xffff
+
def filesize(filepath):
""" Returns size of file """
@@ -76,9 +118,15 @@ def punyencode(orig: str) -> str:
return orig
-def read_be(byte_stream, size_in_bits):
+def read_be_32(byte_stream):
+ """ Return unsigned integer of size_in_bits, assuming the data is big-endian """
+ (uint,) = struct.unpack(">I", byte_stream[:32//8])
+ return uint
+
+
+def read_be_16(byte_stream):
""" Return unsigned integer of size_in_bits, assuming the data is big-endian """
- (uint,) = struct.unpack(">I", byte_stream[:size_in_bits//8])
+ (uint,) = struct.unpack(">H", byte_stream[:16//8])
return uint
@@ -92,23 +140,26 @@ def is_macbin(filepath):
# Preliminary check
# Exclude files that have zero name len, zero data fork, zero name fork and zero type_creator.
- if not header[1] and not read_be(header[83:], 32) and not read_be(header[87:], 32) and not read_be(header[69:], 32):
+ if not header[1] and not read_be_32(header[83:]) and not read_be_32(header[87:]) and not read_be_32(header[69:]):
return False
- checksum = zlib.crc32(header)
- if checksum != read_be(header[124:], 32):
+ checksum = crc16xmodem(header[:124])
+ if checksum != read_be_16(header[124:]):
return False
if not header[0] and not header[74] and not header[82] and header[1] <= 63:
# Get fork lengths
- datalen = read_be(header[83:], 32)
- rsrclen = read_be(header[87:], 32)
+ datalen = read_be_32(header[83:])
+ rsrclen = read_be_32(header[87:])
# Files produced by ISOBuster are not padded, thus, compare with the actual size
datalen_pad = (((datalen + 127) >> 7) << 7)
# Length check
- if (128 + datalen_pad + rsrclen >= len(header)):
+ if (128 + datalen_pad + rsrclen <= filesize(filepath)):
+ res_fork_offset = 128 + datalen_pad
+
+ if res_fork_offset < 0:
return False
return True
@@ -150,31 +201,39 @@ def file_checksum(filepath, alg, size):
with open(filepath, "rb") as file:
return create_checksum_pairs(checksum(file, alg, size, filepath), alg, size)
- # If the file is a MacBinary
- with open(filepath, "rb") as file:
+ # If the file is a MacBinary
+ with open(filepath, "rb") as f:
res = []
- file = macbin_get_resfork(file.read())
+ hashes = []
+ file = macbin_get_resfork(f.read())
prefix = 'r'
if len(file):
for h in checksum(file, alg, size, filepath):
if ':' not in h:
- res.append(f"{prefix}:{h}")
+ hashes.append(f"{prefix}:{h}")
else:
# If the checksum is like "t:..."
- res.append(f"{prefix}{h}")
+ hashes.append(f"{prefix}{h}")
- file = macbin_get_datafork(file)
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
+
+ hashes = []
+ f.seek(0)
+ file = macbin_get_datafork(f.read())
prefix = 'd'
for h in checksum(file, alg, size, filepath):
if ':' not in h:
- res.append(f"{prefix}:{h}")
+ hashes.append(f"{prefix}:{h}")
else:
- res.append(f"{prefix}{h}") # If the checksum is like "t:..."
+ # If the checksum is like "t:..."
+ hashes.append(f"{prefix}{h}")
+
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
- return create_checksum_pairs(res, alg, size, prefix)
+ return res
def checksum(file, alg, size, filepath):
Commit: ef0c8a4108b7d3571e79966ba9c7428b588a5585
https://github.com/scummvm/scummvm-sites/commit/ef0c8a4108b7d3571e79966ba9c7428b588a5585
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-14T22:40:27+05:30
Commit Message:
INTEGRITY: Fix broken $checksize check
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index bbcd9e6..1f1941c 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -193,8 +193,9 @@ function parse_dat($dat_filepath) {
function get_checksum_props($checktype, $checksum) {
$checksize = 0;
if (strpos($checktype, '-') !== false) {
- if (gettype(explode('-', $checktype)[1]) == 'integer')
- $checksize = explode('-', $checktype)[1];
+ $temp = explode('-', $checktype)[1];
+ if (in_array($temp, array('0', '5000', '1M')))
+ $checksize = $temp;
$checktype = explode('-', $checktype)[0];
}
Commit: fe8ed8ef01c7f52bdfe1b5605ed4d3be0b6ce2f1
https://github.com/scummvm/scummvm-sites/commit/fe8ed8ef01c7f52bdfe1b5605ed4d3be0b6ce2f1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-15T19:37:34+05:30
Commit Message:
INTEGRITY: Sanitize values of filters
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index dbb26d5..6f68ee5 100644
--- a/pagination.php
+++ b/pagination.php
@@ -83,6 +83,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if (array_diff(array_keys($_GET), array('page'))) {
$condition = "WHERE ";
foreach ($_GET as $key => $value) {
+ $value = mysqli_real_escape_string($conn, $value);
if ($key == "page" || $value == "")
continue;
Commit: 4446cdeca20d15fbfe6cfd85e14a43b8e24b4979
https://github.com/scummvm/scummvm-sites/commit/4446cdeca20d15fbfe6cfd85e14a43b8e24b4979
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-20T10:20:05+05:30
Commit Message:
INTEGRITY: Skip insertion of duplicate DATs
- Add megakey to fileset table
- Megakey calculated for DAT filesets on insertion
- Fileset with megakey equal to one already in the db is not inserted
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 1f1941c..6ecd544 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -323,31 +323,39 @@ function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform
* Inserting new fileset
* Called for both detection entries and other forms of DATs
*/
-function insert_fileset($src, $detection, $key, $conn) {
+function insert_fileset($src, $detection, $key, $megakey, $conn) {
$status = $detection ? "detection" : $src;
$game = "NULL";
$key = $key == "" ? "NULL" : "'{$key}'";
+ $megakey = $megakey == "" ? "NULL" : "'{$megakey}'";
if ($detection) {
$status = "detection";
$game = "@game_last";
+ }
- // Check if key already exists, if so, skip insertion
+ // Check if key/megakey already exists, if so, skip insertion (no quotes on purpose)
+ if ($detection)
$existing_entry = $conn->query("SELECT id FROM fileset WHERE `key` = {$key}");
- if ($existing_entry->num_rows > 0) {
- $existing_entry = $existing_entry->fetch_array()[0];
- $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
+ else
+ $existing_entry = $conn->query("SELECT id FROM fileset WHERE megakey = {$megakey}");
+
+ if ($existing_entry->num_rows > 0) {
+ if (!$detection)
+ return false;
+
+ $existing_entry = $existing_entry->fetch_array()[0];
+ $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
WHERE id = {$existing_entry}");
- $conn->query("UPDATE fileset SET status = 'detection'
+ $conn->query("UPDATE fileset SET status = 'detection'
WHERE id = {$existing_entry} AND status = 'obsolete'");
- $conn->query("DELETE FROM game WHERE id = @game_last");
- return false;
- }
+ $conn->query("DELETE FROM game WHERE id = @game_last");
+ return false;
}
// $game and $key should not be parsed as a mysql string, hence no quotes
- $query = "INSERT INTO fileset (game, status, src, `key`, `timestamp`)
- VALUES ({$game}, '{$status}', '{$src}', {$key}, FROM_UNIXTIME(@fileset_time_last))";
+ $query = "INSERT INTO fileset (game, status, src, `key`, megakey, `timestamp`)
+ VALUES ({$game}, '{$status}', '{$src}', {$key}, {$megakey}, FROM_UNIXTIME(@fileset_time_last))";
$conn->query($query);
$conn->query("SET @fileset_last = LAST_INSERT_ID()");
@@ -547,7 +555,8 @@ function db_insert($data_arr) {
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
$key = $detection ? calc_key($fileset['rom']) : "";
- if (insert_fileset($src, $detection, $key, $conn)) {
+ $megakey = !$detection ? calc_key($fileset['rom']) : "";
+ if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
foreach ($fileset["rom"] as $file) {
insert_file($file, $detection, $src, $conn);
foreach ($file as $key => $value) {
diff --git a/schema.php b/schema.php
index f0975a3..f9d1d49 100644
--- a/schema.php
+++ b/schema.php
@@ -69,6 +69,7 @@ $table = "CREATE TABLE IF NOT EXISTS fileset (
status VARCHAR(20),
src VARCHAR(20),
`key` VARCHAR(64),
+ `megakey` VARCHAR(64),
`delete` BOOLEAN DEFAULT FALSE NOT NULL,
`timestamp` TIMESTAMP NOT NULL,
FOREIGN KEY (game) REFERENCES game(id)
Commit: 4ad5db171dc3b1400e2372a4324944d8b6d1d14e
https://github.com/scummvm/scummvm-sites/commit/4ad5db171dc3b1400e2372a4324944d8b6d1d14e
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-22T19:54:00+05:30
Commit Message:
INTEGRITY: Add links to filesets of game entries
- Fileset column added to games_list.php, hardcoded to have hyperlinks
to corresponding fileset entry in fileset.php
- Filtering textbox only appears if the column is present in $filters
- Changed error message on using unknown filter
Changed paths:
games_list.php
pagination.php
diff --git a/games_list.php b/games_list.php
index c082fa2..8b17364 100644
--- a/games_list.php
+++ b/games_list.php
@@ -3,7 +3,8 @@ require "pagination.php";
$filename = "games_list.php";
$records_table = "game";
-$select_query = "SELECT engineid, gameid, extra, platform, language, game.name, status
+$select_query = "SELECT engineid, gameid, extra, platform, language, game.name,
+status, fileset.id as fileset
FROM game
JOIN engine ON engine.id = game.engine
JOIN fileset ON game.id = fileset.game";
diff --git a/pagination.php b/pagination.php
index 6f68ee5..0747ba1 100644
--- a/pagination.php
+++ b/pagination.php
@@ -14,7 +14,7 @@ function get_join_columns($table1, $table2, $mapping) {
return "{$primary[0]}.{$primary[1]} = {$foreign[0]}.{$foreign[1]}";
}
- echo "No primary-foreign key mapping provided";
+ echo "No primary-foreign key mapping provided. Filter is invalid";
}
function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
@@ -115,30 +115,42 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<input type='hidden' name='{$k}' value='{$v}'>";
}
+ $fileset_column_index = null;
+
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
echo "<tr class=filter><td></td>";
foreach (array_keys($row) as $key) {
+ if (!isset($filters[$key])) {
+ echo "<td class=filter />";
+ continue;
+ }
+
// Filter textbox
$filter_value = isset($_GET[$key]) ? $_GET[$key] : "";
-
echo "<td class=filter><input type=text class=filter placeholder='{$key}' name='{$key}' value='{$filter_value}'/></td>\n";
}
echo "</tr>";
echo "<tr class=filter><td></td><td class=filter><input type=submit value='Submit'></td></tr>";
echo "<th/>\n"; // Numbering column
- foreach (array_keys($row) as $key) {
+ foreach (array_keys($row) as $index => $key) {
echo "<th>{$key}</th>\n";
+
+ if ($key == "fileset")
+ $fileset_column_index = $index;
}
}
echo "<tr>\n";
echo "<td>{$counter}.</td>\n";
- foreach (array_values($row) as $value) {
+ foreach (array_values($row) as $key => $value) {
// Add hyperlink to filesets
+ if ($fileset_column_index && $key == $fileset_column_index)
+ $value = "<a href='fileset.php?id={$value}'>{$value}</a>";
+
$matches = array();
if (preg_match("/Fileset:(\d+)/", $value, $matches, PREG_OFFSET_CAPTURE)) {
$value = substr($value, 0, $matches[0][1]) . "<a href='fileset.php?id={$matches[1][0]}'>{$matches[0][0]}</a>";
Commit: cbad3a300bdf005e51c2d9859bddec79abe965f1
https://github.com/scummvm/scummvm-sites/commit/cbad3a300bdf005e51c2d9859bddec79abe965f1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-22T22:45:43+05:30
Commit Message:
INTEGRITY: Add all file checksums to fileset page
- Support $records_table with a JOIN in the query
- Hardcode 'id' to not be a filter
- Skip making a filter table row if $filters is empty
Changed paths:
fileset.php
pagination.php
diff --git a/fileset.php b/fileset.php
index 0bc4290..4aa8f08 100644
--- a/fileset.php
+++ b/fileset.php
@@ -86,8 +86,10 @@ echo "</tr>\n";
echo "</table>\n";
echo "<h3>Files in the fileset</h3>";
-create_page($filename, 15, "file WHERE fileset = {$id}",
- "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}", "ORDER BY name");
+$records_table = "filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
+$select_query = "SELECT name, size, filechecksum.checksum, checksize, checktype, detection
+ FROM filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
+create_page($filename, 15, $records_table, $select_query, "ORDER BY name");
// Dev Actions
diff --git a/pagination.php b/pagination.php
index 0747ba1..db8e35b 100644
--- a/pagination.php
+++ b/pagination.php
@@ -39,7 +39,13 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// If there exist get variables that are for filtering
$_GET = array_filter($_GET);
- if (array_diff(array_keys($_GET), array('page'))) {
+
+ // If $records_table has a JOIN (multiple tables)
+ if (preg_match("/JOIN/", $records_table) !== false) {
+ $first_table = explode(" ", $records_table)[0];
+ $num_of_results = $conn->query("SELECT COUNT({$first_table}.id) FROM {$records_table}")->fetch_array()[0];
+ }
+ elseif (array_diff(array_keys($_GET), array('page'))) {
$condition = "WHERE ";
$tables = array();
foreach ($_GET as $key => $value) {
@@ -53,7 +59,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$condition = "";
// If more than one table is to be searched
- $from_query = "$records_table";
+ $from_query = $records_table;
if (count($tables) > 1 || $tables[0] != $records_table)
for ($i = 0; $i < count($tables); $i++) {
if ($tables[$i] == $records_table)
@@ -84,7 +90,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$condition = "WHERE ";
foreach ($_GET as $key => $value) {
$value = mysqli_real_escape_string($conn, $value);
- if ($key == "page" || $value == "")
+ if ($key == "page" || $key == "id" || $value == "")
continue;
$condition .= $condition != "WHERE " ? "AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'";
@@ -97,7 +103,6 @@ function create_page($filename, $results_per_page, $records_table, $select_query
else {
$query = "{$select_query} {$order} LIMIT {$results_per_page} OFFSET {$offset}";
}
-
$result = $conn->query($query);
@@ -120,20 +125,22 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
- echo "<tr class=filter><td></td>";
- foreach (array_keys($row) as $key) {
- if (!isset($filters[$key])) {
- echo "<td class=filter />";
- continue;
+ if (count($filters) > 0) {
+ echo "<tr class=filter><td></td>";
+ foreach (array_keys($row) as $key) {
+ if (!isset($filters[$key])) {
+ echo "<td class=filter />";
+ continue;
+ }
+
+ // Filter textbox
+ $filter_value = isset($_GET[$key]) ? $_GET[$key] : "";
+
+ echo "<td class=filter><input type=text class=filter placeholder='{$key}' name='{$key}' value='{$filter_value}'/></td>\n";
}
-
- // Filter textbox
- $filter_value = isset($_GET[$key]) ? $_GET[$key] : "";
-
- echo "<td class=filter><input type=text class=filter placeholder='{$key}' name='{$key}' value='{$filter_value}'/></td>\n";
+ echo "</tr>";
+ echo "<tr class=filter><td></td><td class=filter><input type=submit value='Submit'></td></tr>";
}
- echo "</tr>";
- echo "<tr class=filter><td></td><td class=filter><input type=submit value='Submit'></td></tr>";
echo "<th/>\n"; // Numbering column
foreach (array_keys($row) as $index => $key) {
Commit: 89c380da54adbe6a76200425c579a494b8b495f1
https://github.com/scummvm/scummvm-sites/commit/89c380da54adbe6a76200425c579a494b8b495f1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T14:46:09+05:30
Commit Message:
INTEGRITY: Index fileset.status for better perf
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index f9d1d49..0883d7a 100644
--- a/schema.php
+++ b/schema.php
@@ -206,6 +206,15 @@ else {
echo "Error creating index for 'fileset.key': " . $conn->error;
}
+$index = "CREATE INDEX status ON fileset (status)";
+
+if ($conn->query($index) === TRUE) {
+ echo "Created index for 'fileset.status'<br/>";
+}
+else {
+ echo "Error creating index for 'fileset.status': " . $conn->error;
+}
+
$index = "CREATE INDEX fileset ON history (fileset)";
if ($conn->query($index) === TRUE) {
Commit: b8835088d2b27bea3eb7f70d611e135a90b30cfb
https://github.com/scummvm/scummvm-sites/commit/b8835088d2b27bea3eb7f70d611e135a90b30cfb
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T14:53:52+05:30
Commit Message:
INTEGRITY: Add prev and next buttons to navigation
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index db8e35b..130c345 100644
--- a/pagination.php
+++ b/pagination.php
@@ -197,8 +197,10 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
echo "<div class=pagination>\n";
- if ($page > 1)
+ if ($page > 1) {
echo "<a href={$filename}?{$vars}>â®â®</a>\n";
+ echo sprintf("<a href=%s?page=%d%s>â®</a>\n", $filename, $page - 1, $vars);
+ }
if ($page - 2 > 1)
echo "<div class=more>...</div>\n";
@@ -215,8 +217,10 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($page + 2 < $num_of_pages)
echo "<div class=more>...</div>\n";
- if ($page < $num_of_pages)
+ if ($page < $num_of_pages) {
+ echo sprintf("<a href=%s?page=%d%s>â¯</a>\n", $filename, $page + 1, $vars);
echo "<a href={$filename}?page={$num_of_pages}{$vars}>â¯â¯</a>\n";
+ }
echo "<input type='text' name='page' placeholder='Page No'>\n";
echo "<input type='submit' value='Submit'>\n";
Commit: 9f60de0aa912b85f58822386dc10bbd02c7a8c39
https://github.com/scummvm/scummvm-sites/commit/9f60de0aa912b85f58822386dc10bbd02c7a8c39
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T15:46:12+05:30
Commit Message:
INTEGRITY: Remove hardcoded exception to filtering
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 130c345..62d6ef6 100644
--- a/pagination.php
+++ b/pagination.php
@@ -90,7 +90,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$condition = "WHERE ";
foreach ($_GET as $key => $value) {
$value = mysqli_real_escape_string($conn, $value);
- if ($key == "page" || $key == "id" || $value == "")
+ if (!isset($filters[$key]))
continue;
$condition .= $condition != "WHERE " ? "AND {$filters[$key]}.{$key} REGEXP '{$value}'" : "{$filters[$key]}.{$key} REGEXP '{$value}'";
Commit: 17f0a4cc3076293f415ec6a4106dacb8fad5f554
https://github.com/scummvm/scummvm-sites/commit/17f0a4cc3076293f415ec6a4106dacb8fad5f554
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T18:25:43+05:30
Commit Message:
INTEGRITY: Fix - empty variables in URL
Changed paths:
js_functions.js
pagination.php
diff --git a/js_functions.js b/js_functions.js
index 8bfb7ca..8cb56cd 100644
--- a/js_functions.js
+++ b/js_functions.js
@@ -11,6 +11,19 @@ function delete_id(value) {
});
}
+function remove_empty_inputs() {
+ var myForm = document.getElementById("filters-form");
+ var allInputs = myForm.getElementsByTagName("input");
+ var input, i;
+
+ for (i = 0; (input = allInputs[i]); i++) {
+ if (input.getAttribute("name") && !input.value) {
+ console.log(input);
+ input.setAttribute("name", "");
+ }
+ }
+}
+
$(document).ready(function () {
$(".hidden").hide();
$("#delete-button").one("click", delete_id);
diff --git a/pagination.php b/pagination.php
index 62d6ef6..1b9d99e 100644
--- a/pagination.php
+++ b/pagination.php
@@ -1,6 +1,10 @@
<?php
$stylesheet = "style.css";
+$jquery_file = 'https://code.jquery.com/jquery-3.7.0.min.js';
+$js_file = 'js_functions.js';
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
+echo "<script type='text/javascript' src='{$jquery_file}'></script>\n";
+echo "<script type='text/javascript' src='{$js_file}'></script>\n";
/**
* Return a string denoting which two columns link two tables
@@ -107,19 +111,9 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// Table
- echo "<form method='GET'>";
+ echo "<form id='filters-form' method='GET' onsubmit='remove_empty_inputs()'>";
echo "<table>\n";
- // Preserve GET variables on form submit
- foreach ($_GET as $k => $v) {
- if ($k == 'page')
- continue;
-
- $k = htmlspecialchars($k);
- $v = htmlspecialchars($v);
- echo "<input type='hidden' name='{$k}' value='{$v}'>";
- }
-
$fileset_column_index = null;
$counter = $offset + 1;
@@ -192,7 +186,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$key = htmlspecialchars($key);
$value = htmlspecialchars($value);
- if ($v != "")
+ if ($value != "")
echo "<input type='hidden' name='{$key}' value='{$value}'>";
}
Commit: a3b045cfe8bb3e94b8c98a8ced044f0dcffc0c59
https://github.com/scummvm/scummvm-sites/commit/a3b045cfe8bb3e94b8c98a8ced044f0dcffc0c59
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T18:40:47+05:30
Commit Message:
INTEGRITY: Add expand table button to fileset page
Expanding the 'Files in the fileset' table shows all checksums of files
TODO: Add the other checksums as other columns instead of extra rows
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index 4aa8f08..17a2fb0 100644
--- a/fileset.php
+++ b/fileset.php
@@ -85,10 +85,41 @@ foreach ($result as $column => $value) {
echo "</tr>\n";
echo "</table>\n";
+if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') {
+ $records_table = "filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
+ $select_query = "SELECT name, size, filechecksum.checksum, checksize, checktype, detection
+ FROM filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
+}
+else {
+ $records_table = "file WHERE fileset = {$id}";
+ $select_query = "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}";
+}
+
echo "<h3>Files in the fileset</h3>";
-$records_table = "filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
-$select_query = "SELECT name, size, filechecksum.checksum, checksize, checktype, detection
- FROM filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
+echo "<form>";
+// Preserve GET variables on form submit
+foreach ($_GET as $k => $v) {
+ if ($k == 'widetable')
+ continue;
+
+ $k = htmlspecialchars($k);
+ $v = htmlspecialchars($v);
+ echo "<input type='hidden' name='{$k}' value='{$v}'>";
+}
+
+// Come up with a better solution to set widetable=true on button click
+// Currently uses hidden text input
+if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') {
+ echo "<input class='hidden' type='text' name='widetable' value='false' />";
+ echo "<input type='submit' value='Hide extra checksums' />";
+}
+else {
+ echo "<input class='hidden' type='text' name='widetable' value='true' />";
+ echo "<input type='submit' value='Expand Table' />";
+}
+
+echo "</form>";
+
create_page($filename, 15, $records_table, $select_query, "ORDER BY name");
Commit: 681af1fd58ddce5592f14ecea8decf9941b2f5bb
https://github.com/scummvm/scummvm-sites/commit/681af1fd58ddce5592f14ecea8decf9941b2f5bb
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-23T22:42:51+05:30
Commit Message:
INTEGRITY: Update check for checktype
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 6ecd544..9683dce 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -194,7 +194,7 @@ function get_checksum_props($checktype, $checksum) {
$checksize = 0;
if (strpos($checktype, '-') !== false) {
$temp = explode('-', $checktype)[1];
- if (in_array($temp, array('0', '5000', '1M')))
+ if ($temp == '1M' || is_numeric($temp))
$checksize = $temp;
$checktype = explode('-', $checktype)[0];
}
Commit: 30bda499973085b1814eef97445f0e63c0dc40a3
https://github.com/scummvm/scummvm-sites/commit/30bda499973085b1814eef97445f0e63c0dc40a3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T00:04:58+05:30
Commit Message:
INTEGRITY: Create file to validate game files
- Create sample request that would come from ScummVM
- Check if key calculated from received request matches an existing
game, if so, game files are all good.
Changed paths:
A checksum_validator.php
A sample_json_request.json
diff --git a/checksum_validator.php b/checksum_validator.php
new file mode 100644
index 0000000..0f425b4
--- /dev/null
+++ b/checksum_validator.php
@@ -0,0 +1,74 @@
+<?php
+
+function calc_key($json_object) {
+ $key_string = '';
+
+ $file_object = $json_object->files;
+ foreach ($file_object as $file) {
+ foreach ($file as $key => $value) {
+ if ($key != 'checksums') {
+ $key_string .= ':' . $value;
+ continue;
+ }
+
+ foreach ($value as $checksum_data) {
+ foreach ($checksum_data as $key => $value) {
+ $key_string .= ':' . $value;
+ }
+ }
+ }
+ }
+
+ $key_string = trim($key_string, ':');
+ return md5($key_string);
+}
+
+$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+$servername = $mysql_cred["servername"];
+$username = $mysql_cred["username"];
+$password = $mysql_cred["password"];
+$dbname = $mysql_cred["dbname"];
+
+// Create connection
+$conn = new mysqli($servername, $username, $password);
+
+// Check connection
+if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+}
+
+$conn->query("USE " . $dbname);
+
+$json_string = file_get_contents('sample_json_request.json');
+$json_object = json_decode($json_string);
+
+$game_metadata = array();
+foreach ($json_object as $key => $value) {
+ if ($key == 'files')
+ continue;
+
+ $game_metadata[$key] = $value;
+}
+
+// Find game(s) that fit the metadata
+// $query = "SELECT game.id FROM game
+// JOIN engine ON game.engine = engine.id
+// WHERE gameid = '{$game_metadata['gameid']}'
+// AND engineid = '{$game_metadata['engineid']}'
+// AND extra = '{$game_metadata['extra']}'
+// AND platform = '{$game_metadata['platform']}'
+// AND language = '{$game_metadata['language']}'";
+// $games = $conn->query($query)->fetch_all();
+
+$matches = $conn->query(sprintf("SELECT game.id FROM fileset
+JOIN game ON fileset.game = game.id
+WHERE `key` = '%s' AND fileset.status = 'fullmatch'", calc_key($json_object)));
+
+if ($matches->num_rows == 0)
+ echo "No games found / Files corrupted";
+else
+ echo "Game files are correct";
+
+$conn->close();
+?>
+
diff --git a/sample_json_request.json b/sample_json_request.json
new file mode 100644
index 0000000..97ece0c
--- /dev/null
+++ b/sample_json_request.json
@@ -0,0 +1,53 @@
+{
+ "gameid": "drascula",
+ "engineid": "drascula",
+ "extra": "",
+ "platform": "pc",
+ "language": "en",
+ "files": [
+ {
+ "name": "Packet.001",
+ "size": 32847563,
+ "checksums": [
+ {
+ "type": "md5",
+ "checksum": "fac946707f07d51696a02c00cc182078"
+ },
+ {
+ "type": "md5-5000",
+ "checksum": "c6a8697396e213a18472542d5f547cb4"
+ },
+ {
+ "type": "md5-1M",
+ "checksum": "9ae6c33420e4e187c9d07a144a6c5fa2"
+ },
+ {
+ "type": "md5-t",
+ "checksum": "e223c341cfd15879d5fa77e696e82167"
+ }
+ ]
+ },
+ {
+ "name": "readme.txt",
+ "size": 2871,
+ "checksums": [
+ {
+ "type": "md5",
+ "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
+ },
+ {
+ "type": "md5-5000",
+ "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
+ },
+ {
+ "type": "md5-1M",
+ "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
+ },
+ {
+ "type": "md5-t",
+ "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
+ }
+ ]
+ }
+ ]
+}
Commit: 7c93faf95fbdeced0c17de9a7869a6ad6a165cfd
https://github.com/scummvm/scummvm-sites/commit/7c93faf95fbdeced0c17de9a7869a6ad6a165cfd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T21:54:44+05:30
Commit Message:
INTEGRITY: Check that checksum is not empty
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 9683dce..391adf2 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -64,8 +64,10 @@ function map_checksum_data($content_string) {
for ($i = 1; $i < count($temp); $i += 2) {
if ($temp[$i] == ')')
continue;
- $temp[$i + 1] = remove_quotes($temp[$i + 1]);
+ $temp[$i + 1] = remove_quotes($temp[$i + 1]);
+ if ($temp[$i + 1] == ')')
+ $temp[$i + 1] = "";
$arr[$temp[$i]] = stripslashes($temp[$i + 1]);
}
@@ -611,8 +613,6 @@ function populate_matching_games() {
}
foreach ($unmatched_filesets as $fileset) {
- // If another fileset with the same key exists, mark current fileset for deletion
-
$matching_games = find_matching_game($fileset);
if (count($matching_games) != 1) // If there is no match/non-unique match
Commit: 961433bf288b577d6cf05df20c46c54a9cf9b2c2
https://github.com/scummvm/scummvm-sites/commit/961433bf288b577d6cf05df20c46c54a9cf9b2c2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T21:54:54+05:30
Commit Message:
INTEGRITY: Update JSON request schema
Changed paths:
sample_json_request.json
diff --git a/sample_json_request.json b/sample_json_request.json
index 97ece0c..3f4dbac 100644
--- a/sample_json_request.json
+++ b/sample_json_request.json
@@ -22,8 +22,8 @@
"checksum": "9ae6c33420e4e187c9d07a144a6c5fa2"
},
{
- "type": "md5-t",
- "checksum": "e223c341cfd15879d5fa77e696e82167"
+ "type": "md5",
+ "checksum": "t:e223c341cfd15879d5fa77e696e82167"
}
]
},
@@ -44,8 +44,8 @@
"checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
},
{
- "type": "md5-t",
- "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
+ "type": "md5",
+ "checksum": "t:2bc04d164f561ee8a8cfd9618869e5d5"
}
]
}
Commit: 09a6d2e6b83b3bcd32353b63a3a6628930d29fd3
https://github.com/scummvm/scummvm-sites/commit/09a6d2e6b83b3bcd32353b63a3a6628930d29fd3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T21:56:45+05:30
Commit Message:
INTEGRITY: Update formatting for schema.php
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 0883d7a..b1ac60d 100644
--- a/schema.php
+++ b/schema.php
@@ -17,7 +17,7 @@ if ($conn->connect_errno) {
// Create database
$sql = "CREATE DATABASE IF NOT EXISTS " . $dbname;
if ($conn->query($sql) === TRUE) {
- echo "Database created successfully<br/>";
+ echo "Database created successfully\n";
}
else {
echo "Error creating database: " . $conn->error;
@@ -37,7 +37,7 @@ $table = "CREATE TABLE IF NOT EXISTS engine (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'engine' created successfully<br/>";
+ echo "Table 'engine' created successfully\n";
}
else {
echo "Error creating 'engine' table: " . $conn->error;
@@ -56,7 +56,7 @@ $table = "CREATE TABLE IF NOT EXISTS game (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'game' created successfully<br/>";
+ echo "Table 'game' created successfully\n";
}
else {
echo "Error creating 'game' table: " . $conn->error;
@@ -76,7 +76,7 @@ $table = "CREATE TABLE IF NOT EXISTS fileset (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'fileset' created successfully<br/>";
+ echo "Table 'fileset' created successfully\n";
}
else {
echo "Error creating 'fileset' table: " . $conn->error;
@@ -94,7 +94,7 @@ $table = "CREATE TABLE IF NOT EXISTS file (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'file' created successfully<br/>";
+ echo "Table 'file' created successfully\n";
}
else {
echo "Error creating 'file' table: " . $conn->error;
@@ -111,7 +111,7 @@ $table = "CREATE TABLE IF NOT EXISTS filechecksum (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'filechecksum' created successfully<br/>";
+ echo "Table 'filechecksum' created successfully\n";
}
else {
echo "Error creating 'filechecksum' table: " . $conn->error;
@@ -130,7 +130,7 @@ $table = "CREATE TABLE IF NOT EXISTS queue (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'queue' created successfully<br/>";
+ echo "Table 'queue' created successfully\n";
}
else {
echo "Error creating 'queue' table: " . $conn->error;
@@ -146,7 +146,7 @@ $table = "CREATE TABLE IF NOT EXISTS log (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'log' created successfully<br/>";
+ echo "Table 'log' created successfully\n";
}
else {
echo "Error creating 'log' table: " . $conn->error;
@@ -162,7 +162,7 @@ $table = "CREATE TABLE IF NOT EXISTS history (
)";
if ($conn->query($table) === TRUE) {
- echo "Table 'history' created successfully<br/>";
+ echo "Table 'history' created successfully\n";
}
else {
echo "Error creating 'history' table: " . $conn->error;
@@ -173,7 +173,7 @@ else {
$index = "CREATE INDEX detection ON file (detection)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'file.detection'<br/>";
+ echo "Created index for 'file.detection'\n";
}
else {
echo "Error creating index for 'file.detection': " . $conn->error;
@@ -182,7 +182,7 @@ else {
$index = "CREATE INDEX checksum ON filechecksum (checksum)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'filechecksum.checksum'<br/>";
+ echo "Created index for 'filechecksum.checksum'\n";
}
else {
echo "Error creating index for 'filechecksum.checksum': " . $conn->error;
@@ -191,7 +191,7 @@ else {
$index = "CREATE INDEX engineid ON engine (engineid)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'engine.engineid'<br/>";
+ echo "Created index for 'engine.engineid'\n";
}
else {
echo "Error creating index for 'engine.engineid': " . $conn->error;
@@ -200,7 +200,7 @@ else {
$index = "CREATE INDEX fileset_key ON fileset (`key`)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'fileset.key'<br/>";
+ echo "Created index for 'fileset.key'\n";
}
else {
echo "Error creating index for 'fileset.key': " . $conn->error;
@@ -209,7 +209,7 @@ else {
$index = "CREATE INDEX status ON fileset (status)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'fileset.status'<br/>";
+ echo "Created index for 'fileset.status'\n";
}
else {
echo "Error creating index for 'fileset.status': " . $conn->error;
@@ -218,7 +218,7 @@ else {
$index = "CREATE INDEX fileset ON history (fileset)";
if ($conn->query($index) === TRUE) {
- echo "Created index for 'history.fileset'<br/>";
+ echo "Created index for 'history.fileset'\n";
}
else {
echo "Error creating index for 'history.fileset': " . $conn->error;
Commit: 0e537969ef2a2e3bb756ab8b2ab1f379ea9b0237
https://github.com/scummvm/scummvm-sites/commit/0e537969ef2a2e3bb756ab8b2ab1f379ea9b0237
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T22:12:18+05:30
Commit Message:
INTEGRITY: Add detection_size to fileset table
detection_size is the checksize of detection entries for that fileset
Changed paths:
dat_parser.php
schema.php
diff --git a/dat_parser.php b/dat_parser.php
index 391adf2..2b17400 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -192,13 +192,13 @@ function parse_dat($dat_filepath) {
* Retrieves the checksum and checktype of a given type + checksum
* eg: md5-5000 t:12345... -> 5000, md5-t, 12345...
*/
-function get_checksum_props($checktype, $checksum) {
+function get_checksum_props($checkcode, $checksum) {
$checksize = 0;
- if (strpos($checktype, '-') !== false) {
- $temp = explode('-', $checktype)[1];
+ if (strpos($checkcode, '-') !== false) {
+ $temp = explode('-', $checkcode)[1];
if ($temp == '1M' || is_numeric($temp))
$checksize = $temp;
- $checktype = explode('-', $checktype)[0];
+ $checktype = explode('-', $checkcode)[0];
}
if (strpos($checksum, ':') !== false) {
@@ -379,7 +379,7 @@ function insert_file($file, $detection, $src, $conn) {
else {
foreach ($file as $key => $value) {
if (strpos($key, "md5") !== false) {
- list($tmp1, $tmp2, $checksum) = get_checksum_props($key, $value);
+ list($checksize, $checktype, $checksum) = get_checksum_props($key, $value);
break;
}
}
@@ -389,6 +389,8 @@ function insert_file($file, $detection, $src, $conn) {
VALUES ('%s', '%s', '%s', @fileset_last, %d)", mysqli_real_escape_string($conn, $file["name"]),
$file["size"], $checksum, $detection);
$conn->query($query);
+
+ $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL");
$conn->query("SET @file_last = LAST_INSERT_ID()");
}
diff --git a/schema.php b/schema.php
index b1ac60d..68add30 100644
--- a/schema.php
+++ b/schema.php
@@ -72,6 +72,7 @@ $table = "CREATE TABLE IF NOT EXISTS fileset (
`megakey` VARCHAR(64),
`delete` BOOLEAN DEFAULT FALSE NOT NULL,
`timestamp` TIMESTAMP NOT NULL,
+ detection_size INT,
FOREIGN KEY (game) REFERENCES game(id)
)";
Commit: 02223fbc586dd2c3a95a772689196e8bdaafa84c
https://github.com/scummvm/scummvm-sites/commit/02223fbc586dd2c3a95a772689196e8bdaafa84c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T23:14:24+05:30
Commit Message:
INTEGRITY: Add special checksum columns
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index 17a2fb0..e7aa284 100644
--- a/fileset.php
+++ b/fileset.php
@@ -85,16 +85,6 @@ foreach ($result as $column => $value) {
echo "</tr>\n";
echo "</table>\n";
-if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') {
- $records_table = "filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
- $select_query = "SELECT name, size, filechecksum.checksum, checksize, checktype, detection
- FROM filechecksum JOIN file ON file.id = filechecksum.file WHERE fileset = {$id}";
-}
-else {
- $records_table = "file WHERE fileset = {$id}";
- $select_query = "SELECT name, size, checksum, detection FROM file WHERE fileset = {$id}";
-}
-
echo "<h3>Files in the fileset</h3>";
echo "<form>";
// Preserve GET variables on form submit
@@ -120,8 +110,48 @@ else {
echo "</form>";
-create_page($filename, 15, $records_table, $select_query, "ORDER BY name");
+// Table
+echo "<table>\n";
+
+$result = $conn->query("SELECT file.id, name, size, checksum, detection
+ FROM file WHERE fileset = {$id}")->fetch_all(MYSQLI_ASSOC);
+
+if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') {
+ foreach (array_values($result) as $index => $file) {
+ $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype
+ FROM filechecksum WHERE file = {$file['id']}");
+
+ while ($spec_checksum = $spec_checksum_res->fetch_assoc()) {
+ $result[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum'];
+ }
+ }
+}
+
+$counter = 1;
+foreach ($result as $row) {
+ if ($counter == 1) {
+ echo "<th/>\n"; // Numbering column
+ foreach (array_keys($row) as $index => $key) {
+ if ($key == 'id')
+ continue;
+
+ echo "<th>{$key}</th>\n";
+ }
+ }
+ echo "<tr>\n";
+ echo "<td>{$counter}.</td>\n";
+ foreach ($row as $key => $value) {
+ if ($key == 'id')
+ continue;
+
+ echo "<td>{$value}</td>\n";
+ }
+ echo "</tr>\n";
+
+ $counter++;
+}
+echo "</table>\n";
// Dev Actions
echo "<h3>Developer Actions</h3>";
Commit: 5d154f93aa78a3b82d457bd90c03dcdf50053a1a
https://github.com/scummvm/scummvm-sites/commit/5d154f93aa78a3b82d457bd90c03dcdf50053a1a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-24T23:15:06+05:30
Commit Message:
INTEGRITY: Fix comment in pagination.php
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 1b9d99e..98ce68e 100644
--- a/pagination.php
+++ b/pagination.php
@@ -148,10 +148,11 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<tr>\n";
echo "<td>{$counter}.</td>\n";
foreach (array_values($row) as $key => $value) {
- // Add hyperlink to filesets
+ // Add hyperlink to fileset in game_list table
if ($fileset_column_index && $key == $fileset_column_index)
$value = "<a href='fileset.php?id={$value}'>{$value}</a>";
+ // Add links to fileset in logs table
$matches = array();
if (preg_match("/Fileset:(\d+)/", $value, $matches, PREG_OFFSET_CAPTURE)) {
$value = substr($value, 0, $matches[0][1]) . "<a href='fileset.php?id={$matches[1][0]}'>{$matches[0][0]}</a>";
Commit: 1678ae13fe682fa5661c9d2471f93a55fdf4c938
https://github.com/scummvm/scummvm-sites/commit/1678ae13fe682fa5661c9d2471f93a55fdf4c938
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T17:29:42+05:30
Commit Message:
INTEGRITY Checksum validation for user files
- Check metadata to find games that match
- Build framework for checking file by file and building a JSON
response for result of every file
Changed paths:
checksum_validator.php
diff --git a/checksum_validator.php b/checksum_validator.php
index 0f425b4..339413b 100644
--- a/checksum_validator.php
+++ b/checksum_validator.php
@@ -1,27 +1,5 @@
<?php
-
-function calc_key($json_object) {
- $key_string = '';
-
- $file_object = $json_object->files;
- foreach ($file_object as $file) {
- foreach ($file as $key => $value) {
- if ($key != 'checksums') {
- $key_string .= ':' . $value;
- continue;
- }
-
- foreach ($value as $checksum_data) {
- foreach ($checksum_data as $key => $value) {
- $key_string .= ':' . $value;
- }
- }
- }
- }
-
- $key_string = trim($key_string, ':');
- return md5($key_string);
-}
+require('dat_parser.php');
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
@@ -51,23 +29,70 @@ foreach ($json_object as $key => $value) {
}
// Find game(s) that fit the metadata
-// $query = "SELECT game.id FROM game
-// JOIN engine ON game.engine = engine.id
-// WHERE gameid = '{$game_metadata['gameid']}'
-// AND engineid = '{$game_metadata['engineid']}'
-// AND extra = '{$game_metadata['extra']}'
-// AND platform = '{$game_metadata['platform']}'
-// AND language = '{$game_metadata['language']}'";
-// $games = $conn->query($query)->fetch_all();
-
-$matches = $conn->query(sprintf("SELECT game.id FROM fileset
-JOIN game ON fileset.game = game.id
-WHERE `key` = '%s' AND fileset.status = 'fullmatch'", calc_key($json_object)));
-
-if ($matches->num_rows == 0)
- echo "No games found / Files corrupted";
-else
- echo "Game files are correct";
+$query = "SELECT game.id FROM game
+JOIN engine ON game.engine = engine.id
+WHERE gameid = '{$game_metadata['gameid']}'
+AND engineid = '{$game_metadata['engineid']}'
+AND extra = '{$game_metadata['extra']}'
+AND platform = '{$game_metadata['platform']}'
+AND language = '{$game_metadata['language']}'";
+$games = $conn->query($query);
+
+$json_response = array(
+ 'error' => 0,
+ 'files' => array()
+);
+
+// Check if all files in fullmatch filesets are present with user
+while ($game = $games->fetch_array()) {
+ $fileset = $conn->query("SELECT name, size,
+ checksize, checktype, filechecksum.checksum
+ FROM filechecksum
+ JOIN file ON file.id = filechecksum.file
+ JOIN fileset ON fileset.id = file.fileset
+ WHERE fileset.game = {$game['id']} AND fileset.status = 'fullmatch'");
+
+ if ($fileset->num_rows == 0)
+ continue;
+
+ $file_object = $json_object->files;
+ print_r($fileset);
+
+ $mismatch = false;
+ foreach ($file_object as $user_file) {
+ $status = 'ok';
+ $db_file = $fileset->fetch_array();
+
+ if (!($db_file['name'] == $user_file['name'])) {
+ $mismatch = true;
+ $status = 'unknown';
+ }
+
+
+ if ($mismatch)
+ break;
+
+ foreach ($value as $checksum_data) {
+ if ($mismatch)
+ break;
+
+ foreach ($checksum_data as $key => $value) {
+ $user_checkcode = $checksum_data->type;
+ $user_checksum = $checksum_data->checksum;
+
+ list($checksize, $checktype, $checksum) = get_checksum_props($user_checkcode, $user_checksum);
+ $mismatch == !($db_file['checksum'] == $checksum) ||
+ !($db_file['checktype'] == $checktype) ||
+ !($db_file['checksize'] == $checksize);
+
+ if ($mismatch)
+ break;
+ }
+ }
+ }
+
+
+}
$conn->close();
?>
Commit: 96b78ae53ff7fe7ba027a36ee62a2012b8546dbf
https://github.com/scummvm/scummvm-sites/commit/96b78ae53ff7fe7ba027a36ee62a2012b8546dbf
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T18:18:56+05:30
Commit Message:
INTEGRITY: Bugfixes for dat_parser.php
- fileset.detection_size updated only for detection entries
- Fix detection_size for entries with checksize 5000
- Fix checktype for full checksums
- Skip copying files not in the scanned dataset
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 2b17400..02074bc 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -194,6 +194,7 @@ function parse_dat($dat_filepath) {
*/
function get_checksum_props($checkcode, $checksum) {
$checksize = 0;
+ $checktype = $checkcode;
if (strpos($checkcode, '-') !== false) {
$temp = explode('-', $checkcode)[1];
if ($temp == '1M' || is_numeric($temp))
@@ -373,6 +374,7 @@ function insert_fileset($src, $detection, $key, $megakey, $conn) {
function insert_file($file, $detection, $src, $conn) {
// Find md5-5000, or else use first checksum value
$checksum = "";
+ $checksize = 5000;
if (isset($file["md5-5000"])) {
$checksum = $file["md5-5000"];
}
@@ -390,7 +392,8 @@ function insert_file($file, $detection, $src, $conn) {
$file["size"], $checksum, $detection);
$conn->query($query);
- $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL");
+ if ($detection)
+ $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL");
$conn->query("SET @file_last = LAST_INSERT_ID()");
}
@@ -436,11 +439,6 @@ function merge_filesets($detection_id, $dat_id) {
$checksize, $checktype, $dat_id, $checksum));
}
- // Move files from the original fileset to the new fileset
- $conn->query(sprintf("UPDATE file
- SET fileset = %d
- WHERE fileset = %d", $dat_id, $detection_id));
-
// Add fileset pair to history ($dat_id is the new fileset for $detection_id)
$conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id));
Commit: 2eff9302200e69e05b5ecf322cb139c1e942951b
https://github.com/scummvm/scummvm-sites/commit/2eff9302200e69e05b5ecf322cb139c1e942951b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T18:19:08+05:30
Commit Message:
INTEGRITY: Change format of tail checksums
Changed paths:
sample_json_request.json
diff --git a/sample_json_request.json b/sample_json_request.json
index 3f4dbac..ae4d134 100644
--- a/sample_json_request.json
+++ b/sample_json_request.json
@@ -22,8 +22,8 @@
"checksum": "9ae6c33420e4e187c9d07a144a6c5fa2"
},
{
- "type": "md5",
- "checksum": "t:e223c341cfd15879d5fa77e696e82167"
+ "type": "md5-t-5000",
+ "checksum": "e223c341cfd15879d5fa77e696e82167"
}
]
},
@@ -44,8 +44,8 @@
"checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
},
{
- "type": "md5",
- "checksum": "t:2bc04d164f561ee8a8cfd9618869e5d5"
+ "type": "md5-t-5000",
+ "checksum": "2bc04d164f561ee8a8cfd9618869e5d5"
}
]
}
Commit: 2773978c0a979d5317dfe1d2c39dcf0859468c59
https://github.com/scummvm/scummvm-sites/commit/2773978c0a979d5317dfe1d2c39dcf0859468c59
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T18:21:22+05:30
Commit Message:
INTEGRITY: Checksum validation for user filesets
- Checksums are checked one by one for every file
- Create json_response object that will be at the endpoint
- Unique error codes for different mismatches
Changed paths:
checksum_validator.php
diff --git a/checksum_validator.php b/checksum_validator.php
index 339413b..354746a 100644
--- a/checksum_validator.php
+++ b/checksum_validator.php
@@ -1,6 +1,4 @@
<?php
-require('dat_parser.php');
-
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -45,55 +43,78 @@ $json_response = array(
// Check if all files in fullmatch filesets are present with user
while ($game = $games->fetch_array()) {
- $fileset = $conn->query("SELECT name, size,
- checksize, checktype, filechecksum.checksum
- FROM filechecksum
- JOIN file ON file.id = filechecksum.file
+ $fileset = $conn->query("SELECT file.id, name, size FROM file
JOIN fileset ON fileset.id = file.fileset
WHERE fileset.game = {$game['id']} AND fileset.status = 'fullmatch'");
if ($fileset->num_rows == 0)
continue;
- $file_object = $json_object->files;
- print_r($fileset);
-
- $mismatch = false;
- foreach ($file_object as $user_file) {
- $status = 'ok';
- $db_file = $fileset->fetch_array();
+ // Convert checktype, checksize to checkcode
+ $fileset = $fileset->fetch_all(MYSQLI_ASSOC);
+ foreach (array_values($fileset) as $index => $file) {
+ $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype
+ FROM filechecksum WHERE file = {$file['id']}");
- if (!($db_file['name'] == $user_file['name'])) {
- $mismatch = true;
- $status = 'unknown';
+ while ($spec_checksum = $spec_checksum_res->fetch_assoc()) {
+ $fileset[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum'];
}
+ }
+ $file_object = $json_object->files;
- if ($mismatch)
- break;
-
- foreach ($value as $checksum_data) {
- if ($mismatch)
- break;
-
- foreach ($checksum_data as $key => $value) {
- $user_checkcode = $checksum_data->type;
- $user_checksum = $checksum_data->checksum;
+ // Sort the filesets by filename
+ usort($file_object, function ($a, $b) {
+ return strcmp($a->name, $b->name) == -1 ? -1 : 1;
+ });
+ usort($fileset, function ($a, $b) {
+ return strcmp($a['name'], $b['name']) == -1 ? -1 : 1;
+ });
- list($checksize, $checktype, $checksum) = get_checksum_props($user_checkcode, $user_checksum);
- $mismatch == !($db_file['checksum'] == $checksum) ||
- !($db_file['checktype'] == $checktype) ||
- !($db_file['checksize'] == $checksize);
+ for ($i = 0, $j = 0; $i < count($fileset), $j < count($file_object); $i++, $j++) {
+ $status = 'ok';
+ $db_file = $fileset[$i];
+ $user_file = $file_object[$j];
+ $filename = $user_file->name;
+
+ if ($db_file['name'] != $user_file->name) {
+ if ($db_file['name'] > $user_file->name) {
+ $status = 'unknown_file';
+ $i--; // Retain same db_file for next iteration
+ }
+ else {
+ $status = 'missing';
+ $filename = $db_file['name'];
+ $j--; // Retain same user_file for next iteration
+ }
+ }
+ elseif ($db_file['size'] != $user_file->size && $status == 'ok') {
+ $status = 'size_mismatch';
+ }
- if ($mismatch)
- break;
+ if ($status == 'ok') {
+ foreach ($user_file->checksums as $checksum_data) {
+ foreach ($checksum_data as $key => $value) {
+ $user_checksum = $checksum_data->checksum;
+ $user_checkcode = $checksum_data->type;
+ if (strpos($user_checkcode, '-') === false)
+ $user_checkcode .= '-0';
+
+ if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
+ $status = 'checksum_mismatch';
+ }
}
}
- }
+ if ($status != 'ok')
+ $json_response['error'] = 1;
+ array_push($json_response['files'], array('status' => $status, 'name' => $filename));
+ }
}
+$json_response = json_encode($json_response);
+
$conn->close();
?>
Commit: 1c747e4ffdb6e818010d8a9414b6ee0b442be940
https://github.com/scummvm/scummvm-sites/commit/1c747e4ffdb6e818010d8a9414b6ee0b442be940
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T18:53:14+05:30
Commit Message:
INTEGRITY: Fix warning while insertion of DATs
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 02074bc..66b2903 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -553,7 +553,7 @@ function db_insert($data_arr) {
insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn);
}
elseif ($src == "dat")
- if (isset($resources[$fileset["romof"]]))
+ if (isset($fileset['romof']) && isset($resources[$fileset['romof']]))
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
$key = $detection ? calc_key($fileset['rom']) : "";
Commit: 32f804546af96f39de6df88421b40b2f4f30ca01
https://github.com/scummvm/scummvm-sites/commit/32f804546af96f39de6df88421b40b2f4f30ca01
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T20:56:10+05:30
Commit Message:
INTEGRITY: Refactor - Create file db_functions.php
Move db functions from dat_parser.php into db_functions so they can be
used without having to import dat_parser
Changed paths:
A db_functions.php
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 66b2903..9ea46b6 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -1,50 +1,8 @@
<?php
+require('db_functions.php');
ini_set('memory_limit', '512M');
-/**
- * Calculate `key` value as md5("file1:size:md5:file2:...")
- */
-function calc_key($files) {
- $key_string = "";
- foreach ($files as $file) {
- foreach ($file as $key => $value) {
- $key_string .= ':' . $value;
- }
- }
- $key_string = trim($key_string, ':');
- return md5($key_string);
-}
-
-/**
- * Get an array containing files in a fileset, to be passed into calc_key()
- */
-function compare_filesets($id1, $id2, $conn) {
- $fileset1 = $conn->query("SELECT name, size, checksum
- FROM file WHERE fileset = '{$id1}'")->fetch_all();
- $fileset2 = $conn->query("SELECT name, size, checksum
- FROM file WHERE fileset = '{$id2}'")->fetch_all();
-
- // Sort filesets on checksum
- usort($fileset1, function ($a, $b) {
- return $a[2] <=> $b[2];
- });
- usort($fileset2, function ($a, $b) {
- return $a[2] <=> $b[2];
- });
-
- if (count($fileset1) != count($fileset2))
- return false;
-
- for ($i = 0; $i < count($fileset1); $i++) {
- // If checksums do not match
- if ($fileset1[2] != $fileset2[2])
- return false;
- }
-
- return True;
-}
-
function remove_quotes($string) {
// Remove quotes from value if they are present
if ($string[0] == "\"")
@@ -188,476 +146,6 @@ function parse_dat($dat_filepath) {
return array($header, $game_data, $resources, $dat_filepath);
}
-/**
- * Retrieves the checksum and checktype of a given type + checksum
- * eg: md5-5000 t:12345... -> 5000, md5-t, 12345...
- */
-function get_checksum_props($checkcode, $checksum) {
- $checksize = 0;
- $checktype = $checkcode;
- if (strpos($checkcode, '-') !== false) {
- $temp = explode('-', $checkcode)[1];
- if ($temp == '1M' || is_numeric($temp))
- $checksize = $temp;
- $checktype = explode('-', $checkcode)[0];
- }
-
- if (strpos($checksum, ':') !== false) {
- $prefix = explode(':', $checksum)[0];
-
- if (strpos($prefix, 't') !== false)
- $checktype .= "-" . 't';
- $checksum = explode(':', $checksum)[1];
- }
-
- return array($checksize, $checktype, $checksum);
-}
-
-/**
- * Return fileset statuses that can be merged with set of given status
- * eg: scan and dat -> detection
- * fullmatch -> partialmatch, detection
- */
-function status_to_match($status) {
- $order = array("detection", "dat", "scan", "partialmatch", "fullmatch");
- return array_slice($order, 0, array_search($status, $order));
-}
-
-/**
- * Detects games based on the file descriptions in $dat_arr
- * Compares the files with those in the detection entries table
- * $game_files consists of both the game ( ) and resources ( ) parts
- */
-function find_matching_game($game_files) {
- $matching_games = array(); // All matching games
- $matching_filesets = array(); // All filesets containing one file from $game_files
- $matches_count = 0; // Number of files with a matching detection entry
-
- $conn = db_connect();
-
- foreach ($game_files as $file) {
- $checksum = $file[1];
-
- $query = "SELECT file.fileset as file_fileset
- FROM filechecksum
- JOIN file ON filechecksum.file = file.id
- WHERE filechecksum.checksum = '{$checksum}' AND file.detection = TRUE";
- $records = $conn->query($query)->fetch_all();
-
- // If file is not part of detection entries, skip it
- if (count($records) == 0)
- continue;
-
- $matches_count++;
- foreach ($records as $record)
- array_push($matching_filesets, $record[0]);
- } // Check if there is a fileset_id that is present in all results
- foreach (array_count_values($matching_filesets) as $key => $value) {
- $count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
- JOIN fileset ON file.fileset = fileset.id
- WHERE fileset.id = '%s'", $key))->fetch_array()[0];
-
- // We use < instead of != since one file may have more than one entry in the fileset
- // We see this in Drascula English version, where one entry is duplicated
- if ($value < $matches_count || $value < $count_files_in_fileset)
- continue;
-
- $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform,
- language, `key`, src, fileset.id as fileset
- FROM game
- JOIN fileset ON fileset.game = game.id
- JOIN engine ON engine.id = game.engine
- WHERE fileset.id = '%s'", $key));
-
- array_push($matching_games, $records->fetch_array());
- }
-
- if (count($matching_games) != 1)
- return $matching_games;
-
- // Check the current fileset priority with that of the match
- $records = $conn->query(sprintf("SELECT id FROM fileset, ({$query}) AS res
- WHERE id = file_fileset AND
- status IN ('%s')", implode("', '", status_to_match($game_files[3]))));
-
- // If priority order is correct
- if ($records->num_rows != 0)
- return $matching_games;
-
- if (compare_filesets($matching_games[0]['fileset'], $game_files[0][0], $conn)) {
- $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0]}");
- return array();
- }
-
- return $matching_games;
-}
-
-/**
- * Routine for inserting a game into the database, inserting into engine and
- * game tables
- */
-function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn) {
- // Set @engine_last if engine already present in table
- $exists = false;
- if ($res = $conn->query(sprintf("SELECT id FROM engine WHERE engineid = '%s'", $engineid))) {
- if ($res->num_rows > 0) {
- $exists = true;
- $conn->query(sprintf("SET @engine_last = '%d'", $res->fetch_array()[0]));
- }
- }
-
- // Insert into table if not present
- if (!$exists) {
- $query = sprintf("INSERT INTO engine (name, engineid)
- VALUES ('%s', '%s')", mysqli_real_escape_string($conn, $engine_name), $engineid);
- $conn->query($query);
- $conn->query("SET @engine_last = LAST_INSERT_ID()");
- }
-
- // Insert into game
- $query = sprintf("INSERT INTO game (name, engine, gameid, extra, platform, language)
- VALUES ('%s', @engine_last, '%s', '%s', '%s', '%s')", mysqli_real_escape_string($conn, $title),
- $gameid, mysqli_real_escape_string($conn, $extra), $platform, $lang);
- $conn->query($query);
- $conn->query("SET @game_last = LAST_INSERT_ID()");
-}
-
-/**
- * Inserting new fileset
- * Called for both detection entries and other forms of DATs
- */
-function insert_fileset($src, $detection, $key, $megakey, $conn) {
- $status = $detection ? "detection" : $src;
- $game = "NULL";
- $key = $key == "" ? "NULL" : "'{$key}'";
- $megakey = $megakey == "" ? "NULL" : "'{$megakey}'";
-
- if ($detection) {
- $status = "detection";
- $game = "@game_last";
- }
-
- // Check if key/megakey already exists, if so, skip insertion (no quotes on purpose)
- if ($detection)
- $existing_entry = $conn->query("SELECT id FROM fileset WHERE `key` = {$key}");
- else
- $existing_entry = $conn->query("SELECT id FROM fileset WHERE megakey = {$megakey}");
-
- if ($existing_entry->num_rows > 0) {
- if (!$detection)
- return false;
-
- $existing_entry = $existing_entry->fetch_array()[0];
- $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
- WHERE id = {$existing_entry}");
- $conn->query("UPDATE fileset SET status = 'detection'
- WHERE id = {$existing_entry} AND status = 'obsolete'");
- $conn->query("DELETE FROM game WHERE id = @game_last");
- return false;
- }
-
- // $game and $key should not be parsed as a mysql string, hence no quotes
- $query = "INSERT INTO fileset (game, status, src, `key`, megakey, `timestamp`)
- VALUES ({$game}, '{$status}', '{$src}', {$key}, {$megakey}, FROM_UNIXTIME(@fileset_time_last))";
- $conn->query($query);
- $conn->query("SET @fileset_last = LAST_INSERT_ID()");
-
- return true;
-}
-
-/**
- * Routine for inserting a file into the database, inserting into all
- * required tables
- * $file is an associated array (the contents of 'rom')
- * If checksum of the given checktype doesn't exists, silently fails
- */
-function insert_file($file, $detection, $src, $conn) {
- // Find md5-5000, or else use first checksum value
- $checksum = "";
- $checksize = 5000;
- if (isset($file["md5-5000"])) {
- $checksum = $file["md5-5000"];
- }
- else {
- foreach ($file as $key => $value) {
- if (strpos($key, "md5") !== false) {
- list($checksize, $checktype, $checksum) = get_checksum_props($key, $value);
- break;
- }
- }
- }
-
- $query = sprintf("INSERT INTO file (name, size, checksum, fileset, detection)
- VALUES ('%s', '%s', '%s', @fileset_last, %d)", mysqli_real_escape_string($conn, $file["name"]),
- $file["size"], $checksum, $detection);
- $conn->query($query);
-
- if ($detection)
- $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL");
- $conn->query("SET @file_last = LAST_INSERT_ID()");
-}
-
-function insert_filechecksum($file, $checktype, $conn) {
- if (!array_key_exists($checktype, $file))
- return;
-
- $checksum = $file[$checktype];
- list($checksize, $checktype, $checksum) = get_checksum_props($checktype, $checksum);
-
- $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
- VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $checksum);
- $conn->query($query);
-}
-
-/**
- * Merge two filesets without duplicating files
- * Used after matching an unconfirmed fileset with a detection entry
- */
-function merge_filesets($detection_id, $dat_id) {
- $conn = db_connect();
-
- $detection_files = $conn->query(sprintf("SELECT DISTINCT(filechecksum.checksum), checksize, checktype
- FROM filechecksum JOIN file on file.id = filechecksum.file
- WHERE fileset = '%d'", $detection_id))->fetch_all();
-
- foreach ($detection_files as $file) {
- $checksum = $file[0];
- $checksize = $file[1];
- $checktype = $file[2];
-
- // Delete original detection entry so newly matched fileset is the only fileset for game
- $conn->query(sprintf("DELETE FROM file
- WHERE checksum = '%s' AND fileset = %d LIMIT 1", $checksum, $detection_id));
-
- // Mark files present in the detection entries
- $conn->query(sprintf("UPDATE file
- JOIN filechecksum ON filechecksum.file = file.id
- SET detection = TRUE,
- checksize = %d,
- checktype = '%s'
- WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
- $checksize, $checktype, $dat_id, $checksum));
- }
-
- // Add fileset pair to history ($dat_id is the new fileset for $detection_id)
- $conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
- VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id));
- $history_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
-
- $conn->query("UPDATE history SET fileset = {$dat_id} WHERE fileset = {$detection_id}");
-
- // Delete original fileset
- $conn->query("DELETE FROM fileset WHERE id = {$detection_id}");
-
- if (!$conn->commit())
- echo "Error merging filesets";
-
- return $history_last;
-}
-
-/**
- * Create and return a mysqli connection
- */
-function db_connect() {
- $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
- $servername = $mysql_cred["servername"];
- $username = $mysql_cred["username"];
- $password = $mysql_cred["password"];
- $dbname = $mysql_cred["dbname"];
-
- // Create connection
- mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
- $conn = new mysqli($servername, $username, $password);
- $conn->set_charset('utf8mb4');
- $conn->autocommit(FALSE);
-
- // Check connection
- if ($conn->connect_errno) {
- die("Connect failed: " . $conn->connect_error);
- }
-
- $conn->query("USE " . $dbname);
-
- return $conn;
-}
-
-/**
- * Create an entry to the log table on each call of db_insert() or
- * populate_matching_games()
- */
-function create_log($category, $user, $text) {
- $conn = db_connect();
- $conn->query(sprintf("INSERT INTO log (`timestamp`, category, user, `text`)
- VALUES (FROM_UNIXTIME(%d), '%s', '%s', '%s')", time(), $category, $user, $text));
- $log_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
-
- if (!$conn->commit())
- echo "Creating log failed<br/>";
-
- return $log_last;
-}
-
-/**
- * Insert values from the associated array into the DB
- * They will be inserted under gameid NULL as the game itself is unconfirmed
- */
-function db_insert($data_arr) {
- $header = $data_arr[0];
- $game_data = $data_arr[1];
- $resources = $data_arr[2];
- $filepath = $data_arr[3];
-
- $conn = db_connect();
-
- /**
- * Author can be:
- * scummvm -> Detection Entries
- * scanner -> CLI scanner tool in python
- * _anything else_ -> DAT file
- */
- $author = $header["author"];
- $version = $header["version"];
-
- /**
- * status can be:
- * detection -> Detection entries (source of truth)
- * user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
- * scan -> Submitted by cli/scanner, unmatched
- * dat -> Submitted by DAT, unmatched
- * partialmatch -> Submitted by DAT, matched
- * fullmatch -> Submitted by cli/scanner, matched
- * obsolete -> Detection entries that are no longer part of the detection set
- */
- $src = "";
- if ($author == "scan" || $author == "scummvm")
- $src = $author;
- else
- $src = "dat";
-
- $detection = ($src == "scummvm");
- $status = $detection ? "detection" : $src;
-
- // Set timestamp of fileset insertion
- $conn->query(sprintf("SET @fileset_time_last = %d", time()));
-
- foreach ($game_data as $fileset) {
- if ($detection) {
- $engine_name = $fileset["engine"];
- $engineid = $fileset["sourcefile"];
- $gameid = $fileset["name"];
- $title = $fileset["title"];
- $extra = $fileset["extra"];
- $platform = $fileset["platform"];
- $lang = $fileset["language"];
-
- insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn);
- }
- elseif ($src == "dat")
- if (isset($fileset['romof']) && isset($resources[$fileset['romof']]))
- $fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
-
- $key = $detection ? calc_key($fileset['rom']) : "";
- $megakey = !$detection ? calc_key($fileset['rom']) : "";
- if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
- foreach ($fileset["rom"] as $file) {
- insert_file($file, $detection, $src, $conn);
- foreach ($file as $key => $value) {
- if ($key != "name" && $key != "size")
- insert_filechecksum($file, $key, $conn);
- }
- }
- }
- }
-
- if ($detection)
- $conn->query("UPDATE fileset SET status = 'obsolete'
- WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last)
- AND status = 'detection'");
-
- $category_text = "Uploaded from {$src}";
- $log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
- State '%s'. Fileset:%d.",
- $filepath, filesize($filepath), $author, $version, $status,
- $conn->query("SELECT @fileset_last")->fetch_array()[0]);
-
- if (!$conn->commit())
- echo "Inserting failed<br/>";
- else {
- $user = 'cli:' . get_current_user();
- create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
- }
-}
-
-function populate_matching_games() {
- $conn = db_connect();
-
- // Getting unmatched filesets
- $unmatched_filesets = array();
-
- $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status
- FROM fileset
- JOIN file ON file.fileset = fileset.id
- JOIN filechecksum ON file.id = filechecksum.file
- WHERE fileset.game IS NULL");
- $unmatched_files = $unmatched_files->fetch_all();
-
- // Splitting them into different filesets
- for ($i = 0; $i < count($unmatched_files); $i++) {
- $cur_fileset = $unmatched_files[$i][0];
- $temp = array();
- while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
- array_push($temp, $unmatched_files[$i]);
- $i++;
- }
- array_push($unmatched_filesets, $temp);
- }
-
- foreach ($unmatched_filesets as $fileset) {
- $matching_games = find_matching_game($fileset);
-
- if (count($matching_games) != 1) // If there is no match/non-unique match
- continue;
-
- $matched_game = $matching_games[0];
-
- // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
- $status = $fileset[0][2];
- if ($fileset[0][2] == "dat")
- $status = "partialmatch";
- elseif ($fileset[0][2] == "scan")
- $status = "fullmatch";
-
- // Convert NULL values to string with value NULL for printing
- $matched_game = array_map(function ($val) {
- return (is_null($val)) ? "NULL" : $val;
- }, $matched_game);
-
- $category_text = "Matched from {$fileset[0][2]}";
- $log_text = "Matched game {$matched_game['engineid']}:
- {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']}
- variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}.";
-
- // Updating the fileset.game value to be $matched_game["id"]
- $query = sprintf("UPDATE fileset
- SET game = %d, status = '%s', `key` = '%s'
- WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
-
- $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
-
- if ($conn->query($query)) {
- $user = 'cli:' . get_current_user();
- $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user,
- mysqli_real_escape_string($conn, $log_text));
-
- // Add log id to the history table
- $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
- }
-
- if (!$conn->commit())
- echo "Updating matched games failed<br/>";
- }
-}
-
// Process command line args
if ($index = array_search("--upload", $argv)) {
foreach (array_slice($argv, $index + 1) as $filepath) {
diff --git a/db_functions.php b/db_functions.php
new file mode 100644
index 0000000..c0d468c
--- /dev/null
+++ b/db_functions.php
@@ -0,0 +1,519 @@
+<?php
+
+/**
+ * Create and return a mysqli connection
+ */
+function db_connect() {
+ $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+ $servername = $mysql_cred["servername"];
+ $username = $mysql_cred["username"];
+ $password = $mysql_cred["password"];
+ $dbname = $mysql_cred["dbname"];
+
+ // Create connection
+ mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+ $conn = new mysqli($servername, $username, $password);
+ $conn->set_charset('utf8mb4');
+ $conn->autocommit(FALSE);
+
+ // Check connection
+ if ($conn->connect_errno) {
+ die("Connect failed: " . $conn->connect_error);
+ }
+
+ $conn->query("USE " . $dbname);
+
+ return $conn;
+}
+
+/**
+ * Retrieves the checksum and checktype of a given type + checksum
+ * eg: md5-5000 t:12345... -> 5000, md5-t, 12345...
+ */
+function get_checksum_props($checkcode, $checksum) {
+ $checksize = 0;
+ $checktype = $checkcode;
+ if (strpos($checkcode, '-') !== false) {
+ $temp = explode('-', $checkcode)[1];
+ if ($temp == '1M' || is_numeric($temp))
+ $checksize = $temp;
+ $checktype = explode('-', $checkcode)[0];
+ }
+
+ if (strpos($checksum, ':') !== false) {
+ $prefix = explode(':', $checksum)[0];
+
+ if (strpos($prefix, 't') !== false)
+ $checktype .= "-" . 't';
+ $checksum = explode(':', $checksum)[1];
+ }
+
+ return array($checksize, $checktype, $checksum);
+}
+
+/**
+ * Routine for inserting a game into the database - inserting into engine and
+ * game tables
+ */
+function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn) {
+ // Set @engine_last if engine already present in table
+ $exists = false;
+ if ($res = $conn->query(sprintf("SELECT id FROM engine WHERE engineid = '%s'", $engineid))) {
+ if ($res->num_rows > 0) {
+ $exists = true;
+ $conn->query(sprintf("SET @engine_last = '%d'", $res->fetch_array()[0]));
+ }
+ }
+
+ // Insert into table if not present
+ if (!$exists) {
+ $query = sprintf("INSERT INTO engine (name, engineid)
+ VALUES ('%s', '%s')", mysqli_real_escape_string($conn, $engine_name), $engineid);
+ $conn->query($query);
+ $conn->query("SET @engine_last = LAST_INSERT_ID()");
+ }
+
+ // Insert into game
+ $query = sprintf("INSERT INTO game (name, engine, gameid, extra, platform, language)
+ VALUES ('%s', @engine_last, '%s', '%s', '%s', '%s')", mysqli_real_escape_string($conn, $title),
+ $gameid, mysqli_real_escape_string($conn, $extra), $platform, $lang);
+ $conn->query($query);
+ $conn->query("SET @game_last = LAST_INSERT_ID()");
+}
+
+function insert_fileset($src, $detection, $key, $megakey, $conn) {
+ $status = $detection ? "detection" : $src;
+ $game = "NULL";
+ $key = $key == "" ? "NULL" : "'{$key}'";
+ $megakey = $megakey == "" ? "NULL" : "'{$megakey}'";
+
+ if ($detection) {
+ $status = "detection";
+ $game = "@game_last";
+ }
+
+ // Check if key/megakey already exists, if so, skip insertion (no quotes on purpose)
+ if ($detection)
+ $existing_entry = $conn->query("SELECT id FROM fileset WHERE `key` = {$key}");
+ else
+ $existing_entry = $conn->query("SELECT id FROM fileset WHERE megakey = {$megakey}");
+
+ if ($existing_entry->num_rows > 0) {
+ if (!$detection)
+ return false;
+
+ $existing_entry = $existing_entry->fetch_array()[0];
+ $conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
+ WHERE id = {$existing_entry}");
+ $conn->query("UPDATE fileset SET status = 'detection'
+ WHERE id = {$existing_entry} AND status = 'obsolete'");
+ $conn->query("DELETE FROM game WHERE id = @game_last");
+ return false;
+ }
+
+ // $game and $key should not be parsed as a mysql string, hence no quotes
+ $query = "INSERT INTO fileset (game, status, src, `key`, megakey, `timestamp`)
+ VALUES ({$game}, '{$status}', '{$src}', {$key}, {$megakey}, FROM_UNIXTIME(@fileset_time_last))";
+ $conn->query($query);
+ $conn->query("SET @fileset_last = LAST_INSERT_ID()");
+
+ return true;
+}
+
+/**
+ * Routine for inserting a file into the database, inserting into all
+ * required tables
+ * $file is an associated array (the contents of 'rom')
+ * If checksum of the given checktype doesn't exists, silently fails
+ */
+function insert_file($file, $detection, $src, $conn) {
+ // Find md5-5000, or else use first checksum value
+ $checksum = "";
+ $checksize = 5000;
+ if (isset($file["md5-5000"])) {
+ $checksum = $file["md5-5000"];
+ }
+ else {
+ foreach ($file as $key => $value) {
+ if (strpos($key, "md5") !== false) {
+ list($checksize, $checktype, $checksum) = get_checksum_props($key, $value);
+ break;
+ }
+ }
+ }
+
+ $query = sprintf("INSERT INTO file (name, size, checksum, fileset, detection)
+ VALUES ('%s', '%s', '%s', @fileset_last, %d)", mysqli_real_escape_string($conn, $file["name"]),
+ $file["size"], $checksum, $detection);
+ $conn->query($query);
+
+ if ($detection)
+ $conn->query("UPDATE fileset SET detection_size = {$checksize} WHERE id = @fileset_last AND detection_size IS NULL");
+ $conn->query("SET @file_last = LAST_INSERT_ID()");
+}
+
+function insert_filechecksum($file, $checktype, $conn) {
+ if (!array_key_exists($checktype, $file))
+ return;
+
+ $checksum = $file[$checktype];
+ list($checksize, $checktype, $checksum) = get_checksum_props($checktype, $checksum);
+
+ $query = sprintf("INSERT INTO filechecksum (file, checksize, checktype, checksum)
+ VALUES (@file_last, '%s', '%s', '%s')", $checksize, $checktype, $checksum);
+ $conn->query($query);
+}
+
+/**
+ * Create an entry to the log table on each call of db_insert() or
+ * populate_matching_games()
+ */
+function create_log($category, $user, $text) {
+ $conn = db_connect();
+ $conn->query(sprintf("INSERT INTO log (`timestamp`, category, user, `text`)
+ VALUES (FROM_UNIXTIME(%d), '%s', '%s', '%s')", time(), $category, $user, $text));
+ $log_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
+
+ if (!$conn->commit())
+ echo "Creating log failed<br/>";
+
+ return $log_last;
+}
+
+/**
+ * Calculate `key` value as md5("file1:size:md5:file2:...")
+ */
+function calc_key($files) {
+ $key_string = "";
+ foreach ($files as $file) {
+ foreach ($file as $key => $value) {
+ $key_string .= ':' . $value;
+ }
+ }
+ $key_string = trim($key_string, ':');
+ return md5($key_string);
+}
+
+/**
+ * Insert values from the associated array into the DB
+ * They will be inserted under gameid NULL as the game itself is unconfirmed
+ */
+function db_insert($data_arr) {
+ $header = $data_arr[0];
+ $game_data = $data_arr[1];
+ $resources = $data_arr[2];
+ $filepath = $data_arr[3];
+
+ $conn = db_connect();
+
+ /**
+ * Author can be:
+ * scummvm -> Detection Entries
+ * scanner -> CLI scanner tool in python
+ * _anything else_ -> DAT file
+ */
+ $author = $header["author"];
+ $version = $header["version"];
+
+ /**
+ * status can be:
+ * detection -> Detection entries (source of truth)
+ * user -> Submitted by users via ScummVM, unmatched (Not used in the parser)
+ * scan -> Submitted by cli/scanner, unmatched
+ * dat -> Submitted by DAT, unmatched
+ * partialmatch -> Submitted by DAT, matched
+ * fullmatch -> Submitted by cli/scanner, matched
+ * obsolete -> Detection entries that are no longer part of the detection set
+ */
+ $src = "";
+ if ($author == "scan" || $author == "scummvm")
+ $src = $author;
+ else
+ $src = "dat";
+
+ $detection = ($src == "scummvm");
+ $status = $detection ? "detection" : $src;
+
+ // Set timestamp of fileset insertion
+ $conn->query(sprintf("SET @fileset_time_last = %d", time()));
+
+ foreach ($game_data as $fileset) {
+ if ($detection) {
+ $engine_name = $fileset["engine"];
+ $engineid = $fileset["sourcefile"];
+ $gameid = $fileset["name"];
+ $title = $fileset["title"];
+ $extra = $fileset["extra"];
+ $platform = $fileset["platform"];
+ $lang = $fileset["language"];
+
+ insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform, $lang, $conn);
+ }
+ elseif ($src == "dat")
+ if (isset($fileset['romof']) && isset($resources[$fileset['romof']]))
+ $fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
+
+ $key = $detection ? calc_key($fileset['rom']) : "";
+ $megakey = !$detection ? calc_key($fileset['rom']) : "";
+ if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
+ foreach ($fileset["rom"] as $file) {
+ insert_file($file, $detection, $src, $conn);
+ foreach ($file as $key => $value) {
+ if ($key != "name" && $key != "size")
+ insert_filechecksum($file, $key, $conn);
+ }
+ }
+ }
+ }
+
+ if ($detection)
+ $conn->query("UPDATE fileset SET status = 'obsolete'
+ WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last)
+ AND status = 'detection'");
+
+ $category_text = "Uploaded from {$src}";
+ $log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
+ State '%s'. Fileset:%d.",
+ $filepath, filesize($filepath), $author, $version, $status,
+ $conn->query("SELECT @fileset_last")->fetch_array()[0]);
+
+ if (!$conn->commit())
+ echo "Inserting failed<br/>";
+ else {
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
+ }
+}
+
+/**
+ * Compare 2 dat filesets to find if they are equivalent or not
+ */
+function compare_filesets($id1, $id2, $conn) {
+ $fileset1 = $conn->query("SELECT name, size, checksum
+ FROM file WHERE fileset = '{$id1}'")->fetch_all();
+ $fileset2 = $conn->query("SELECT name, size, checksum
+ FROM file WHERE fileset = '{$id2}'")->fetch_all();
+
+ // Sort filesets on checksum
+ usort($fileset1, function ($a, $b) {
+ return $a[2] <=> $b[2];
+ });
+ usort($fileset2, function ($a, $b) {
+ return $a[2] <=> $b[2];
+ });
+
+ if (count($fileset1) != count($fileset2))
+ return false;
+
+ for ($i = 0; $i < count($fileset1); $i++) {
+ // If checksums do not match
+ if ($fileset1[2] != $fileset2[2])
+ return false;
+ }
+
+ return True;
+}
+
+/**
+ * Return fileset statuses that can be merged with set of given status
+ * eg: scan and dat -> detection
+ * fullmatch -> partialmatch, detection
+ */
+function status_to_match($status) {
+ $order = array("detection", "dat", "scan", "partialmatch", "fullmatch");
+ return array_slice($order, 0, array_search($status, $order));
+}
+
+/**
+ * Detects games based on the file descriptions in $dat_arr
+ * Compares the files with those in the detection entries table
+ * $game_files consists of both the game ( ) and resources ( ) parts
+ */
+function find_matching_game($game_files) {
+ $matching_games = array(); // All matching games
+ $matching_filesets = array(); // All filesets containing one file from $game_files
+ $matches_count = 0; // Number of files with a matching detection entry
+
+ $conn = db_connect();
+
+ foreach ($game_files as $file) {
+ $checksum = $file[1];
+
+ $query = "SELECT file.fileset as file_fileset
+ FROM filechecksum
+ JOIN file ON filechecksum.file = file.id
+ WHERE filechecksum.checksum = '{$checksum}' AND file.detection = TRUE";
+ $records = $conn->query($query)->fetch_all();
+
+ // If file is not part of detection entries, skip it
+ if (count($records) == 0)
+ continue;
+
+ $matches_count++;
+ foreach ($records as $record)
+ array_push($matching_filesets, $record[0]);
+ } // Check if there is a fileset_id that is present in all results
+ foreach (array_count_values($matching_filesets) as $key => $value) {
+ $count_files_in_fileset = $conn->query(sprintf("SELECT COUNT(file.id) FROM file
+ JOIN fileset ON file.fileset = fileset.id
+ WHERE fileset.id = '%s'", $key))->fetch_array()[0];
+
+ // We use < instead of != since one file may have more than one entry in the fileset
+ // We see this in Drascula English version, where one entry is duplicated
+ if ($value < $matches_count || $value < $count_files_in_fileset)
+ continue;
+
+ $records = $conn->query(sprintf("SELECT engineid, game.id, gameid, platform,
+ language, `key`, src, fileset.id as fileset
+ FROM game
+ JOIN fileset ON fileset.game = game.id
+ JOIN engine ON engine.id = game.engine
+ WHERE fileset.id = '%s'", $key));
+
+ array_push($matching_games, $records->fetch_array());
+ }
+
+ if (count($matching_games) != 1)
+ return $matching_games;
+
+ // Check the current fileset priority with that of the match
+ $records = $conn->query(sprintf("SELECT id FROM fileset, ({$query}) AS res
+ WHERE id = file_fileset AND
+ status IN ('%s')", implode("', '", status_to_match($game_files[3]))));
+
+ // If priority order is correct
+ if ($records->num_rows != 0)
+ return $matching_games;
+
+ if (compare_filesets($matching_games[0]['fileset'], $game_files[0][0], $conn)) {
+ $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0]}");
+ return array();
+ }
+
+ return $matching_games;
+}
+
+/**
+ * Merge two filesets without duplicating files
+ * Used after matching an unconfirmed fileset with a detection entry
+ */
+function merge_filesets($detection_id, $dat_id) {
+ $conn = db_connect();
+
+ $detection_files = $conn->query(sprintf("SELECT DISTINCT(filechecksum.checksum), checksize, checktype
+ FROM filechecksum JOIN file on file.id = filechecksum.file
+ WHERE fileset = '%d'", $detection_id))->fetch_all();
+
+ foreach ($detection_files as $file) {
+ $checksum = $file[0];
+ $checksize = $file[1];
+ $checktype = $file[2];
+
+ // Delete original detection entry so newly matched fileset is the only fileset for game
+ $conn->query(sprintf("DELETE FROM file
+ WHERE checksum = '%s' AND fileset = %d LIMIT 1", $checksum, $detection_id));
+
+ // Mark files present in the detection entries
+ $conn->query(sprintf("UPDATE file
+ JOIN filechecksum ON filechecksum.file = file.id
+ SET detection = TRUE,
+ checksize = %d,
+ checktype = '%s'
+ WHERE fileset = '%d' AND filechecksum.checksum = '%s'",
+ $checksize, $checktype, $dat_id, $checksum));
+ }
+
+ // Add fileset pair to history ($dat_id is the new fileset for $detection_id)
+ $conn->query(sprintf("INSERT INTO history (`timestamp`, fileset, oldfileset)
+ VALUES (FROM_UNIXTIME(%d), %d, %d)", time(), $dat_id, $detection_id));
+ $history_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
+
+ $conn->query("UPDATE history SET fileset = {$dat_id} WHERE fileset = {$detection_id}");
+
+ // Delete original fileset
+ $conn->query("DELETE FROM fileset WHERE id = {$detection_id}");
+
+ if (!$conn->commit())
+ echo "Error merging filesets";
+
+ return $history_last;
+}
+
+/**
+ * (Attempt to) match fileset that have fileset.game as NULL
+ * This will delete the original detection fileset and replace it with the newly
+ * matched fileset
+ */
+function populate_matching_games() {
+ $conn = db_connect();
+
+ // Getting unmatched filesets
+ $unmatched_filesets = array();
+
+ $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status
+ FROM fileset
+ JOIN file ON file.fileset = fileset.id
+ JOIN filechecksum ON file.id = filechecksum.file
+ WHERE fileset.game IS NULL");
+ $unmatched_files = $unmatched_files->fetch_all();
+
+ // Splitting them into different filesets
+ for ($i = 0; $i < count($unmatched_files); $i++) {
+ $cur_fileset = $unmatched_files[$i][0];
+ $temp = array();
+ while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
+ array_push($temp, $unmatched_files[$i]);
+ $i++;
+ }
+ array_push($unmatched_filesets, $temp);
+ }
+
+ foreach ($unmatched_filesets as $fileset) {
+ $matching_games = find_matching_game($fileset);
+
+ if (count($matching_games) != 1) // If there is no match/non-unique match
+ continue;
+
+ $matched_game = $matching_games[0];
+
+ // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
+ $status = $fileset[0][2];
+ if ($fileset[0][2] == "dat")
+ $status = "partialmatch";
+ elseif ($fileset[0][2] == "scan")
+ $status = "fullmatch";
+
+ // Convert NULL values to string with value NULL for printing
+ $matched_game = array_map(function ($val) {
+ return (is_null($val)) ? "NULL" : $val;
+ }, $matched_game);
+
+ $category_text = "Matched from {$fileset[0][2]}";
+ $log_text = "Matched game {$matched_game['engineid']}:
+ {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']}
+ variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}.";
+
+ // Updating the fileset.game value to be $matched_game["id"]
+ $query = sprintf("UPDATE fileset
+ SET game = %d, status = '%s', `key` = '%s'
+ WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
+
+ $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
+
+ if ($conn->query($query)) {
+ $user = 'cli:' . get_current_user();
+ $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user,
+ mysqli_real_escape_string($conn, $log_text));
+
+ // Add log id to the history table
+ $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
+ }
+
+ if (!$conn->commit())
+ echo "Updating matched games failed<br/>";
+ }
+}
+
+
+?>
+
Commit: 7ffbef0a5043b329d284885f260efe8cb3f94e5f
https://github.com/scummvm/scummvm-sites/commit/7ffbef0a5043b329d284885f260efe8cb3f94e5f
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T21:17:28+05:30
Commit Message:
INTEGRITY: Update error logs to end with newlines
Changed paths:
db_functions.php
diff --git a/db_functions.php b/db_functions.php
index c0d468c..2319cc2 100644
--- a/db_functions.php
+++ b/db_functions.php
@@ -175,7 +175,7 @@ function create_log($category, $user, $text) {
$log_last = $conn->query("SELECT LAST_INSERT_ID()")->fetch_array()[0];
if (!$conn->commit())
- echo "Creating log failed<br/>";
+ echo "Creating log failed\n";
return $log_last;
}
@@ -278,7 +278,7 @@ function db_insert($data_arr) {
$conn->query("SELECT @fileset_last")->fetch_array()[0]);
if (!$conn->commit())
- echo "Inserting failed<br/>";
+ echo "Inserting failed\n";
else {
$user = 'cli:' . get_current_user();
create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
@@ -434,7 +434,7 @@ function merge_filesets($detection_id, $dat_id) {
$conn->query("DELETE FROM fileset WHERE id = {$detection_id}");
if (!$conn->commit())
- echo "Error merging filesets";
+ echo "Error merging filesets\n";
return $history_last;
}
@@ -510,7 +510,7 @@ function populate_matching_games() {
}
if (!$conn->commit())
- echo "Updating matched games failed<br/>";
+ echo "Updating matched games failed\n";
}
}
Commit: e8519ab2b8b6c8b1069b4e0f6e3b0e820f158d2e
https://github.com/scummvm/scummvm-sites/commit/e8519ab2b8b6c8b1069b4e0f6e3b0e820f158d2e
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T22:21:02+05:30
Commit Message:
INTEGRITY: Improve formatting for schema.php
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index 68add30..a3cf706 100644
--- a/schema.php
+++ b/schema.php
@@ -169,6 +169,9 @@ else {
echo "Error creating 'history' table: " . $conn->error;
}
+
+///////////////////////// CREATE INDEX /////////////////////////
+
// Create indices for fast data retrieval
// PK and FK are automatically indexed in InnoDB, so they are not included
$index = "CREATE INDEX detection ON file (detection)";
Commit: a3bb29e70184271548c923d332d74f38dad03d1b
https://github.com/scummvm/scummvm-sites/commit/a3bb29e70184271548c923d332d74f38dad03d1b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T23:27:00+05:30
Commit Message:
INTEGRITY: Update format of checkcodes
Changed paths:
db_functions.php
diff --git a/db_functions.php b/db_functions.php
index 2319cc2..400a546 100644
--- a/db_functions.php
+++ b/db_functions.php
@@ -33,18 +33,21 @@ function db_connect() {
function get_checksum_props($checkcode, $checksum) {
$checksize = 0;
$checktype = $checkcode;
+
if (strpos($checkcode, '-') !== false) {
- $temp = explode('-', $checkcode)[1];
- if ($temp == '1M' || is_numeric($temp))
- $checksize = $temp;
- $checktype = explode('-', $checkcode)[0];
+ $exploded_checkcode = explode('-', $checkcode);
+ $last = array_pop($exploded_checkcode);
+ if ($last == '1M' || is_numeric($last))
+ $checksize = $last;
+
+ $checktype = implode('-', $exploded_checkcode);
}
+ // Detection entries have checktypes as part of the checksum prefix
if (strpos($checksum, ':') !== false) {
$prefix = explode(':', $checksum)[0];
+ $checktype .= "-" . $prefix;
- if (strpos($prefix, 't') !== false)
- $checktype .= "-" . 't';
$checksum = explode(':', $checksum)[1];
}
Commit: ecaadeafa11440005a60d734b59fec70eb4be84f
https://github.com/scummvm/scummvm-sites/commit/ecaadeafa11440005a60d734b59fec70eb4be84f
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-25T23:27:37+05:30
Commit Message:
INTEGRITY: Add functions to add user fileset to db
Changed paths:
checksum_validator.php
diff --git a/checksum_validator.php b/checksum_validator.php
index 354746a..a04c010 100644
--- a/checksum_validator.php
+++ b/checksum_validator.php
@@ -1,19 +1,7 @@
<?php
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
-$servername = $mysql_cred["servername"];
-$username = $mysql_cred["username"];
-$password = $mysql_cred["password"];
-$dbname = $mysql_cred["dbname"];
-
-// Create connection
-$conn = new mysqli($servername, $username, $password);
-
-// Check connection
-if ($conn->connect_errno) {
- die("Connect failed: " . $conn->connect_error);
-}
+require('db_functions.php');
-$conn->query("USE " . $dbname);
+$conn = db_connect();
$json_string = file_get_contents('sample_json_request.json');
$json_object = json_decode($json_string);
@@ -115,6 +103,60 @@ while ($game = $games->fetch_array()) {
$json_response = json_encode($json_response);
+function user_calc_key($user_fileset) {
+ $key_string = "";
+ foreach ($user_fileset as $file) {
+ foreach ($file as $key => $value) {
+ if ($key != 'checksums') {
+ $key_string .= ':' . $value;
+ continue;
+ }
+
+ foreach ($value as $checksum_pair)
+ $key_string .= ':' . $checksum_pair->checksum;
+ }
+ }
+ $key_string = trim($key_string, ':');
+
+ return md5($key_string);
+}
+
+function file_json_to_array($file_json_object) {
+ $res = array();
+
+ foreach ($file_json_object as $key => $value) {
+ if ($key != 'checksums') {
+ $res[$key] = $value;
+ continue;
+ }
+
+ foreach ($value as $checksum_pair)
+ $res[$checksum_pair->type] = $checksum_pair->checksum;
+ }
+
+ return $res;
+}
+
+function user_insert_fileset($user_fileset, $conn) {
+ $src = 'user';
+ $detection = 'false';
+ $key = '';
+ $megakey = user_calc_key($user_fileset);
+ $conn = db_connect();
+
+ if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
+ foreach ($user_fileset as $file) {
+ $file = file_json_to_array($file);
+
+ insert_file($file, $detection, $src, $conn);
+ foreach ($file as $key => $value) {
+ if ($key != "name" && $key != "size")
+ insert_filechecksum($file, $key, $conn);
+ }
+ }
+ }
+}
+
$conn->close();
?>
Commit: 358e252540fb6a7ed40a1fc65d3c639aa8829b1c
https://github.com/scummvm/scummvm-sites/commit/358e252540fb6a7ed40a1fc65d3c639aa8829b1c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-26T12:59:29+05:30
Commit Message:
INTEGRITY: Change format of checkcode
Also remove redundant prefix from checksum
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index a604eed..5961f70 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -184,11 +184,21 @@ def macbin_get_datafork(file_byte_stream):
def create_checksum_pairs(hashes, alg, size, prefix=None):
res = []
- keys = [f"{alg}", f"{alg}-5000", f"{alg}-1M", f"{alg}-5000-t"]
+ keys = [f"{alg}", f"{alg}-5000", f"{alg}-1M", f"{alg}-t-5000"]
+
if size:
keys.append(f"{alg}-{size}")
if prefix:
- keys = [key+f'-{prefix}' for key in keys]
+ for i, key in enumerate(keys):
+ key_split = key.split('-')
+
+ # If key is of the form "md5-t-5000"
+ if (len(key_split) == 3):
+ key_split[1] = f"{prefix}{key_split[1]}"
+ else:
+ key_split.insert(1, prefix)
+
+ keys[i] = '-'.join(key_split)
for i, h in enumerate(hashes):
res.append((keys[i], h))
@@ -205,32 +215,18 @@ def file_checksum(filepath, alg, size):
with open(filepath, "rb") as f:
res = []
- hashes = []
file = macbin_get_resfork(f.read())
+ hashes = checksum(file, alg, size, filepath)
prefix = 'r'
if len(file):
- for h in checksum(file, alg, size, filepath):
- if ':' not in h:
- hashes.append(f"{prefix}:{h}")
- else:
- # If the checksum is like "t:..."
- hashes.append(f"{prefix}{h}")
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
- res.extend(create_checksum_pairs(hashes, alg, size, prefix))
-
- hashes = []
f.seek(0)
file = macbin_get_datafork(f.read())
+ hashes = checksum(file, alg, size, filepath)
prefix = 'd'
- for h in checksum(file, alg, size, filepath):
- if ':' not in h:
- hashes.append(f"{prefix}:{h}")
- else:
- # If the checksum is like "t:..."
- hashes.append(f"{prefix}{h}")
-
res.extend(create_checksum_pairs(hashes, alg, size, prefix))
return res
@@ -300,7 +296,6 @@ def checksum(file, alg, size, filepath):
hashes[4] = None
hashes = [h.hexdigest() for h in hashes if h]
- hashes[3] = 't:' + hashes[3] # Add tail prefix
return hashes
Commit: 0e047b2bdf67aae258621e09b03983270ae50a19
https://github.com/scummvm/scummvm-sites/commit/0e047b2bdf67aae258621e09b03983270ae50a19
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-26T13:13:25+05:30
Commit Message:
INTEGRITY: Fix punyencoding for filepaths
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 5961f70..dabb308 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -118,6 +118,13 @@ def punyencode(orig: str) -> str:
return orig
+def punyencode_filepath(filepath):
+ filepath = filepath.rstrip("/")
+ prefix, filename = os.path.split(filepath)
+
+ return os.path.join(prefix, punyencode(filename))
+
+
def read_be_32(byte_stream):
""" Return unsigned integer of size_in_bits, assuming the data is big-endian """
(uint,) = struct.unpack(">I", byte_stream[:32//8])
@@ -334,7 +341,7 @@ def create_dat_file(hash_of_dirs, path, checksum_size=0):
for hash_of_dir in hash_of_dirs:
file.write("game (\n")
for filename, (hashes, filesize) in hash_of_dir.items():
- filename = (punyencode(filename)
+ filename = (punyencode_filepath(filename)
if needs_punyencoding(filename) else filename)
data = f"name \"{filename}\" size {filesize}"
for key, value in hashes:
Commit: 452ad3c9df6ae6b939509c132a9abd4860154e64
https://github.com/scummvm/scummvm-sites/commit/452ad3c9df6ae6b939509c132a9abd4860154e64
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-26T13:38:34+05:30
Commit Message:
INTEGRITY: Support 'm:' type checksums in scanner
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index dabb308..c462f34 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -222,18 +222,22 @@ def file_checksum(filepath, alg, size):
with open(filepath, "rb") as f:
res = []
- file = macbin_get_resfork(f.read())
- hashes = checksum(file, alg, size, filepath)
- prefix = 'r'
+ resfork = macbin_get_resfork(f.read())
+ f.seek(0)
+ datafork = macbin_get_datafork(f.read())
+ combined_forks = datafork + resfork
- if len(file):
+ hashes = checksum(resfork, alg, size, filepath)
+ prefix = 'r'
+ if len(resfork):
res.extend(create_checksum_pairs(hashes, alg, size, prefix))
- f.seek(0)
- file = macbin_get_datafork(f.read())
- hashes = checksum(file, alg, size, filepath)
+ hashes = checksum(datafork, alg, size, filepath)
prefix = 'd'
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
+ hashes = checksum(combined_forks, alg, size, filepath)
+ prefix = 'm'
res.extend(create_checksum_pairs(hashes, alg, size, prefix))
return res
Commit: 0358983805bc02adbcefb65a3726f9023e7581ac
https://github.com/scummvm/scummvm-sites/commit/0358983805bc02adbcefb65a3726f9023e7581ac
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-26T16:23:43+05:30
Commit Message:
INTEGRITY: Fix macbin_get_resfork()
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index c462f34..74f61c0 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -159,7 +159,6 @@ def is_macbin(filepath):
datalen = read_be_32(header[83:])
rsrclen = read_be_32(header[87:])
- # Files produced by ISOBuster are not padded, thus, compare with the actual size
datalen_pad = (((datalen + 127) >> 7) << 7)
# Length check
@@ -177,7 +176,10 @@ def macbin_get_resfork(file_byte_stream):
return file_byte_stream
(datalen,) = struct.unpack(">I", file_byte_stream[0x53:0x57])
- return file_byte_stream[0x80 + datalen:]
+ datalen_padded = ((datalen + 127) >> 7) << 7
+ (rsrclen,) = struct.unpack(">I", file_byte_stream[0x57:0x5B])
+
+ return file_byte_stream[0x80 + datalen_padded: 0x80 + datalen_padded + rsrclen]
def macbin_get_datafork(file_byte_stream):
Commit: d6541a10a98b700c630e0f41a82a9fb10d445851
https://github.com/scummvm/scummvm-sites/commit/d6541a10a98b700c630e0f41a82a9fb10d445851
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-26T22:40:38+05:30
Commit Message:
INTEGRITY: Add support for AppleDouble and .rsrc files
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 74f61c0..fcae5b3 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -190,6 +190,83 @@ def macbin_get_datafork(file_byte_stream):
return file_byte_stream[0x80: 0x80 + datalen]
+def is_appledouble(file_byte_stream):
+ # Check AppleDouble magic number
+ if (not file_byte_stream or read_be_32(file_byte_stream) != 0x00051607):
+ return False
+
+ return True
+
+
+def appledouble_get_resfork(file_byte_stream):
+ entry_count = read_be_32(file_byte_stream[24:])
+ for _ in range(entry_count):
+ id = read_be_32(file_byte_stream[28:])
+ offset = read_be_32(file_byte_stream[32:])
+ length = read_be_32(file_byte_stream[36:])
+
+ if id == 2:
+ return file_byte_stream[offset:offset+length]
+
+ return b''
+
+
+def appledouble_get_datafork(filepath):
+ return open(filepath[2:], "rb")
+
+
+def rsrc_get_datafork(filepath):
+ return b''
+
+
+def file_checksum(filepath, alg, size):
+ filename = os.path.basename(filepath)
+
+ # If it is Apple file with 2 forks
+ if (filepath.endswith('.rsrc') or is_macbin(filepath) or
+ filename.startswith('._') or filename.startswith('__MACOS')):
+ res = []
+ resfork = b''
+ datafork = b''
+
+ with open(filepath, "rb") as f:
+ if is_appledouble(f.read()):
+ f.seek(0)
+ resfork = appledouble_get_resfork(f.read())
+ datafork = appledouble_get_datafork(filepath)
+
+ elif is_macbin(filepath):
+ f.seek(0)
+ resfork = macbin_get_resfork(f.read())
+ f.seek(0)
+ datafork = macbin_get_datafork(f.read())
+
+ elif filepath.endswith('.rsrc'):
+ resfork = f.read()
+ datafork = rsrc_get_datafork(filepath)
+
+ combined_forks = datafork + resfork
+
+ hashes = checksum(resfork, alg, size, filepath)
+ prefix = 'r'
+ if len(resfork):
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
+
+ hashes = checksum(datafork, alg, size, filepath)
+ prefix = 'd'
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
+
+ hashes = checksum(combined_forks, alg, size, filepath)
+ prefix = 'm'
+ res.extend(create_checksum_pairs(hashes, alg, size, prefix))
+
+ return res
+
+ # If it is a normal file
+ with open(filepath, "rb") as file:
+ return create_checksum_pairs(checksum(file, alg, size, filepath), alg, size)
+
+
def create_checksum_pairs(hashes, alg, size, prefix=None):
res = []
@@ -215,36 +292,6 @@ def create_checksum_pairs(hashes, alg, size, prefix=None):
return res
-def file_checksum(filepath, alg, size):
- if not is_macbin(filepath):
- with open(filepath, "rb") as file:
- return create_checksum_pairs(checksum(file, alg, size, filepath), alg, size)
-
- # If the file is a MacBinary
- with open(filepath, "rb") as f:
- res = []
-
- resfork = macbin_get_resfork(f.read())
- f.seek(0)
- datafork = macbin_get_datafork(f.read())
- combined_forks = datafork + resfork
-
- hashes = checksum(resfork, alg, size, filepath)
- prefix = 'r'
- if len(resfork):
- res.extend(create_checksum_pairs(hashes, alg, size, prefix))
-
- hashes = checksum(datafork, alg, size, filepath)
- prefix = 'd'
- res.extend(create_checksum_pairs(hashes, alg, size, prefix))
-
- hashes = checksum(combined_forks, alg, size, filepath)
- prefix = 'm'
- res.extend(create_checksum_pairs(hashes, alg, size, prefix))
-
- return res
-
-
def checksum(file, alg, size, filepath):
""" Returns checksum value of file buffer using a specific algoritm """
# Will contain 5 elements:
Commit: 9ac93e363de24149278c04b00d94dc1342e47ec2
https://github.com/scummvm/scummvm-sites/commit/9ac93e363de24149278c04b00d94dc1342e47ec2
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T17:36:22+05:30
Commit Message:
INTEGRITY: Update datafork for AppleDouble, .rsrc
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index fcae5b3..7657cd2 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -212,13 +212,31 @@ def appledouble_get_resfork(file_byte_stream):
def appledouble_get_datafork(filepath):
- return open(filepath[2:], "rb")
+ try:
+ index = filepath.index("__MACOSX")
+ except ValueError:
+ index = None
+ if index is not None:
+ # Remove '__MACOSX/' from filepath
+ filepath = filepath[:index] + filepath[index+8+1:]
+
+ # Remove '._' from filepath
+ filename = os.path.basename(filepath)
+ filepath = filepath[:-len(filename)] + (filename[2:]
+ if filename.startswith('._') else filename)
+
+ return filepath
+ # return open(filepath, "rb")
-def rsrc_get_datafork(filepath):
return b''
+def rsrc_get_datafork(filepath):
+ # Data fork is the same filename without the .rsrc extension
+ return open(filepath[:-5], "rb")
+
+
def file_checksum(filepath, alg, size):
filename = os.path.basename(filepath)
@@ -230,21 +248,21 @@ def file_checksum(filepath, alg, size):
datafork = b''
with open(filepath, "rb") as f:
+ if filepath.endswith('.rsrc'):
+ resfork = f.read()
+ datafork = rsrc_get_datafork(filepath)
+
if is_appledouble(f.read()):
f.seek(0)
resfork = appledouble_get_resfork(f.read())
datafork = appledouble_get_datafork(filepath)
- elif is_macbin(filepath):
+ if is_macbin(filepath):
f.seek(0)
resfork = macbin_get_resfork(f.read())
f.seek(0)
datafork = macbin_get_datafork(f.read())
- elif filepath.endswith('.rsrc'):
- resfork = f.read()
- datafork = rsrc_get_datafork(filepath)
-
combined_forks = datafork + resfork
hashes = checksum(resfork, alg, size, filepath)
Commit: e4ce93644b034ba1cbbcf9a4bca0a1c771567761
https://github.com/scummvm/scummvm-sites/commit/e4ce93644b034ba1cbbcf9a4bca0a1c771567761
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T17:36:31+05:30
Commit Message:
INTEGRITY: Skip crc and sha1 checksums from DATs
Changed paths:
dat_parser.php
diff --git a/dat_parser.php b/dat_parser.php
index 9ea46b6..34dd305 100644
--- a/dat_parser.php
+++ b/dat_parser.php
@@ -23,6 +23,9 @@ function map_checksum_data($content_string) {
if ($temp[$i] == ')')
continue;
+ if ($temp[$i] == 'crc' || $temp[$i] == 'sha1')
+ continue;
+
$temp[$i + 1] = remove_quotes($temp[$i + 1]);
if ($temp[$i + 1] == ')')
$temp[$i + 1] = "";
Commit: 6984317465eff61c1dd4645191c273e66de9b574
https://github.com/scummvm/scummvm-sites/commit/6984317465eff61c1dd4645191c273e66de9b574
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T17:36:31+05:30
Commit Message:
INTEGRITY: Use full md5 in file.checksum
Changed paths:
db_functions.php
diff --git a/db_functions.php b/db_functions.php
index 400a546..3112112 100644
--- a/db_functions.php
+++ b/db_functions.php
@@ -130,11 +130,11 @@ function insert_fileset($src, $detection, $key, $megakey, $conn) {
* If checksum of the given checktype doesn't exists, silently fails
*/
function insert_file($file, $detection, $src, $conn) {
- // Find md5-5000, or else use first checksum value
+ // Find full md5, or else use first checksum value
$checksum = "";
$checksize = 5000;
- if (isset($file["md5-5000"])) {
- $checksum = $file["md5-5000"];
+ if (isset($file["md5"])) {
+ $checksum = $file["md5"];
}
else {
foreach ($file as $key => $value) {
Commit: 7d9c180bd9ac618ba357ed514242775d77aabcbf
https://github.com/scummvm/scummvm-sites/commit/7d9c180bd9ac618ba357ed514242775d77aabcbf
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T18:06:54+05:30
Commit Message:
INTEGRITY: Fix $num_of_results while filtering
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 98ce68e..19aad60 100644
--- a/pagination.php
+++ b/pagination.php
@@ -44,12 +44,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// If there exist get variables that are for filtering
$_GET = array_filter($_GET);
- // If $records_table has a JOIN (multiple tables)
- if (preg_match("/JOIN/", $records_table) !== false) {
- $first_table = explode(" ", $records_table)[0];
- $num_of_results = $conn->query("SELECT COUNT({$first_table}.id) FROM {$records_table}")->fetch_array()[0];
- }
- elseif (array_diff(array_keys($_GET), array('page'))) {
+ if (array_diff(array_keys($_GET), array('page'))) {
$condition = "WHERE ";
$tables = array();
foreach ($_GET as $key => $value) {
@@ -75,6 +70,11 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$num_of_results = $conn->query(
"SELECT COUNT({$records_table}.id) FROM {$from_query} {$condition}")->fetch_array()[0];
}
+ // If $records_table has a JOIN (multiple tables)
+ elseif (preg_match("/JOIN/", $records_table) !== false) {
+ $first_table = explode(" ", $records_table)[0];
+ $num_of_results = $conn->query("SELECT COUNT({$first_table}.id) FROM {$records_table}")->fetch_array()[0];
+ }
else {
$num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
}
Commit: e915c4f81ace76ed86169ce8130d40f779e3b922
https://github.com/scummvm/scummvm-sites/commit/e915c4f81ace76ed86169ce8130d40f779e3b922
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T18:17:46+05:30
Commit Message:
INTEGRITY: Skip showing md5-0 in widetable fileset
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index e7aa284..43c9a08 100644
--- a/fileset.php
+++ b/fileset.php
@@ -122,6 +122,10 @@ if (isset($_GET['widetable']) && $_GET['widetable'] == 'true') {
FROM filechecksum WHERE file = {$file['id']}");
while ($spec_checksum = $spec_checksum_res->fetch_assoc()) {
+ // md5-0 is skipped since it is already shown as file.checksum
+ if ($spec_checksum['checksize'] == 0)
+ continue;
+
$result[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum'];
}
}
Commit: d7d1d39dafbb9aeec2ade4a429eb6ed9941c041d
https://github.com/scummvm/scummvm-sites/commit/d7d1d39dafbb9aeec2ade4a429eb6ed9941c041d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T23:09:19+05:30
Commit Message:
INTEGRITY: Change log table order, filters
Changed paths:
logs.php
diff --git a/logs.php b/logs.php
index 3649fbc..72f6773 100644
--- a/logs.php
+++ b/logs.php
@@ -5,11 +5,14 @@ $filename = "logs.php";
$records_table = "log";
$select_query = "SELECT id, `timestamp`, category, user, `text`
FROM log";
-$order = "ORDER BY `timestamp` DESC";
+$order = "ORDER BY `timestamp` DESC, id DESC";
$filters = array(
- "category" => "log",
- "user" => "log"
+ 'id' => 'log',
+ 'timestamp' => 'log',
+ 'category' => 'log',
+ 'user' => 'log',
+ 'text' => 'log'
);
create_page($filename, 25, $records_table, $select_query, $order, $filters);
Commit: 2b1ac7118850d11b92d73a7fc9acf1ca25b7ee68
https://github.com/scummvm/scummvm-sites/commit/2b1ac7118850d11b92d73a7fc9acf1ca25b7ee68
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T23:09:48+05:30
Commit Message:
INTEGRITY: Add transactions table
Changed paths:
schema.php
diff --git a/schema.php b/schema.php
index a3cf706..3be273b 100644
--- a/schema.php
+++ b/schema.php
@@ -169,6 +169,20 @@ else {
echo "Error creating 'history' table: " . $conn->error;
}
+// Create transactions table
+$table = "CREATE TABLE IF NOT EXISTS transactions (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ `transaction` INT NOT NULL,
+ fileset INT NOT NULL
+)";
+
+if ($conn->query($table) === TRUE) {
+ echo "Table 'transactions' created successfully\n";
+}
+else {
+ echo "Error creating 'transactions' table: " . $conn->error;
+}
+
///////////////////////// CREATE INDEX /////////////////////////
Commit: ec9c0a5fcbcbce5a5ddab3ccce97fc1a18db9b84
https://github.com/scummvm/scummvm-sites/commit/ec9c0a5fcbcbce5a5ddab3ccce97fc1a18db9b84
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T23:11:13+05:30
Commit Message:
INTEGRITY: Add message if paginated table is empty
Changed paths:
pagination.php
diff --git a/pagination.php b/pagination.php
index 19aad60..88c5eb6 100644
--- a/pagination.php
+++ b/pagination.php
@@ -79,6 +79,10 @@ function create_page($filename, $results_per_page, $records_table, $select_query
$num_of_results = $conn->query("SELECT COUNT(id) FROM {$records_table}")->fetch_array()[0];
}
$num_of_pages = ceil($num_of_results / $results_per_page);
+ if ($num_of_results == 0) {
+ echo "No results for given filters";
+ return;
+ }
if (!isset($_GET['page'])) {
$page = 1;
Commit: d38e84599087244938e521e06f6baadcd08fe215
https://github.com/scummvm/scummvm-sites/commit/d38e84599087244938e521e06f6baadcd08fe215
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-27T23:27:27+05:30
Commit Message:
INTEGRITY: Create log entries for every fileset
Changed paths:
db_functions.php
pagination.php
diff --git a/db_functions.php b/db_functions.php
index 3112112..93cdc78 100644
--- a/db_functions.php
+++ b/db_functions.php
@@ -84,7 +84,7 @@ function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform
$conn->query("SET @game_last = LAST_INSERT_ID()");
}
-function insert_fileset($src, $detection, $key, $megakey, $conn) {
+function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_text, $conn) {
$status = $detection ? "detection" : $src;
$game = "NULL";
$key = $key == "" ? "NULL" : "'{$key}'";
@@ -102,10 +102,17 @@ function insert_fileset($src, $detection, $key, $megakey, $conn) {
$existing_entry = $conn->query("SELECT id FROM fileset WHERE megakey = {$megakey}");
if ($existing_entry->num_rows > 0) {
+ $existing_entry = $existing_entry->fetch_array()[0];
+
+ $category_text = "Uploaded from {$src}";
+ $log_text = "Duplicate of Fileset:{$existing_entry}, {$log_text}";
+
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
+
if (!$detection)
return false;
- $existing_entry = $existing_entry->fetch_array()[0];
$conn->query("UPDATE fileset SET `timestamp` = FROM_UNIXTIME(@fileset_time_last)
WHERE id = {$existing_entry}");
$conn->query("UPDATE fileset SET status = 'detection'
@@ -120,6 +127,14 @@ function insert_fileset($src, $detection, $key, $megakey, $conn) {
$conn->query($query);
$conn->query("SET @fileset_last = LAST_INSERT_ID()");
+ $category_text = "Uploaded from {$src}";
+ $fileset_last = $conn->query("SELECT @fileset_last")->fetch_array()[0];
+ $log_text = "Created Fileset:{$fileset_last}, {$log_text}";
+
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
+ $conn->query("INSERT INTO transactions (`transaction`, fileset) VALUES ({$transaction}, {$fileset_last})");
+
return true;
}
@@ -240,6 +255,17 @@ function db_insert($data_arr) {
// Set timestamp of fileset insertion
$conn->query(sprintf("SET @fileset_time_last = %d", time()));
+ // Create start log entry
+ $transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1;
+
+ $category_text = "Uploaded from {$src}";
+ $log_text = sprintf("Started loading DAT file, filename '%s', size %d, author '%s', version %s.
+ State '%s'. Transaction: %d",
+ $filepath, filesize($filepath), $author, $version, $status, $transaction_id);
+
+ $user = 'cli:' . get_current_user();
+ create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
+
foreach ($game_data as $fileset) {
if ($detection) {
$engine_name = $fileset["engine"];
@@ -258,7 +284,11 @@ function db_insert($data_arr) {
$key = $detection ? calc_key($fileset['rom']) : "";
$megakey = !$detection ? calc_key($fileset['rom']) : "";
- if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
+ $log_text = sprintf("from filename '%s', size %d, author '%s', version %s.
+ State '%s'.",
+ $filepath, filesize($filepath), $author, $version, $status);
+
+ if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) {
foreach ($fileset["rom"] as $file) {
insert_file($file, $detection, $src, $conn);
foreach ($file as $key => $value) {
@@ -274,11 +304,11 @@ function db_insert($data_arr) {
WHERE `timestamp` != FROM_UNIXTIME(@fileset_time_last)
AND status = 'detection'");
+ $fileset_insertion_count = $conn->query("SELECT COUNT(fileset) from transactions WHERE `transaction` = {$transaction_id}")->fetch_array()[0];
$category_text = "Uploaded from {$src}";
- $log_text = sprintf("Loaded DAT file, filename '%s', size %d, author '%s', version %s.
- State '%s'. Fileset:%d.",
- $filepath, filesize($filepath), $author, $version, $status,
- $conn->query("SELECT @fileset_last")->fetch_array()[0]);
+ $log_text = sprintf("Completed loading DAT file, filename '%s', size %d, author '%s', version %s.
+ State '%s'. Number of filesets: %d. Transaction: %d",
+ $filepath, filesize($filepath), $author, $version, $status, $fileset_insertion_count, $transaction_id);
if (!$conn->commit())
echo "Inserting failed\n";
diff --git a/pagination.php b/pagination.php
index 88c5eb6..ad39659 100644
--- a/pagination.php
+++ b/pagination.php
@@ -159,7 +159,9 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// Add links to fileset in logs table
$matches = array();
if (preg_match("/Fileset:(\d+)/", $value, $matches, PREG_OFFSET_CAPTURE)) {
- $value = substr($value, 0, $matches[0][1]) . "<a href='fileset.php?id={$matches[1][0]}'>{$matches[0][0]}</a>";
+ $value = substr($value, 0, $matches[0][1]) .
+ "<a href='fileset.php?id={$matches[1][0]}'>{$matches[0][0]}</a>" .
+ substr($value, $matches[0][1] + strlen($matches[0][0]));
}
echo "<td>{$value}</td>\n";
Commit: 0989136e508b3c45e73a41f9f34e90f4d4b8e2f3
https://github.com/scummvm/scummvm-sites/commit/0989136e508b3c45e73a41f9f34e90f4d4b8e2f3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-28T15:38:46+05:30
Commit Message:
INTEGRITY: Update calc_key() for detection entries
Changed paths:
db_functions.php
diff --git a/db_functions.php b/db_functions.php
index 93cdc78..4f652ad 100644
--- a/db_functions.php
+++ b/db_functions.php
@@ -199,9 +199,33 @@ function create_log($category, $user, $text) {
}
/**
- * Calculate `key` value as md5("file1:size:md5:file2:...")
+ * Calculate `key` value as md5("name:title:...:engine:file1:size:md5:file2:...")
*/
-function calc_key($files) {
+function calc_key($fileset) {
+ $key_string = "";
+
+ foreach ($fileset as $key => $value) {
+ if ($key == 'rom')
+ continue;
+
+ $key_string .= ':' . $value;
+ }
+
+ $files = $fileset['rom'];
+ foreach ($files as $file) {
+ foreach ($file as $key => $value) {
+ $key_string .= ':' . $value;
+ }
+ }
+
+ $key_string = trim($key_string, ':');
+ return md5($key_string);
+}
+
+/**
+ * Calculate `megakey` value as md5("file1:size:md5:file2:...")
+ */
+function calc_megakey($files) {
$key_string = "";
foreach ($files as $file) {
foreach ($file as $key => $value) {
@@ -282,8 +306,8 @@ function db_insert($data_arr) {
if (isset($fileset['romof']) && isset($resources[$fileset['romof']]))
$fileset["rom"] = array_merge($fileset["rom"], $resources[$fileset["romof"]]["rom"]);
- $key = $detection ? calc_key($fileset['rom']) : "";
- $megakey = !$detection ? calc_key($fileset['rom']) : "";
+ $key = $detection ? calc_key($fileset) : "";
+ $megakey = !$detection ? calc_megakey($fileset['rom']) : "";
$log_text = sprintf("from filename '%s', size %d, author '%s', version %s.
State '%s'.",
$filepath, filesize($filepath), $author, $version, $status);
Commit: b1d84ea6ac546d89b96352b33a443fb849b0b127
https://github.com/scummvm/scummvm-sites/commit/b1d84ea6ac546d89b96352b33a443fb849b0b127
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-28T16:26:44+05:30
Commit Message:
INTEGRITY: Make games_list rows link to filesets
Changed paths:
games_list.php
js_functions.js
logs.php
pagination.php
style.css
diff --git a/games_list.php b/games_list.php
index 8b17364..ef7bde1 100644
--- a/games_list.php
+++ b/games_list.php
@@ -26,6 +26,6 @@ $mapping = array(
'game.id' => 'fileset.game',
);
-create_page($filename, 25, $records_table, $select_query, $order, $filters, $mapping);
+create_page($filename, 25, $records_table, $select_query, $order, "games_list.php", $filters, $mapping);
?>
diff --git a/js_functions.js b/js_functions.js
index 8cb56cd..d31db37 100644
--- a/js_functions.js
+++ b/js_functions.js
@@ -24,6 +24,10 @@ function remove_empty_inputs() {
}
}
+function hyperlink(link) {
+ window.location = link;
+}
+
$(document).ready(function () {
$(".hidden").hide();
$("#delete-button").one("click", delete_id);
diff --git a/logs.php b/logs.php
index 72f6773..70c5000 100644
--- a/logs.php
+++ b/logs.php
@@ -15,6 +15,6 @@ $filters = array(
'text' => 'log'
);
-create_page($filename, 25, $records_table, $select_query, $order, $filters);
+create_page($filename, 25, $records_table, $select_query, $order, "logs.php", $filters);
?>
diff --git a/pagination.php b/pagination.php
index ad39659..1143a34 100644
--- a/pagination.php
+++ b/pagination.php
@@ -21,7 +21,7 @@ function get_join_columns($table1, $table2, $mapping) {
echo "No primary-foreign key mapping provided. Filter is invalid";
}
-function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
+function create_page($filename, $results_per_page, $records_table, $select_query, $order, $webpage = "", $filters = array(), $mapping = array()) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -118,8 +118,6 @@ function create_page($filename, $results_per_page, $records_table, $select_query
echo "<form id='filters-form' method='GET' onsubmit='remove_empty_inputs()'>";
echo "<table>\n";
- $fileset_column_index = null;
-
$counter = $offset + 1;
while ($row = $result->fetch_assoc()) {
if ($counter == $offset + 1) { // If it is the first run of the loop
@@ -141,20 +139,22 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
echo "<th/>\n"; // Numbering column
- foreach (array_keys($row) as $index => $key) {
- echo "<th>{$key}</th>\n";
+ foreach (array_keys($row) as $key) {
+ if ($key == 'fileset')
+ continue;
- if ($key == "fileset")
- $fileset_column_index = $index;
+ echo "<th>{$key}</th>\n";
}
}
- echo "<tr>\n";
+ if ($webpage == 'games_list.php')
+ echo "<tr class=games_list onclick='hyperlink(\"fileset.php?id={$row['fileset']}\")'>\n";
+ else
+ echo "<tr>\n";
echo "<td>{$counter}.</td>\n";
- foreach (array_values($row) as $key => $value) {
- // Add hyperlink to fileset in game_list table
- if ($fileset_column_index && $key == $fileset_column_index)
- $value = "<a href='fileset.php?id={$value}'>{$value}</a>";
+ foreach ($row as $key => $value) {
+ if ($key == 'fileset')
+ continue;
// Add links to fileset in logs table
$matches = array();
diff --git a/style.css b/style.css
index 519aa25..4b2bd00 100644
--- a/style.css
+++ b/style.css
@@ -11,6 +11,7 @@ tr:nth-child(even) {background-color: #f2f2f2;}
tr {background-color: white;}
tr:hover {background-color: #ddd;}
+tr.games_list:hover {cursor: pointer;}
tr.filter:hover {background-color:inherit;}
td.filter {text-align: center;}
Commit: 863e4612f352fb50b78d60f9fc0b301f965231c9
https://github.com/scummvm/scummvm-sites/commit/863e4612f352fb50b78d60f9fc0b301f965231c9
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-28T18:28:01+05:30
Commit Message:
INTEGRITY: Add sorting to all paginated columns
Changed paths:
games_list.php
logs.php
pagination.php
style.css
diff --git a/games_list.php b/games_list.php
index ef7bde1..8b17364 100644
--- a/games_list.php
+++ b/games_list.php
@@ -26,6 +26,6 @@ $mapping = array(
'game.id' => 'fileset.game',
);
-create_page($filename, 25, $records_table, $select_query, $order, "games_list.php", $filters, $mapping);
+create_page($filename, 25, $records_table, $select_query, $order, $filters, $mapping);
?>
diff --git a/logs.php b/logs.php
index 70c5000..72f6773 100644
--- a/logs.php
+++ b/logs.php
@@ -15,6 +15,6 @@ $filters = array(
'text' => 'log'
);
-create_page($filename, 25, $records_table, $select_query, $order, "logs.php", $filters);
+create_page($filename, 25, $records_table, $select_query, $order, $filters);
?>
diff --git a/pagination.php b/pagination.php
index 1143a34..ec0b58d 100644
--- a/pagination.php
+++ b/pagination.php
@@ -21,7 +21,7 @@ function get_join_columns($table1, $table2, $mapping) {
echo "No primary-foreign key mapping provided. Filter is invalid";
}
-function create_page($filename, $results_per_page, $records_table, $select_query, $order, $webpage = "", $filters = array(), $mapping = array()) {
+function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
@@ -43,12 +43,20 @@ function create_page($filename, $results_per_page, $records_table, $select_query
// If there exist get variables that are for filtering
$_GET = array_filter($_GET);
+ if (isset($_GET['sort'])) {
+ $column = $_GET['sort'];
+ $column = explode('-', $column);
+ $order = "ORDER BY {$column[0]}";
- if (array_diff(array_keys($_GET), array('page'))) {
+ if (strpos($_GET['sort'], 'desc') !== false)
+ $order .= " DESC";
+ }
+
+ if (array_diff(array_keys($_GET), array('page', 'sort'))) {
$condition = "WHERE ";
$tables = array();
foreach ($_GET as $key => $value) {
- if ($key == "page" || $value == "")
+ if ($key == 'page' || $key == 'sort' || $value == '')
continue;
array_push($tables, $filters[$key]);
@@ -143,11 +151,23 @@ function create_page($filename, $results_per_page, $records_table, $select_query
if ($key == 'fileset')
continue;
- echo "<th>{$key}</th>\n";
+ // Preserve GET variables
+ $vars = "";
+ foreach ($_GET as $k => $v) {
+ if ($k == 'sort' && $v == $key)
+ $vars .= "&{$k}={$v}-desc";
+ elseif ($k != 'sort')
+ $vars .= "&{$k}={$v}";
+ }
+
+ if (strpos($vars, "&sort={$key}") === false)
+ echo "<th><a href='{$filename}?{$vars}&sort={$key}'>{$key}</th>\n";
+ else
+ echo "<th><a href='{$filename}?{$vars}'>{$key}</th>\n";
}
}
- if ($webpage == 'games_list.php')
+ if ($filename == 'games_list.php')
echo "<tr class=games_list onclick='hyperlink(\"fileset.php?id={$row['fileset']}\")'>\n";
else
echo "<tr>\n";
diff --git a/style.css b/style.css
index 4b2bd00..1c9e599 100644
--- a/style.css
+++ b/style.css
@@ -24,6 +24,11 @@ th {
color: white;
}
+th a {
+ color: white;
+ text-decoration: none; /* no underline */
+}
+
button {
color: white;
padding: 6px 12px;
Commit: 7a321265c4f84abb0fc57a5b1f74023419f61280
https://github.com/scummvm/scummvm-sites/commit/7a321265c4f84abb0fc57a5b1f74023419f61280
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-28T22:57:56+05:30
Commit Message:
INTEGRITY: Full user checksums used for validation
- Only full checksums used instead of all checksums
- Add $log_text and $transaction_id
Changed paths:
checksum_validator.php
diff --git a/checksum_validator.php b/checksum_validator.php
index a04c010..73cee61 100644
--- a/checksum_validator.php
+++ b/checksum_validator.php
@@ -83,13 +83,18 @@ while ($game = $games->fetch_array()) {
if ($status == 'ok') {
foreach ($user_file->checksums as $checksum_data) {
foreach ($checksum_data as $key => $value) {
+ // If it's not the full checksum
+ if (strpos($user_checkcode, '-') !== false)
+ continue;
+
$user_checksum = $checksum_data->checksum;
$user_checkcode = $checksum_data->type;
- if (strpos($user_checkcode, '-') === false)
$user_checkcode .= '-0';
if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
$status = 'checksum_mismatch';
+
+ break;
}
}
}
@@ -142,9 +147,11 @@ function user_insert_fileset($user_fileset, $conn) {
$detection = 'false';
$key = '';
$megakey = user_calc_key($user_fileset);
+ $transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1;
+ $log_text = "from user submitted files";
$conn = db_connect();
- if (insert_fileset($src, $detection, $key, $megakey, $conn)) {
+ if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) {
foreach ($user_fileset as $file) {
$file = file_json_to_array($file);
Commit: c45e850a37746c16ae101b66b33554df9569ac62
https://github.com/scummvm/scummvm-sites/commit/c45e850a37746c16ae101b66b33554df9569ac62
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-07-29T23:11:13+05:30
Commit Message:
INTEGRITY: Add insert_user_queue() for user queue
Changed paths:
checksum_validator.php
schema.php
diff --git a/checksum_validator.php b/checksum_validator.php
index 73cee61..9cc1f9b 100644
--- a/checksum_validator.php
+++ b/checksum_validator.php
@@ -142,6 +142,13 @@ function file_json_to_array($file_json_object) {
return $res;
}
+function user_insert_queue($user_fileset, $conn) {
+ $query = sprintf("INSERT INTO queue (time, notes, fileset, ticketid, userid, commit)
+ VALUES (%d, NULL, @fileset_last, NULL, NULL, NULL)", time());
+
+ $conn->query($query);
+}
+
function user_insert_fileset($user_fileset, $conn) {
$src = 'user';
$detection = 'false';
diff --git a/schema.php b/schema.php
index 3be273b..dc507e5 100644
--- a/schema.php
+++ b/schema.php
@@ -121,7 +121,7 @@ else {
// Create queue table
$table = "CREATE TABLE IF NOT EXISTS queue (
id INT AUTO_INCREMENT PRIMARY KEY,
- date DATETIME NOT NULL,
+ time TIMESTAMP NOT NULL,
notes varchar(300),
fileset INT,
ticketid INT NOT NULL,
Commit: bbd9cf8d2c87f9460ab80757dffa51bec2cddf09
https://github.com/scummvm/scummvm-sites/commit/bbd9cf8d2c87f9460ab80757dffa51bec2cddf09
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-02T20:00:19+05:30
Commit Message:
INTEGRITY: Restructure source code
- Create /bin for CLI utilities; file access blocked from server
- Create /include for modules with utility functions; access blocked
- Rename index.php to index.php
Changed paths:
A bin/dat_parser.php
A bin/schema.php
A bin/seeds.php
A include/checksum_validator.php
A include/db_functions.php
A include/pagination.php
A index.html
R checksum_validator.php
R dat_parser.php
R db_functions.php
R pagination.php
R schema.php
R seeds.php
fileset.php
games_list.php
logs.php
diff --git a/dat_parser.php b/bin/dat_parser.php
similarity index 99%
rename from dat_parser.php
rename to bin/dat_parser.php
index 34dd305..46ad90a 100644
--- a/dat_parser.php
+++ b/bin/dat_parser.php
@@ -1,6 +1,6 @@
<?php
-require('db_functions.php');
+require 'include/db_functions.php';
ini_set('memory_limit', '512M');
function remove_quotes($string) {
diff --git a/schema.php b/bin/schema.php
similarity index 99%
rename from schema.php
rename to bin/schema.php
index dc507e5..c44fd41 100644
--- a/schema.php
+++ b/bin/schema.php
@@ -124,7 +124,6 @@ $table = "CREATE TABLE IF NOT EXISTS queue (
time TIMESTAMP NOT NULL,
notes varchar(300),
fileset INT,
- ticketid INT NOT NULL,
userid INT NOT NULL,
commit VARCHAR(64) NOT NULL,
FOREIGN KEY (fileset) REFERENCES fileset(id)
diff --git a/seeds.php b/bin/seeds.php
similarity index 100%
rename from seeds.php
rename to bin/seeds.php
diff --git a/fileset.php b/fileset.php
index 43c9a08..9f60aa0 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,5 +1,5 @@
<?php
-require 'pagination.php';
+require 'include/pagination.php';
$filename = 'fileset.php';
$stylesheet = 'style.css';
diff --git a/games_list.php b/games_list.php
index 8b17364..8def221 100644
--- a/games_list.php
+++ b/games_list.php
@@ -1,5 +1,5 @@
<?php
-require "pagination.php";
+require 'include/pagination.php';
$filename = "games_list.php";
$records_table = "game";
diff --git a/checksum_validator.php b/include/checksum_validator.php
similarity index 97%
rename from checksum_validator.php
rename to include/checksum_validator.php
index 9cc1f9b..5a4829f 100644
--- a/checksum_validator.php
+++ b/include/checksum_validator.php
@@ -1,9 +1,9 @@
<?php
-require('db_functions.php');
+require '../include/db_functions.php';
$conn = db_connect();
-$json_string = file_get_contents('sample_json_request.json');
+$json_string = file_get_contents('../sample_json_request.json');
$json_object = json_decode($json_string);
$game_metadata = array();
@@ -89,7 +89,7 @@ while ($game = $games->fetch_array()) {
$user_checksum = $checksum_data->checksum;
$user_checkcode = $checksum_data->type;
- $user_checkcode .= '-0';
+ $user_checkcode .= '-0';
if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
$status = 'checksum_mismatch';
diff --git a/db_functions.php b/include/db_functions.php
similarity index 100%
rename from db_functions.php
rename to include/db_functions.php
diff --git a/pagination.php b/include/pagination.php
similarity index 99%
rename from pagination.php
rename to include/pagination.php
index ec0b58d..846a753 100644
--- a/pagination.php
+++ b/include/pagination.php
@@ -1,5 +1,5 @@
<?php
-$stylesheet = "style.css";
+$stylesheet = 'style.css';
$jquery_file = 'https://code.jquery.com/jquery-3.7.0.min.js';
$js_file = 'js_functions.js';
echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..a6c5e6d
--- /dev/null
+++ b/index.html
@@ -0,0 +1,2 @@
+<a href="games_list.php">List of Detection entries</a><br/>
+<a href="logs.php">Logs of developer actions</a><br/>
diff --git a/logs.php b/logs.php
index 72f6773..0755999 100644
--- a/logs.php
+++ b/logs.php
@@ -1,5 +1,5 @@
<?php
-require "pagination.php";
+require 'include/pagination.php';
$filename = "logs.php";
$records_table = "log";
Commit: 0fa955ee0b010ab232bc6441596a0824a78781f0
https://github.com/scummvm/scummvm-sites/commit/0fa955ee0b010ab232bc6441596a0824a78781f0
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-02T20:00:26+05:30
Commit Message:
INTEGRITY: Create apache config file .htaccess
Changed paths:
A .htaccess
diff --git a/.htaccess b/.htaccess
new file mode 100644
index 0000000..3af6f96
--- /dev/null
+++ b/.htaccess
@@ -0,0 +1,16 @@
+RewriteCond %{REQUEST_FILENAME} !-d
+RewriteCond %{REQUEST_FILENAME}\.php -f
+RewriteRule ^(.*)$ $1.php [NC,L]
+
+<Files "mysql_config.json">
+ Order allow,deny
+ Deny from all
+</Files>
+<Files "bin/*">
+ Order allow,deny
+ Deny from all
+</Files>
+<Files "include/*">
+ Order allow,deny
+ Deny from all
+</Files>
Commit: 0675081f03b39b6f3dcce50e54755dbd718243c3
https://github.com/scummvm/scummvm-sites/commit/0675081f03b39b6f3dcce50e54755dbd718243c3
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-02T20:00:26+05:30
Commit Message:
INTEGRITY: Fix variable not defined error
Changed paths:
include/checksum_validator.php
diff --git a/include/checksum_validator.php b/include/checksum_validator.php
index 5a4829f..d8785a4 100644
--- a/include/checksum_validator.php
+++ b/include/checksum_validator.php
@@ -83,12 +83,12 @@ while ($game = $games->fetch_array()) {
if ($status == 'ok') {
foreach ($user_file->checksums as $checksum_data) {
foreach ($checksum_data as $key => $value) {
+ $user_checkcode = $checksum_data->type;
// If it's not the full checksum
if (strpos($user_checkcode, '-') !== false)
continue;
$user_checksum = $checksum_data->checksum;
- $user_checkcode = $checksum_data->type;
$user_checkcode .= '-0';
if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
Commit: 84bfff264c9a96c58242dc9708978a338bc414dd
https://github.com/scummvm/scummvm-sites/commit/84bfff264c9a96c58242dc9708978a338bc414dd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-02T20:04:42+05:30
Commit Message:
INTEGRITY: Create endpoint for checksum validation
- Move non-function code from checksum_validator to validate.php
- Route endpoint /api/validate to /endpoints/validate.php
Changed paths:
A endpoints/validate.php
include/checksum_validator.php
index.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
new file mode 100644
index 0000000..c4cefb1
--- /dev/null
+++ b/endpoints/validate.php
@@ -0,0 +1,115 @@
+<?php
+require 'include/db_functions.php';
+
+header('Access-Contol-Allow-Origin: *');
+// header('Content-Type: application/json');
+
+$conn = db_connect();
+
+$json_string = file_get_contents('php://input');
+$json_object = json_decode($json_string);
+
+$game_metadata = array();
+foreach ($json_object as $key => $value) {
+ if ($key == 'files')
+ continue;
+
+ $game_metadata[$key] = $value;
+}
+
+// Find game(s) that fit the metadata
+$query = "SELECT game.id FROM game
+JOIN engine ON game.engine = engine.id
+WHERE gameid = '{$game_metadata['gameid']}'
+AND engineid = '{$game_metadata['engineid']}'
+AND extra = '{$game_metadata['extra']}'
+AND platform = '{$game_metadata['platform']}'
+AND language = '{$game_metadata['language']}'";
+$games = $conn->query($query);
+
+$json_response = array(
+ 'error' => 0,
+ 'files' => array()
+);
+
+// Check if all files in fullmatch filesets are present with user
+while ($game = $games->fetch_array()) {
+ $fileset = $conn->query("SELECT file.id, name, size FROM file
+ JOIN fileset ON fileset.id = file.fileset
+ WHERE fileset.game = {$game['id']} AND fileset.status = 'fullmatch'");
+
+ if ($fileset->num_rows == 0)
+ continue;
+
+ // Convert checktype, checksize to checkcode
+ $fileset = $fileset->fetch_all(MYSQLI_ASSOC);
+ foreach (array_values($fileset) as $index => $file) {
+ $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype
+ FROM filechecksum WHERE file = {$file['id']}");
+
+ while ($spec_checksum = $spec_checksum_res->fetch_assoc()) {
+ $fileset[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum'];
+ }
+ }
+
+ $file_object = $json_object->files;
+
+ // Sort the filesets by filename
+ usort($file_object, function ($a, $b) {
+ return strcmp($a->name, $b->name) == -1 ? -1 : 1;
+ });
+ usort($fileset, function ($a, $b) {
+ return strcmp($a['name'], $b['name']) == -1 ? -1 : 1;
+ });
+
+ for ($i = 0, $j = 0; $i < count($fileset), $j < count($file_object); $i++, $j++) {
+ $status = 'ok';
+ $db_file = $fileset[$i];
+ $user_file = $file_object[$j];
+ $filename = $user_file->name;
+
+ if ($db_file['name'] != $user_file->name) {
+ if ($db_file['name'] > $user_file->name) {
+ $status = 'unknown_file';
+ $i--; // Retain same db_file for next iteration
+ }
+ else {
+ $status = 'missing';
+ $filename = $db_file['name'];
+ $j--; // Retain same user_file for next iteration
+ }
+ }
+ elseif ($db_file['size'] != $user_file->size && $status == 'ok') {
+ $status = 'size_mismatch';
+ }
+
+ if ($status == 'ok') {
+ foreach ($user_file->checksums as $checksum_data) {
+ foreach ($checksum_data as $key => $value) {
+ $user_checkcode = $checksum_data->type;
+ // If it's not the full checksum
+ if (strpos($user_checkcode, '-') !== false)
+ continue;
+
+ $user_checksum = $checksum_data->checksum;
+ $user_checkcode .= '-0';
+
+ if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
+ $status = 'checksum_mismatch';
+
+ break;
+ }
+ }
+ }
+
+ if ($status != 'ok')
+ $json_response['error'] = 1;
+
+ array_push($json_response['files'], array('status' => $status, 'name' => $filename));
+ }
+}
+
+$json_response = json_encode($json_response);
+echo $json_response;
+?>
+
diff --git a/include/checksum_validator.php b/include/checksum_validator.php
index d8785a4..00715e9 100644
--- a/include/checksum_validator.php
+++ b/include/checksum_validator.php
@@ -1,112 +1,5 @@
<?php
-require '../include/db_functions.php';
-
-$conn = db_connect();
-
-$json_string = file_get_contents('../sample_json_request.json');
-$json_object = json_decode($json_string);
-
-$game_metadata = array();
-foreach ($json_object as $key => $value) {
- if ($key == 'files')
- continue;
-
- $game_metadata[$key] = $value;
-}
-
-// Find game(s) that fit the metadata
-$query = "SELECT game.id FROM game
-JOIN engine ON game.engine = engine.id
-WHERE gameid = '{$game_metadata['gameid']}'
-AND engineid = '{$game_metadata['engineid']}'
-AND extra = '{$game_metadata['extra']}'
-AND platform = '{$game_metadata['platform']}'
-AND language = '{$game_metadata['language']}'";
-$games = $conn->query($query);
-
-$json_response = array(
- 'error' => 0,
- 'files' => array()
-);
-
-// Check if all files in fullmatch filesets are present with user
-while ($game = $games->fetch_array()) {
- $fileset = $conn->query("SELECT file.id, name, size FROM file
- JOIN fileset ON fileset.id = file.fileset
- WHERE fileset.game = {$game['id']} AND fileset.status = 'fullmatch'");
-
- if ($fileset->num_rows == 0)
- continue;
-
- // Convert checktype, checksize to checkcode
- $fileset = $fileset->fetch_all(MYSQLI_ASSOC);
- foreach (array_values($fileset) as $index => $file) {
- $spec_checksum_res = $conn->query("SELECT checksum, checksize, checktype
- FROM filechecksum WHERE file = {$file['id']}");
-
- while ($spec_checksum = $spec_checksum_res->fetch_assoc()) {
- $fileset[$index][$spec_checksum['checktype'] . '-' . $spec_checksum['checksize']] = $spec_checksum['checksum'];
- }
- }
-
- $file_object = $json_object->files;
-
- // Sort the filesets by filename
- usort($file_object, function ($a, $b) {
- return strcmp($a->name, $b->name) == -1 ? -1 : 1;
- });
- usort($fileset, function ($a, $b) {
- return strcmp($a['name'], $b['name']) == -1 ? -1 : 1;
- });
-
- for ($i = 0, $j = 0; $i < count($fileset), $j < count($file_object); $i++, $j++) {
- $status = 'ok';
- $db_file = $fileset[$i];
- $user_file = $file_object[$j];
- $filename = $user_file->name;
-
- if ($db_file['name'] != $user_file->name) {
- if ($db_file['name'] > $user_file->name) {
- $status = 'unknown_file';
- $i--; // Retain same db_file for next iteration
- }
- else {
- $status = 'missing';
- $filename = $db_file['name'];
- $j--; // Retain same user_file for next iteration
- }
- }
- elseif ($db_file['size'] != $user_file->size && $status == 'ok') {
- $status = 'size_mismatch';
- }
-
- if ($status == 'ok') {
- foreach ($user_file->checksums as $checksum_data) {
- foreach ($checksum_data as $key => $value) {
- $user_checkcode = $checksum_data->type;
- // If it's not the full checksum
- if (strpos($user_checkcode, '-') !== false)
- continue;
-
- $user_checksum = $checksum_data->checksum;
- $user_checkcode .= '-0';
-
- if (strcasecmp($db_file[$user_checkcode], $user_checksum) != 0)
- $status = 'checksum_mismatch';
-
- break;
- }
- }
- }
-
- if ($status != 'ok')
- $json_response['error'] = 1;
-
- array_push($json_response['files'], array('status' => $status, 'name' => $filename));
- }
-}
-
-$json_response = json_encode($json_response);
+require 'include/db_functions.php';
function user_calc_key($user_fileset) {
$key_string = "";
@@ -170,7 +63,5 @@ function user_insert_fileset($user_fileset, $conn) {
}
}
}
-
-$conn->close();
?>
diff --git a/index.php b/index.php
index 2bc2461..f76a86a 100644
--- a/index.php
+++ b/index.php
@@ -1,5 +1,15 @@
<?php
-echo '<a href="games_list.php">List of Detection entries</a><br/>';
-echo '<a href="logs.php">Logs of developer actions</a><br/>';
+$request = $_SERVER['REQUEST_URI'];
+$api_root = '/endpoints/';
+
+switch ($request) {
+ case '':
+ case '/':
+ require __DIR__ . '/index.html';
+ break;
+
+ case '/api/validate':
+ require __DIR__ . $api_root . 'validate.php';
+}
?>
Commit: c73a95096aec04dc5bac2e420523d7cc4ae6babf
https://github.com/scummvm/scummvm-sites/commit/c73a95096aec04dc5bac2e420523d7cc4ae6babf
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-03T16:57:54+05:30
Commit Message:
INTEGRITY: Rename checksum_validation
Changed paths:
A include/user_fileset_functions.php
R include/checksum_validator.php
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index c4cefb1..14be638 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -1,5 +1,5 @@
<?php
-require 'include/db_functions.php';
+require 'include/user_fileset_functions.php';
header('Access-Contol-Allow-Origin: *');
// header('Content-Type: application/json');
diff --git a/include/checksum_validator.php b/include/user_fileset_functions.php
similarity index 100%
rename from include/checksum_validator.php
rename to include/user_fileset_functions.php
Commit: 161abc5bda647a14ad246df714130ae117484f26
https://github.com/scummvm/scummvm-sites/commit/161abc5bda647a14ad246df714130ae117484f26
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-03T16:57:59+05:30
Commit Message:
INTEGRITY: Update fileset sorting algorithm
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 14be638..7a02237 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -56,10 +56,10 @@ while ($game = $games->fetch_array()) {
// Sort the filesets by filename
usort($file_object, function ($a, $b) {
- return strcmp($a->name, $b->name) == -1 ? -1 : 1;
+ return strcmp($a->name, $b->name);
});
usort($fileset, function ($a, $b) {
- return strcmp($a['name'], $b['name']) == -1 ? -1 : 1;
+ return strcmp($a['name'], $b['name']);
});
for ($i = 0, $j = 0; $i < count($fileset), $j < count($file_object); $i++, $j++) {
Commit: 3dc5a3a86fe351416f10f4a70a279961303b4187
https://github.com/scummvm/scummvm-sites/commit/3dc5a3a86fe351416f10f4a70a279961303b4187
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-03T16:58:13+05:30
Commit Message:
INTEGRITY: Insert user fileset on unknown variants
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 7a02237..6a8d873 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -2,7 +2,7 @@
require 'include/user_fileset_functions.php';
header('Access-Contol-Allow-Origin: *');
-// header('Content-Type: application/json');
+header('Content-Type: application/json');
$conn = db_connect();
@@ -32,6 +32,14 @@ $json_response = array(
'files' => array()
);
+if ($games->num_rows == 0) {
+ $json_response['error'] = 1;
+ unset($json_response['files']);
+ $json_response['status'] = 'unknown_variant';
+
+ user_insert_fileset($json_object->files, $conn);
+}
+
// Check if all files in fullmatch filesets are present with user
while ($game = $games->fetch_array()) {
$fileset = $conn->query("SELECT file.id, name, size FROM file
Commit: 3b7527c4384222ca053c169be13666f5e261bed4
https://github.com/scummvm/scummvm-sites/commit/3b7527c4384222ca053c169be13666f5e261bed4
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-03T16:58:55+05:30
Commit Message:
INTEGRITY: Fix user fileset insertion
Changed paths:
include/user_fileset_functions.php
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 00715e9..77221b5 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -44,13 +44,16 @@ function user_insert_queue($user_fileset, $conn) {
function user_insert_fileset($user_fileset, $conn) {
$src = 'user';
- $detection = 'false';
+ $detection = false;
$key = '';
$megakey = user_calc_key($user_fileset);
$transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1;
$log_text = "from user submitted files";
$conn = db_connect();
+ // Set timestamp of fileset insertion
+ $conn->query(sprintf("SET @fileset_time_last = %d", time()));
+
if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) {
foreach ($user_fileset as $file) {
$file = file_json_to_array($file);
@@ -62,6 +65,8 @@ function user_insert_fileset($user_fileset, $conn) {
}
}
}
+
+ $conn->commit();
}
?>
Commit: 9fd1d86f7ed390f5c00aab57cae29ecdbbc1363d
https://github.com/scummvm/scummvm-sites/commit/9fd1d86f7ed390f5c00aab57cae29ecdbbc1363d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-05T23:14:13+05:30
Commit Message:
INTEGRITY: Include paths now use absolute paths
Changed paths:
bin/dat_parser.php
bin/schema.php
bin/seeds.php
endpoints/validate.php
fileset.php
games_list.php
include/db_functions.php
include/pagination.php
include/user_fileset_functions.php
index.php
logs.php
diff --git a/bin/dat_parser.php b/bin/dat_parser.php
index 46ad90a..935a5ca 100644
--- a/bin/dat_parser.php
+++ b/bin/dat_parser.php
@@ -1,6 +1,6 @@
<?php
-require 'include/db_functions.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/db_functions.php';
ini_set('memory_limit', '512M');
function remove_quotes($string) {
diff --git a/bin/schema.php b/bin/schema.php
index c44fd41..7e23b7d 100644
--- a/bin/schema.php
+++ b/bin/schema.php
@@ -1,6 +1,6 @@
<?php
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/bin/seeds.php b/bin/seeds.php
index 858580b..e651df8 100644
--- a/bin/seeds.php
+++ b/bin/seeds.php
@@ -1,6 +1,6 @@
<?php
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 6a8d873..3c1b10d 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -1,5 +1,5 @@
<?php
-require 'include/user_fileset_functions.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/user_fileset_functions.php';
header('Access-Contol-Allow-Origin: *');
header('Content-Type: application/json');
diff --git a/fileset.php b/fileset.php
index 9f60aa0..2d6c57b 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,5 +1,5 @@
<?php
-require 'include/pagination.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
$filename = 'fileset.php';
$stylesheet = 'style.css';
@@ -14,7 +14,7 @@ function get_log_page($log_id) {
return intdiv($log_id, $records_per_page) + 1;
}
-$mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/games_list.php b/games_list.php
index 8def221..1bec999 100644
--- a/games_list.php
+++ b/games_list.php
@@ -1,5 +1,5 @@
<?php
-require 'include/pagination.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
$filename = "games_list.php";
$records_table = "game";
diff --git a/include/db_functions.php b/include/db_functions.php
index 4f652ad..3ff42c6 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -4,7 +4,7 @@
* Create and return a mysqli connection
*/
function db_connect() {
- $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+ $mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/include/pagination.php b/include/pagination.php
index 846a753..bc96679 100644
--- a/include/pagination.php
+++ b/include/pagination.php
@@ -22,7 +22,7 @@ function get_join_columns($table1, $table2, $mapping) {
}
function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
- $mysql_cred = json_decode(file_get_contents('mysql_config.json'), true);
+ $mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 77221b5..4a13e82 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -1,5 +1,5 @@
<?php
-require 'include/db_functions.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/db_functions.php';
function user_calc_key($user_fileset) {
$key_string = "";
diff --git a/index.php b/index.php
index f76a86a..0f16bb7 100644
--- a/index.php
+++ b/index.php
@@ -6,10 +6,10 @@ $api_root = '/endpoints/';
switch ($request) {
case '':
case '/':
- require __DIR__ . '/index.html';
+ require $_SERVER['DOCUMENT_ROOT'] . '/index.html';
break;
case '/api/validate':
- require __DIR__ . $api_root . 'validate.php';
+ require $_SERVER['DOCUMENT_ROOT'] . $api_root . 'validate.php';
}
?>
diff --git a/logs.php b/logs.php
index 0755999..13e25f3 100644
--- a/logs.php
+++ b/logs.php
@@ -1,5 +1,5 @@
<?php
-require 'include/pagination.php';
+require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
$filename = "logs.php";
$records_table = "log";
Commit: 49d8b4dcd47b6b3a34187ce6937195c4e4d3838f
https://github.com/scummvm/scummvm-sites/commit/49d8b4dcd47b6b3a34187ce6937195c4e4d3838f
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-05T23:28:37+05:30
Commit Message:
INTEGRITY: Fix user fileset insertion
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 3c1b10d..1af0dad 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -114,6 +114,9 @@ while ($game = $games->fetch_array()) {
$json_response['error'] = 1;
array_push($json_response['files'], array('status' => $status, 'name' => $filename));
+
+ if ($i < count($fileset) && $j < count($file_object))
+ break;
}
}
Commit: 5cc750df0d9b1c1a35de3ac2ef706c414c8d125a
https://github.com/scummvm/scummvm-sites/commit/5cc750df0d9b1c1a35de3ac2ef706c414c8d125a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-05T23:33:16+05:30
Commit Message:
INTEGRITY: fileset.key no longer uses gameid
Changed paths:
include/db_functions.php
diff --git a/include/db_functions.php b/include/db_functions.php
index 3ff42c6..64387a4 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -205,7 +205,7 @@ function calc_key($fileset) {
$key_string = "";
foreach ($fileset as $key => $value) {
- if ($key == 'rom')
+ if ($key == 'engineid' || $key == 'gameid' || $key == 'rom')
continue;
$key_string .= ':' . $value;
Commit: 8a99da8324177b0352e976e4fda6008d545bd16d
https://github.com/scummvm/scummvm-sites/commit/8a99da8324177b0352e976e4fda6008d545bd16d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-08T22:47:23+05:30
Commit Message:
INTEGRITY: Update response for unknown filesets
Changed paths:
endpoints/validate.php
include/user_fileset_functions.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 1af0dad..95de28c 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -37,7 +37,8 @@ if ($games->num_rows == 0) {
unset($json_response['files']);
$json_response['status'] = 'unknown_variant';
- user_insert_fileset($json_object->files, $conn);
+ $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $json_response['fileset'] = $fileset_id;
}
// Check if all files in fullmatch filesets are present with user
@@ -114,9 +115,6 @@ while ($game = $games->fetch_array()) {
$json_response['error'] = 1;
array_push($json_response['files'], array('status' => $status, 'name' => $filename));
-
- if ($i < count($fileset) && $j < count($file_object))
- break;
}
}
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 4a13e82..b25d733 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -66,7 +66,9 @@ function user_insert_fileset($user_fileset, $conn) {
}
}
+ $fileset_id = $conn->query("SELECT @fileset_last");
$conn->commit();
+ return $fileset_id;
}
?>
Commit: 7115f285c767bc87dc4084c82cd6ca0692d2ae98
https://github.com/scummvm/scummvm-sites/commit/7115f285c767bc87dc4084c82cd6ca0692d2ae98
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-08T22:47:32+05:30
Commit Message:
INTEGRITY: Add delete_filesets for marked filesets
Changed paths:
include/db_functions.php
diff --git a/include/db_functions.php b/include/db_functions.php
index 64387a4..99a4c69 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -182,6 +182,14 @@ function insert_filechecksum($file, $checktype, $conn) {
$conn->query($query);
}
+/**
+ * Delete filesets marked for deletion
+ */
+function delete_filesets($conn) {
+ $query = "DELETE FROM fileset WHERE `delete` == TRUE";
+ $conn->query($query);
+}
+
/**
* Create an entry to the log table on each call of db_insert() or
* populate_matching_games()
Commit: fce070e8add1d59400e460bbda379497ebd37116
https://github.com/scummvm/scummvm-sites/commit/fce070e8add1d59400e460bbda379497ebd37116
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-09T22:56:58+05:30
Commit Message:
INTEGRITY: Display user submitted filesets
Changed paths:
A user_games_list.php
include/pagination.php
diff --git a/include/pagination.php b/include/pagination.php
index bc96679..b4d95ca 100644
--- a/include/pagination.php
+++ b/include/pagination.php
@@ -167,7 +167,7 @@ function create_page($filename, $results_per_page, $records_table, $select_query
}
}
- if ($filename == 'games_list.php')
+ if ($filename == 'games_list.php' || $filename == 'user_games_list.php')
echo "<tr class=games_list onclick='hyperlink(\"fileset.php?id={$row['fileset']}\")'>\n";
else
echo "<tr>\n";
diff --git a/user_games_list.php b/user_games_list.php
new file mode 100644
index 0000000..bbccc7c
--- /dev/null
+++ b/user_games_list.php
@@ -0,0 +1,32 @@
+<?php
+require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
+
+$filename = "user_games_list.php";
+$records_table = "game";
+$select_query = "SELECT engineid, gameid, extra, platform, language, game.name,
+status, fileset.id as fileset
+FROM fileset
+LEFT JOIN game ON game.id = fileset.game
+LEFT JOIN engine ON engine.id = game.engine
+WHERE status = 'user'";
+$order = "ORDER BY gameid";
+
+// Filter column => table
+$filters = array(
+ "engineid" => "engine",
+ "gameid" => "game",
+ "extra" => "game",
+ "platform" => "game",
+ "language" => "game",
+ "name" => "game",
+ "status" => "fileset"
+);
+
+$mapping = array(
+ 'engine.id' => 'game.engine',
+ 'game.id' => 'fileset.game',
+);
+
+create_page($filename, 200, $records_table, $select_query, $order, $filters, $mapping);
+?>
+
Commit: a030fa15ce9049765a4f529138a227fd728469d5
https://github.com/scummvm/scummvm-sites/commit/a030fa15ce9049765a4f529138a227fd728469d5
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-10T21:36:41+05:30
Commit Message:
INTEGRITY: Chnage error code for unknown variants
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 95de28c..a6625e6 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -33,7 +33,7 @@ $json_response = array(
);
if ($games->num_rows == 0) {
- $json_response['error'] = 1;
+ $json_response['error'] = -1;
unset($json_response['files']);
$json_response['status'] = 'unknown_variant';
Commit: b29080e5082bf1f4abc200e76ccb3c9e65d0573e
https://github.com/scummvm/scummvm-sites/commit/b29080e5082bf1f4abc200e76ccb3c9e65d0573e
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-14T22:03:01+05:30
Commit Message:
INTEGRITY: Punyencode each directory in path
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 7657cd2..0f18208 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -120,9 +120,11 @@ def punyencode(orig: str) -> str:
def punyencode_filepath(filepath):
filepath = filepath.rstrip("/")
- prefix, filename = os.path.split(filepath)
+ path_components = filepath.split(os.path.sep)
+ for i, component in enumerate(path_components):
+ path_components[i] = punyencode(component)
- return os.path.join(prefix, punyencode(filename))
+ return os.path.join(*path_components)
def read_be_32(byte_stream):
Commit: 9c1bdb508c43a8c3e4da1e38f96883109d110d5b
https://github.com/scummvm/scummvm-sites/commit/9c1bdb508c43a8c3e4da1e38f96883109d110d5b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-17T23:11:10+05:30
Commit Message:
INTEGRITY: Handle filesets with no metadata/files
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index a6625e6..f5e02ea 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -6,6 +6,12 @@ header('Content-Type: application/json');
$conn = db_connect();
+$error_codes = array(
+ "unknown" => -1,
+ "success" => 0,
+ "empty" => 2
+);
+
$json_string = file_get_contents('php://input');
$json_object = json_decode($json_string);
@@ -17,6 +23,35 @@ foreach ($json_object as $key => $value) {
$game_metadata[$key] = $value;
}
+$json_response = array(
+ 'error' => $error_codes['success'],
+ 'files' => array()
+);
+
+if (count($game_metadata) == 0) {
+ if (count($json_object->files) == 0) {
+ $json_response['error'] = $error_codes['empty'];
+ unset($json_response['files']);
+ $json_response['status'] = 'empty_fileset';
+
+
+ $json_response = json_encode($json_response);
+ echo $json_response;
+ return;
+ }
+
+ $json_response['error'] = $error_codes['unknown'];
+ unset($json_response['files']);
+ $json_response['status'] = 'unknown_variant';
+
+ $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $json_response['fileset'] = $fileset_id;
+
+ $json_response = json_encode($json_response);
+ echo $json_response;
+ return;
+}
+
// Find game(s) that fit the metadata
$query = "SELECT game.id FROM game
JOIN engine ON game.engine = engine.id
@@ -27,13 +62,8 @@ AND platform = '{$game_metadata['platform']}'
AND language = '{$game_metadata['language']}'";
$games = $conn->query($query);
-$json_response = array(
- 'error' => 0,
- 'files' => array()
-);
-
if ($games->num_rows == 0) {
- $json_response['error'] = -1;
+ $json_response['error'] = $error_codes['unknown'];
unset($json_response['files']);
$json_response['status'] = 'unknown_variant';
Commit: bae271732d1d57b384d092caea508d12996688fd
https://github.com/scummvm/scummvm-sites/commit/bae271732d1d57b384d092caea508d12996688fd
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-21T20:18:03+05:30
Commit Message:
INTEGRITY: Fix require paths
Changed paths:
bin/dat_parser.php
bin/schema.php
bin/seeds.php
endpoints/validate.php
fileset.php
games_list.php
include/db_functions.php
include/pagination.php
include/user_fileset_functions.php
index.php
logs.php
user_games_list.php
diff --git a/bin/dat_parser.php b/bin/dat_parser.php
index 935a5ca..92cd848 100644
--- a/bin/dat_parser.php
+++ b/bin/dat_parser.php
@@ -1,6 +1,6 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/db_functions.php';
+require __DIR__ . '/../include/db_functions.php';
ini_set('memory_limit', '512M');
function remove_quotes($string) {
diff --git a/bin/schema.php b/bin/schema.php
index 7e23b7d..108f16d 100644
--- a/bin/schema.php
+++ b/bin/schema.php
@@ -1,6 +1,6 @@
<?php
-$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents(__DIR__ . '/../mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/bin/seeds.php b/bin/seeds.php
index e651df8..2c3d75f 100644
--- a/bin/seeds.php
+++ b/bin/seeds.php
@@ -1,6 +1,6 @@
<?php
-$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents(__DIR__ . '/../mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/endpoints/validate.php b/endpoints/validate.php
index f5e02ea..3bd7beb 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/user_fileset_functions.php';
+require __DIR__ . '/../include/user_fileset_functions.php';
header('Access-Contol-Allow-Origin: *');
header('Content-Type: application/json');
diff --git a/fileset.php b/fileset.php
index 2d6c57b..ccd0cc3 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
+require __DIR__ . '/include/pagination.php';
$filename = 'fileset.php';
$stylesheet = 'style.css';
@@ -14,7 +14,7 @@ function get_log_page($log_id) {
return intdiv($log_id, $records_per_page) + 1;
}
-$mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
+$mysql_cred = json_decode(file_get_contents(__DIR__ . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/games_list.php b/games_list.php
index 1bec999..c6950aa 100644
--- a/games_list.php
+++ b/games_list.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
+require __DIR__ . '/include/pagination.php';
$filename = "games_list.php";
$records_table = "game";
diff --git a/include/db_functions.php b/include/db_functions.php
index 99a4c69..7c44cb4 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -4,7 +4,7 @@
* Create and return a mysqli connection
*/
function db_connect() {
- $mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
+ $mysql_cred = json_decode(file_get_contents(__DIR__ . '/../mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/include/pagination.php b/include/pagination.php
index b4d95ca..7269137 100644
--- a/include/pagination.php
+++ b/include/pagination.php
@@ -22,7 +22,7 @@ function get_join_columns($table1, $table2, $mapping) {
}
function create_page($filename, $results_per_page, $records_table, $select_query, $order, $filters = array(), $mapping = array()) {
- $mysql_cred = json_decode(file_get_contents($_SERVER['DOCUMENT_ROOT'] . '/mysql_config.json'), true);
+ $mysql_cred = json_decode(file_get_contents(__DIR__ . '/../mysql_config.json'), true);
$servername = $mysql_cred["servername"];
$username = $mysql_cred["username"];
$password = $mysql_cred["password"];
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index b25d733..4a95aea 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/db_functions.php';
+require __DIR__ . '/../include/db_functions.php';
function user_calc_key($user_fileset) {
$key_string = "";
diff --git a/index.php b/index.php
index 0f16bb7..f76a86a 100644
--- a/index.php
+++ b/index.php
@@ -6,10 +6,10 @@ $api_root = '/endpoints/';
switch ($request) {
case '':
case '/':
- require $_SERVER['DOCUMENT_ROOT'] . '/index.html';
+ require __DIR__ . '/index.html';
break;
case '/api/validate':
- require $_SERVER['DOCUMENT_ROOT'] . $api_root . 'validate.php';
+ require __DIR__ . $api_root . 'validate.php';
}
?>
diff --git a/logs.php b/logs.php
index 13e25f3..53b538a 100644
--- a/logs.php
+++ b/logs.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
+require __DIR__ . '/include/pagination.php';
$filename = "logs.php";
$records_table = "log";
diff --git a/user_games_list.php b/user_games_list.php
index bbccc7c..e6706b4 100644
--- a/user_games_list.php
+++ b/user_games_list.php
@@ -1,5 +1,5 @@
<?php
-require $_SERVER['DOCUMENT_ROOT'] . '/include/pagination.php';
+require __DIR__ . '/include/pagination.php';
$filename = "user_games_list.php";
$records_table = "game";
Commit: 4b54e96fffb78f1ea7f8926636254980caf07c0c
https://github.com/scummvm/scummvm-sites/commit/4b54e96fffb78f1ea7f8926636254980caf07c0c
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-21T20:28:22+05:30
Commit Message:
INTEGRITY: User filesets added on all mismatches
Changed paths:
endpoints/validate.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 3bd7beb..586a079 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -141,9 +141,13 @@ while ($game = $games->fetch_array()) {
}
}
- if ($status != 'ok')
+ if ($status != 'ok') {
$json_response['error'] = 1;
+ $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $json_response['fileset'] = $fileset_id;
+ }
+
array_push($json_response['files'], array('status' => $status, 'name' => $filename));
}
}
Commit: 1942aba2760e21a3def96769f44ab9de04eb911d
https://github.com/scummvm/scummvm-sites/commit/1942aba2760e21a3def96769f44ab9de04eb911d
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-21T21:55:59+05:30
Commit Message:
INTEGRITY: Fix user fileset id in JSON response
Changed paths:
include/db_functions.php
include/user_fileset_functions.php
diff --git a/include/db_functions.php b/include/db_functions.php
index 7c44cb4..8c7381b 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -103,6 +103,7 @@ function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_tex
if ($existing_entry->num_rows > 0) {
$existing_entry = $existing_entry->fetch_array()[0];
+ $conn->query("SET @fileset_last = {$existing_entry}");
$category_text = "Uploaded from {$src}";
$log_text = "Duplicate of Fileset:{$existing_entry}, {$log_text}";
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 4a95aea..c904826 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -66,7 +66,7 @@ function user_insert_fileset($user_fileset, $conn) {
}
}
- $fileset_id = $conn->query("SELECT @fileset_last");
+ $fileset_id = $conn->query("SELECT @fileset_last")->fetch_array()[0];
$conn->commit();
return $fileset_id;
}
Commit: 7ee0516f6df3b9452ffb8bd87ee94e3b60508041
https://github.com/scummvm/scummvm-sites/commit/7ee0516f6df3b9452ffb8bd87ee94e3b60508041
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-21T22:33:50+05:30
Commit Message:
INTEGRITY: Display error on providing wrong args
Changed paths:
compute_hash.py
diff --git a/compute_hash.py b/compute_hash.py
index 0f18208..db3b793 100644
--- a/compute_hash.py
+++ b/compute_hash.py
@@ -2,6 +2,7 @@ import hashlib
import os
import argparse
import struct
+import sys
script_version = "0.1"
@@ -424,6 +425,13 @@ def create_dat_file(hash_of_dirs, path, checksum_size=0):
file.write(")\n\n")
+class MyParser(argparse.ArgumentParser):
+ def error(self, message):
+ sys.stderr.write('Error: %s\n' % message)
+ self.print_help()
+ sys.exit(2)
+
+
parser = argparse.ArgumentParser()
parser.add_argument("--directory",
help="Path of directory with game files")
Commit: 329b0989d70b93f76708627223fce2cb8dd9d84b
https://github.com/scummvm/scummvm-sites/commit/329b0989d70b93f76708627223fce2cb8dd9d84b
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-21T23:33:56+05:30
Commit Message:
INTEGRITY: Fix logs page no in fileset history
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index ccd0cc3..fb86979 100644
--- a/fileset.php
+++ b/fileset.php
@@ -9,9 +9,12 @@ echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
echo "<script type='text/javascript' src='{$jquery_file}'></script>\n";
echo "<script type='text/javascript' src='{$js_file}'></script>\n";
-function get_log_page($log_id) {
- $records_per_page = 25; // FIXME: Fetch this directly from logs.php
- return intdiv($log_id, $records_per_page) + 1;
+function get_log_page($log_id, $conn) {
+ $results_per_page = 25; // FIXME: Fetch this directly from logs.php
+ $num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_array()[0];
+ $num_of_pages = ceil($num_of_results / $results_per_page);
+
+ return $num_of_pages - (intdiv($log_id, $results_per_page) + 1);
}
$mysql_cred = json_decode(file_get_contents(__DIR__ . '/mysql_config.json'), true);
@@ -180,7 +183,7 @@ else {
echo "<th>Changed on</th>";
echo "<th>Log ID</th>";
while ($row = $history->fetch_assoc()) {
- $log_page = get_log_page($row['log']);
+ $log_page = get_log_page($row['log'], $conn);
echo "<tr>\n";
echo "<td>{$row['oldfileset']}</td>\n";
echo "<td>{$row['timestamp']}</td>\n";
Commit: 11ac2b31171222c81465c745ee2d2c50cf6184c1
https://github.com/scummvm/scummvm-sites/commit/11ac2b31171222c81465c745ee2d2c50cf6184c1
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-22T00:36:08+05:30
Commit Message:
INTEGRITY: Improve fileset history in fileset.php
Changed paths:
fileset.php
include/db_functions.php
diff --git a/fileset.php b/fileset.php
index fb86979..e47e1c2 100644
--- a/fileset.php
+++ b/fileset.php
@@ -172,26 +172,49 @@ if (isset($_POST['delete'])) {
echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
-// Display history
+// Display history and logs
echo "<h3>Fileset history</h3>";
-if ($history->num_rows == 0) {
- echo "<p>Fileset has no history.</p>";
+
+echo "<table>\n";
+echo "<th>Timestamp</th>";
+echo "<th>Category</th>";
+echo "<th>Description</th>";
+echo "<th>Log ID</th>";
+
+$logs = $conn->query("SELECT `timestamp`, category, `text`, id FROM log
+WHERE `text` REGEXP 'Fileset:{$id}'
+ORDER BY `timestamp` DESC, id DESC");
+
+while ($row = $logs->fetch_assoc()) {
+ $log_page = get_log_page($row['log'], $conn);
+
+ echo "<tr>\n";
+ echo "<td>{$row['timestamp']}</td>\n";
+ echo "<td>{$row['category']}</td>\n";
+ echo "<td>{$row['text']}</td>\n";
+ echo "<td><a href='logs.php?page={$log_page}'>{$row['id']}</a></td>\n";
+ echo "</tr>\n";
}
-else {
- echo "<table>\n";
- echo "<th>Old ID</th>";
- echo "<th>Changed on</th>";
- echo "<th>Log ID</th>";
- while ($row = $history->fetch_assoc()) {
+
+while ($history_row = $history->fetch_assoc()) {
+ $logs = $conn->query("SELECT `timestamp`, category, `text`, id FROM log
+ WHERE `text` REGEXP 'Fileset:{$history_row['oldfileset']}'
+ AND `category` NOT REGEXP 'merge'
+ ORDER BY `timestamp` DESC, id DESC");
+
+ while ($row = $logs->fetch_assoc()) {
$log_page = get_log_page($row['log'], $conn);
+
echo "<tr>\n";
- echo "<td>{$row['oldfileset']}</td>\n";
echo "<td>{$row['timestamp']}</td>\n";
- echo "<td><a href='logs.php?page={$log_page}'>{$row['log']}</a></td>\n";
+ echo "<td>{$row['category']}</td>\n";
+ echo "<td>{$row['text']}</td>\n";
+ echo "<td><a href='logs.php?page={$log_page}'>{$row['id']}</a></td>\n";
echo "</tr>\n";
}
- echo "</table>\n";
}
+echo "</table>\n";
+
?>
diff --git a/include/db_functions.php b/include/db_functions.php
index 8c7381b..497fb80 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -568,6 +568,12 @@ function populate_matching_games() {
if ($conn->query($query)) {
$user = 'cli:' . get_current_user();
+
+ // Merge log
+ create_log("Fileset merge", $user,
+ mysqli_real_escape_string($conn, "Merged Fileset:{$matched_game['fileset']} and Fileset:{$fileset[0][0]}"));
+
+ // Matching log
$log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user,
mysqli_real_escape_string($conn, $log_text));
Commit: e4b3497ecbce152fbfb71cc1feb9da7cd08d99ba
https://github.com/scummvm/scummvm-sites/commit/e4b3497ecbce152fbfb71cc1feb9da7cd08d99ba
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-22T20:05:54+05:30
Commit Message:
INTEGRITY: Add ip address in user fileset logs
Changed paths:
endpoints/validate.php
include/db_functions.php
include/user_fileset_functions.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 586a079..9ca433f 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -15,6 +15,11 @@ $error_codes = array(
$json_string = file_get_contents('php://input');
$json_object = json_decode($json_string);
+$ip = $_SERVER['REMOTE_ADDR'];
+// Take only first 3 bytes, set 4th byte as '.X'
+// FIXME: Assumes IPv4
+$ip = implode('.', array_slice(explode('.', $ip), 0, 3)) . '.X';
+
$game_metadata = array();
foreach ($json_object as $key => $value) {
if ($key == 'files')
@@ -44,7 +49,7 @@ if (count($game_metadata) == 0) {
unset($json_response['files']);
$json_response['status'] = 'unknown_variant';
- $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $fileset_id = user_insert_fileset($json_object->files, $ip, $conn);
$json_response['fileset'] = $fileset_id;
$json_response = json_encode($json_response);
@@ -67,7 +72,7 @@ if ($games->num_rows == 0) {
unset($json_response['files']);
$json_response['status'] = 'unknown_variant';
- $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $fileset_id = user_insert_fileset($json_object->files, $ip, $conn);
$json_response['fileset'] = $fileset_id;
}
@@ -144,7 +149,7 @@ while ($game = $games->fetch_array()) {
if ($status != 'ok') {
$json_response['error'] = 1;
- $fileset_id = user_insert_fileset($json_object->files, $conn);
+ $fileset_id = user_insert_fileset($json_object->files, $ip, $conn);
$json_response['fileset'] = $fileset_id;
}
diff --git a/include/db_functions.php b/include/db_functions.php
index 497fb80..04e6512 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -84,7 +84,7 @@ function insert_game($engine_name, $engineid, $title, $gameid, $extra, $platform
$conn->query("SET @game_last = LAST_INSERT_ID()");
}
-function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_text, $conn) {
+function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_text, $conn, $ip = '') {
$status = $detection ? "detection" : $src;
$game = "NULL";
$key = $key == "" ? "NULL" : "'{$key}'";
@@ -107,6 +107,8 @@ function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_tex
$category_text = "Uploaded from {$src}";
$log_text = "Duplicate of Fileset:{$existing_entry}, {$log_text}";
+ if ($src == 'user')
+ $log_text = "Duplicate of Fileset:{$existing_entry}, from user IP {$ip}, {$log_text}";
$user = 'cli:' . get_current_user();
create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
@@ -131,6 +133,8 @@ function insert_fileset($src, $detection, $key, $megakey, $transaction, $log_tex
$category_text = "Uploaded from {$src}";
$fileset_last = $conn->query("SELECT @fileset_last")->fetch_array()[0];
$log_text = "Created Fileset:{$fileset_last}, {$log_text}";
+ if ($src == 'user')
+ $log_text = "Created Fileset:{$fileset_last}, from user IP {$ip}, {$log_text}";
$user = 'cli:' . get_current_user();
create_log(mysqli_real_escape_string($conn, $category_text), $user, mysqli_real_escape_string($conn, $log_text));
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index c904826..23683e7 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -42,7 +42,7 @@ function user_insert_queue($user_fileset, $conn) {
$conn->query($query);
}
-function user_insert_fileset($user_fileset, $conn) {
+function user_insert_fileset($user_fileset, $ip, $conn) {
$src = 'user';
$detection = false;
$key = '';
@@ -54,7 +54,7 @@ function user_insert_fileset($user_fileset, $conn) {
// Set timestamp of fileset insertion
$conn->query(sprintf("SET @fileset_time_last = %d", time()));
- if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) {
+ if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn, $ip)) {
foreach ($user_fileset as $file) {
$file = file_json_to_array($file);
Commit: e6db6133addcfdf5707ecc2c6396095ce17b8191
https://github.com/scummvm/scummvm-sites/commit/e6db6133addcfdf5707ecc2c6396095ce17b8191
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-22T23:31:00+05:30
Commit Message:
INTEGRITY: Remove DAT filename from fileset logs
Changed paths:
include/db_functions.php
diff --git a/include/db_functions.php b/include/db_functions.php
index 04e6512..8e32947 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -296,7 +296,7 @@ function db_insert($data_arr) {
$transaction_id = $conn->query("SELECT MAX(`transaction`) FROM transactions")->fetch_array()[0] + 1;
$category_text = "Uploaded from {$src}";
- $log_text = sprintf("Started loading DAT file, filename '%s', size %d, author '%s', version %s.
+ $log_text = sprintf("Started loading DAT file, size %d, author '%s', version %s.
State '%s'. Transaction: %d",
$filepath, filesize($filepath), $author, $version, $status, $transaction_id);
@@ -321,9 +321,9 @@ function db_insert($data_arr) {
$key = $detection ? calc_key($fileset) : "";
$megakey = !$detection ? calc_megakey($fileset['rom']) : "";
- $log_text = sprintf("from filename '%s', size %d, author '%s', version %s.
+ $log_text = sprintf("size %d, author '%s', version %s.
State '%s'.",
- $filepath, filesize($filepath), $author, $version, $status);
+ filesize($filepath), $author, $version, $status);
if (insert_fileset($src, $detection, $key, $megakey, $transaction_id, $log_text, $conn)) {
foreach ($fileset["rom"] as $file) {
Commit: 6e4705b61ae1c8df08bd88b493511d1476b16657
https://github.com/scummvm/scummvm-sites/commit/6e4705b61ae1c8df08bd88b493511d1476b16657
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-23T01:55:30+05:30
Commit Message:
INTEGRITY: Add button to match user fileset
Changed paths:
fileset.php
include/db_functions.php
include/user_fileset_functions.php
js_functions.js
diff --git a/fileset.php b/fileset.php
index e47e1c2..c28bafc 100644
--- a/fileset.php
+++ b/fileset.php
@@ -1,5 +1,6 @@
<?php
require __DIR__ . '/include/pagination.php';
+require __DIR__ . '/include/user_fileset_functions.php';
$filename = 'fileset.php';
$stylesheet = 'style.css';
@@ -163,11 +164,15 @@ echo "</table>\n";
// Dev Actions
echo "<h3>Developer Actions</h3>";
echo "<button id='delete-button' type='button' onclick='delete_id({$id})'>Mark Fileset for Deletion</button>";
+echo "<button id='match-button' type='button' onclick='match_id({$id})'>Match and Merge Fileset</button>";
if (isset($_POST['delete'])) {
$conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$_POST['delete']}");
$conn->commit();
}
+if (isset($_POST['match'])) {
+ populate_matching_user_filesets();
+}
echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
diff --git a/include/db_functions.php b/include/db_functions.php
index 8e32947..af9e503 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -524,7 +524,7 @@ function populate_matching_games() {
FROM fileset
JOIN file ON file.fileset = fileset.id
JOIN filechecksum ON file.id = filechecksum.file
- WHERE fileset.game IS NULL");
+ WHERE fileset.game IS NULL AND status != 'user'");
$unmatched_files = $unmatched_files->fetch_all();
// Splitting them into different filesets
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 23683e7..0003448 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -70,5 +70,89 @@ function user_insert_fileset($user_fileset, $ip, $conn) {
$conn->commit();
return $fileset_id;
}
+
+
+/**
+ * (Attempt to) match fileset that have fileset.game as NULL
+ * This will delete the original detection fileset and replace it with the newly
+ * matched fileset
+ */
+function populate_matching_user_filesets() {
+ $conn = db_connect();
+
+ // Getting unmatched filesets
+ $unmatched_filesets = array();
+
+ $unmatched_files = $conn->query("SELECT fileset.id, filechecksum.checksum, src, status
+ FROM fileset
+ JOIN file ON file.fileset = fileset.id
+ JOIN filechecksum ON file.id = filechecksum.file
+ WHERE fileset.game IS NULL AND status = 'user'");
+ $unmatched_files = $unmatched_files->fetch_all();
+
+ // Splitting them into different filesets
+ for ($i = 0; $i < count($unmatched_files); $i++) {
+ $cur_fileset = $unmatched_files[$i][0];
+ $temp = array();
+ while ($i < count($unmatched_files) - 1 && $cur_fileset == $unmatched_files[$i][0]) {
+ array_push($temp, $unmatched_files[$i]);
+ $i++;
+ }
+ array_push($unmatched_filesets, $temp);
+ }
+
+ foreach ($unmatched_filesets as $fileset) {
+ $matching_games = find_matching_game($fileset);
+
+ if (count($matching_games) != 1) // If there is no match/non-unique match
+ continue;
+
+ $matched_game = $matching_games[0];
+
+ // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
+ $status = $fileset[0][2];
+ if ($fileset[0][2] == "dat")
+ $status = "partialmatch";
+ elseif ($fileset[0][2] == "scan")
+ $status = "fullmatch";
+
+ // Convert NULL values to string with value NULL for printing
+ $matched_game = array_map(function ($val) {
+ return (is_null($val)) ? "NULL" : $val;
+ }, $matched_game);
+
+ $category_text = "Matched from {$fileset[0][2]}";
+ $log_text = "Matched game {$matched_game['engineid']}:
+ {$matched_game['gameid']}-{$matched_game['platform']}-{$matched_game['language']}
+ variant {$matched_game['key']}. State {$status}. Fileset:{$fileset[0][0]}.";
+
+ // Updating the fileset.game value to be $matched_game["id"]
+ $query = sprintf("UPDATE fileset
+ SET game = %d, status = '%s', `key` = '%s'
+ WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
+
+ $history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
+
+ if ($conn->query($query)) {
+ $user = 'cli:' . get_current_user();
+
+ // Merge log
+ create_log("Fileset merge", $user,
+ mysqli_real_escape_string($conn, "Merged Fileset:{$matched_game['fileset']} and Fileset:{$fileset[0][0]}"));
+
+ // Matching log
+ $log_last = create_log(mysqli_real_escape_string($conn, $category_text), $user,
+ mysqli_real_escape_string($conn, $log_text));
+
+ // Add log id to the history table
+ $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
+ }
+
+ if (!$conn->commit())
+ echo "Updating matched games failed\n";
+ }
+}
+
+
?>
diff --git a/js_functions.js b/js_functions.js
index d31db37..187556e 100644
--- a/js_functions.js
+++ b/js_functions.js
@@ -11,6 +11,17 @@ function delete_id(value) {
});
}
+function match_id(value) {
+ $.ajax({
+ url: "fileset.php",
+ type: "post",
+ dataType: "json",
+ data: {
+ match: value,
+ },
+ });
+}
+
function remove_empty_inputs() {
var myForm = document.getElementById("filters-form");
var allInputs = myForm.getElementsByTagName("input");
@@ -31,4 +42,5 @@ function hyperlink(link) {
$(document).ready(function () {
$(".hidden").hide();
$("#delete-button").one("click", delete_id);
+ $("#match-button").one("click", match_id);
});
Commit: bb8d7fe7b116f2a06da9e325d579599666c097db
https://github.com/scummvm/scummvm-sites/commit/bb8d7fe7b116f2a06da9e325d579599666c097db
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-23T22:23:33+05:30
Commit Message:
INTEGRITY: Hyperlink in fileset only shows one log
Changed paths:
fileset.php
diff --git a/fileset.php b/fileset.php
index c28bafc..e97507d 100644
--- a/fileset.php
+++ b/fileset.php
@@ -10,13 +10,6 @@ echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
echo "<script type='text/javascript' src='{$jquery_file}'></script>\n";
echo "<script type='text/javascript' src='{$js_file}'></script>\n";
-function get_log_page($log_id, $conn) {
- $results_per_page = 25; // FIXME: Fetch this directly from logs.php
- $num_of_results = $conn->query("SELECT COUNT(id) FROM log")->fetch_array()[0];
- $num_of_pages = ceil($num_of_results / $results_per_page);
-
- return $num_of_pages - (intdiv($log_id, $results_per_page) + 1);
-}
$mysql_cred = json_decode(file_get_contents(__DIR__ . '/mysql_config.json'), true);
$servername = $mysql_cred["servername"];
@@ -191,13 +184,11 @@ WHERE `text` REGEXP 'Fileset:{$id}'
ORDER BY `timestamp` DESC, id DESC");
while ($row = $logs->fetch_assoc()) {
- $log_page = get_log_page($row['log'], $conn);
-
echo "<tr>\n";
echo "<td>{$row['timestamp']}</td>\n";
echo "<td>{$row['category']}</td>\n";
echo "<td>{$row['text']}</td>\n";
- echo "<td><a href='logs.php?page={$log_page}'>{$row['id']}</a></td>\n";
+ echo "<td><a href='logs.php?id={$row['id']}'>{$row['id']}</a></td>\n";
echo "</tr>\n";
}
@@ -208,13 +199,11 @@ while ($history_row = $history->fetch_assoc()) {
ORDER BY `timestamp` DESC, id DESC");
while ($row = $logs->fetch_assoc()) {
- $log_page = get_log_page($row['log'], $conn);
-
echo "<tr>\n";
echo "<td>{$row['timestamp']}</td>\n";
echo "<td>{$row['category']}</td>\n";
echo "<td>{$row['text']}</td>\n";
- echo "<td><a href='logs.php?page={$log_page}'>{$row['id']}</a></td>\n";
+ echo "<td><a href='logs.php?id={$row['id']}'>{$row['id']}</a></td>\n";
echo "</tr>\n";
}
}
Commit: 8348d707e92d8e369e8f15b38460d2d496ed231a
https://github.com/scummvm/scummvm-sites/commit/8348d707e92d8e369e8f15b38460d2d496ed231a
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-23T22:44:07+05:30
Commit Message:
UNFINISHED: Add mod_actions developer dashboard
Changed paths:
A mod_actions.php
fileset.php
include/user_fileset_functions.php
diff --git a/fileset.php b/fileset.php
index e97507d..4dd38af 100644
--- a/fileset.php
+++ b/fileset.php
@@ -164,7 +164,7 @@ if (isset($_POST['delete'])) {
$conn->commit();
}
if (isset($_POST['match'])) {
- populate_matching_user_filesets();
+ match_user_filesets();
}
echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 0003448..fec0248 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -74,10 +74,8 @@ function user_insert_fileset($user_fileset, $ip, $conn) {
/**
* (Attempt to) match fileset that have fileset.game as NULL
- * This will delete the original detection fileset and replace it with the newly
- * matched fileset
*/
-function populate_matching_user_filesets() {
+function match_user_filesets() {
$conn = db_connect();
// Getting unmatched filesets
@@ -131,6 +129,16 @@ function populate_matching_user_filesets() {
SET game = %d, status = '%s', `key` = '%s'
WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
+ if (!$conn->commit())
+ echo "Matching user fileset failed\n";
+ }
+}
+
+/*
+ * Delete the original detection fileset and replace it with the newly matched
+ * fileset
+ */
+function merge_user_filesets() {
$history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
if ($conn->query($query)) {
@@ -145,14 +153,9 @@ function populate_matching_user_filesets() {
mysqli_real_escape_string($conn, $log_text));
// Add log id to the history table
- $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
- }
-
- if (!$conn->commit())
- echo "Updating matched games failed\n";
+ $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
}
}
-
?>
diff --git a/mod_actions.php b/mod_actions.php
new file mode 100644
index 0000000..845b285
--- /dev/null
+++ b/mod_actions.php
@@ -0,0 +1,28 @@
+<?php
+// require __DIR__ . '/include/pagination.php';
+
+$stylesheet = 'style.css';
+$jquery_file = 'https://code.jquery.com/jquery-3.7.0.min.js';
+$js_file = 'js_functions.js';
+echo "<link rel='stylesheet' href='{$stylesheet}'>\n";
+echo "<script type='text/javascript' src='{$jquery_file}'></script>\n";
+echo "<script type='text/javascript' src='{$js_file}'></script>\n";
+
+
+// Dev Tools
+echo "<h3>Developer Moderation Tools</h3>";
+echo "<button id='delete-button' type='button' onclick='delete_id(0)'>Delete filesets from last uploaded DAT</button>";
+echo "<br/>";
+echo "<button id='match-button' type='button' onclick='match_id(0)'>Merge Uploaded Fileset</button>";
+echo "<br/>";
+echo "<button id='match-button' type='button' onclick='match_id(0)'>Merge User Fileset</button>";
+
+if (isset($_POST['delete'])) {
+}
+if (isset($_POST['match'])) {
+ // merge_user_filesets();
+}
+
+echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
+?>
+
Commit: 9acec1b3d55bb48fe72e36ad743fd5b4dd37d1ce
https://github.com/scummvm/scummvm-sites/commit/9acec1b3d55bb48fe72e36ad743fd5b4dd37d1ce
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-08-25T00:05:19+05:30
Commit Message:
INTEGRITY: Fix user filesets matching and merging
Changed paths:
endpoints/validate.php
fileset.php
include/db_functions.php
include/user_fileset_functions.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 9ca433f..3df3886 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -62,7 +62,6 @@ $query = "SELECT game.id FROM game
JOIN engine ON game.engine = engine.id
WHERE gameid = '{$game_metadata['gameid']}'
AND engineid = '{$game_metadata['engineid']}'
-AND extra = '{$game_metadata['extra']}'
AND platform = '{$game_metadata['platform']}'
AND language = '{$game_metadata['language']}'";
$games = $conn->query($query);
@@ -76,11 +75,12 @@ if ($games->num_rows == 0) {
$json_response['fileset'] = $fileset_id;
}
-// Check if all files in fullmatch filesets are present with user
+// Check if all files in the (first) fileset are present with user
while ($game = $games->fetch_array()) {
$fileset = $conn->query("SELECT file.id, name, size FROM file
JOIN fileset ON fileset.id = file.fileset
- WHERE fileset.game = {$game['id']} AND fileset.status = 'fullmatch'");
+ WHERE fileset.game = {$game['id']} AND
+ (status = 'fullmatch' OR status = 'partialmatch' OR status = 'detection')");
if ($fileset->num_rows == 0)
continue;
@@ -106,14 +106,14 @@ while ($game = $games->fetch_array()) {
return strcmp($a['name'], $b['name']);
});
- for ($i = 0, $j = 0; $i < count($fileset), $j < count($file_object); $i++, $j++) {
+ for ($i = 0, $j = 0; $i < count($fileset) && $j < count($file_object); $i++, $j++) {
$status = 'ok';
$db_file = $fileset[$i];
$user_file = $file_object[$j];
- $filename = $user_file->name;
+ $filename = strtolower($user_file->name);
- if ($db_file['name'] != $user_file->name) {
- if ($db_file['name'] > $user_file->name) {
+ if (strtolower($db_file['name']) != $filename) {
+ if (strtolower($db_file['name']) > $filename) {
$status = 'unknown_file';
$i--; // Retain same db_file for next iteration
}
@@ -155,6 +155,8 @@ while ($game = $games->fetch_array()) {
array_push($json_response['files'], array('status' => $status, 'name' => $filename));
}
+
+ break;
}
$json_response = json_encode($json_response);
diff --git a/fileset.php b/fileset.php
index 4dd38af..33b86db 100644
--- a/fileset.php
+++ b/fileset.php
@@ -164,7 +164,8 @@ if (isset($_POST['delete'])) {
$conn->commit();
}
if (isset($_POST['match'])) {
- match_user_filesets();
+ match_and_merge_user_filesets();
+ header("Location: {$filename}?id={$id}");
}
echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
diff --git a/include/db_functions.php b/include/db_functions.php
index af9e503..81f21ff 100644
--- a/include/db_functions.php
+++ b/include/db_functions.php
@@ -360,9 +360,9 @@ function db_insert($data_arr) {
*/
function compare_filesets($id1, $id2, $conn) {
$fileset1 = $conn->query("SELECT name, size, checksum
- FROM file WHERE fileset = '{$id1}'")->fetch_all();
+ FROM file WHERE fileset = '{$id1}'")->fetch_array();
$fileset2 = $conn->query("SELECT name, size, checksum
- FROM file WHERE fileset = '{$id2}'")->fetch_all();
+ FROM file WHERE fileset = '{$id2}'")->fetch_array();
// Sort filesets on checksum
usort($fileset1, function ($a, $b) {
@@ -390,7 +390,7 @@ function compare_filesets($id1, $id2, $conn) {
* fullmatch -> partialmatch, detection
*/
function status_to_match($status) {
- $order = array("detection", "dat", "scan", "partialmatch", "fullmatch");
+ $order = array("detection", "dat", "scan", "partialmatch", "fullmatch", "user");
return array_slice($order, 0, array_search($status, $order));
}
@@ -456,7 +456,7 @@ function find_matching_game($game_files) {
return $matching_games;
if (compare_filesets($matching_games[0]['fileset'], $game_files[0][0], $conn)) {
- $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0]}");
+ $conn->query("UPDATE fileset SET `delete` = TRUE WHERE id = {$game_files[0][0]}");
return array();
}
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index fec0248..4339f9e 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -74,8 +74,10 @@ function user_insert_fileset($user_fileset, $ip, $conn) {
/**
* (Attempt to) match fileset that have fileset.game as NULL
+ * This will delete the original detection fileset and replace it with the newly
+ * matched fileset
*/
-function match_user_filesets() {
+function match_and_merge_user_filesets() {
$conn = db_connect();
// Getting unmatched filesets
@@ -107,12 +109,7 @@ function match_user_filesets() {
$matched_game = $matching_games[0];
- // Update status depending on $matched_game["src"] (dat -> partialmatch, scan -> fullmatch)
- $status = $fileset[0][2];
- if ($fileset[0][2] == "dat")
- $status = "partialmatch";
- elseif ($fileset[0][2] == "scan")
- $status = "fullmatch";
+ $status = 'fullmatch';
// Convert NULL values to string with value NULL for printing
$matched_game = array_map(function ($val) {
@@ -129,16 +126,6 @@ function match_user_filesets() {
SET game = %d, status = '%s', `key` = '%s'
WHERE id = %d", $matched_game["id"], $status, $matched_game["key"], $fileset[0][0]);
- if (!$conn->commit())
- echo "Matching user fileset failed\n";
- }
-}
-
-/*
- * Delete the original detection fileset and replace it with the newly matched
- * fileset
- */
-function merge_user_filesets() {
$history_last = merge_filesets($matched_game["fileset"], $fileset[0][0]);
if ($conn->query($query)) {
@@ -153,9 +140,14 @@ function merge_user_filesets() {
mysqli_real_escape_string($conn, $log_text));
// Add log id to the history table
- $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
+ $conn->query("UPDATE history SET log = {$log_last} WHERE id = {$history_last}");
+ }
+
+ if (!$conn->commit())
+ echo "Updating matched games failed\n";
}
}
+
?>
Commit: a7fb08e10cfd8eac890ef35f3bc990f990614f24
https://github.com/scummvm/scummvm-sites/commit/a7fb08e10cfd8eac890ef35f3bc990f990614f24
Author: Abhinav Chennubhotla (51199011+PhoenixFlame101 at users.noreply.github.com)
Date: 2023-09-05T09:18:18+05:30
Commit Message:
INTEGRITY: Add no_metadata error code
Changed paths:
endpoints/validate.php
fileset.php
include/user_fileset_functions.php
diff --git a/endpoints/validate.php b/endpoints/validate.php
index 3df3886..1963a5c 100644
--- a/endpoints/validate.php
+++ b/endpoints/validate.php
@@ -9,7 +9,8 @@ $conn = db_connect();
$error_codes = array(
"unknown" => -1,
"success" => 0,
- "empty" => 2
+ "empty" => 2,
+ "no_metadata" => 3,
);
$json_string = file_get_contents('php://input');
@@ -45,9 +46,9 @@ if (count($game_metadata) == 0) {
return;
}
- $json_response['error'] = $error_codes['unknown'];
+ $json_response['error'] = $error_codes['no_metadata'];
unset($json_response['files']);
- $json_response['status'] = 'unknown_variant';
+ $json_response['status'] = 'no_metadata';
$fileset_id = user_insert_fileset($json_object->files, $ip, $conn);
$json_response['fileset'] = $fileset_id;
diff --git a/fileset.php b/fileset.php
index 33b86db..73a09e5 100644
--- a/fileset.php
+++ b/fileset.php
@@ -164,8 +164,8 @@ if (isset($_POST['delete'])) {
$conn->commit();
}
if (isset($_POST['match'])) {
- match_and_merge_user_filesets();
- header("Location: {$filename}?id={$id}");
+ match_and_merge_user_filesets($_POST['match']);
+ header("Location: {$filename}?id={$_POST['match']}");
}
echo "<p id='delete-confirm' class='hidden'>Fileset marked for deletion</p>"; // Hidden
diff --git a/include/user_fileset_functions.php b/include/user_fileset_functions.php
index 4339f9e..b160436 100644
--- a/include/user_fileset_functions.php
+++ b/include/user_fileset_functions.php
@@ -77,7 +77,7 @@ function user_insert_fileset($user_fileset, $ip, $conn) {
* This will delete the original detection fileset and replace it with the newly
* matched fileset
*/
-function match_and_merge_user_filesets() {
+function match_and_merge_user_filesets($id) {
$conn = db_connect();
// Getting unmatched filesets
@@ -87,7 +87,7 @@ function match_and_merge_user_filesets() {
FROM fileset
JOIN file ON file.fileset = fileset.id
JOIN filechecksum ON file.id = filechecksum.file
- WHERE fileset.game IS NULL AND status = 'user'");
+ WHERE status = 'user' AND fileset.id = {$id}");
$unmatched_files = $unmatched_files->fetch_all();
// Splitting them into different filesets
More information about the Scummvm-git-logs
mailing list