Merge pull request #114 from nesfit/hashlist

Hashlist

Dockerfile

@@ -90,7 +90,13 @@ COPY . /srv/fitcrack/
 WORKDIR /srv/fitcrack
 
 # Apply fitcrack specific changes to boinc
-RUN patch -p0 < installer/fitcrack_changes_in_boinc.patch
+# First check if the patch is not already applied locally (e.g. installer was used before)
+RUN if ! patch -p0 --dry-run --silent < installer/fitcrack_changes_in_boinc.patch 2>/dev/null; then \
+    echo "Patch already applied or failed to apply in dry run. Skipping."; \
+else \
+    patch -p0 < installer/fitcrack_changes_in_boinc.patch; \
+fi
+
 
 # Copy server files to BOINC server Root
 RUN cp -f server/src/headers/*.h              boinc/sched/

remove_docker_installation.sh

@@ -6,7 +6,7 @@
 docker kill $(docker ps --no-trunc --filter name=fitcrack -q) > /dev/null 2>&1
 docker rm $(docker ps -a --no-trunc --filter name=fitcrack -q) > /dev/null 2>&1
 docker network rm $(docker network ls -q --no-trunc --filter name=fitcrack) > /dev/null 2>&1
-docker volume rm $(docker volume ls --filter name=fitcrack) --force > /dev/null 2>&1
+docker volume rm $(docker volume ls -q --filter name=fitcrack) --force > /dev/null 2>&1
 docker container prune --force
 docker volume prune --force
 rm -rf fitcrack-data

server/sql/10_create_tables.sql

@@ -59,13 +59,28 @@ CREATE TABLE IF NOT EXISTS `fc_dictionary` (
 
 -- --------------------------------------------------------
 
+--
+-- Table definition for `fc_hash_list`
+--
+
+CREATE TABLE IF NOT EXISTS`fc_hash_list` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `hash_type` int(11) unsigned,
+  `name` varchar(255) NOT NULL,
+  `added` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `deleted` tinyint(1) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB  DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1 ;
+
+-- --------------------------------------------------------
+
 --
 -- Table definition for `fc_hash`
 --
 
 CREATE TABLE IF NOT EXISTS`fc_hash` (
   `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
-  `job_id` bigint(20) unsigned NOT NULL,
+  `hash_list_id` bigint(20) unsigned NOT NULL,
   `hash_type` int(11) unsigned NOT NULL,
   `hash` longblob NOT NULL,
   `result` longtext DEFAULT NULL,
@@ -322,8 +337,10 @@ CREATE TABLE IF NOT EXISTS `fc_job` (
   `kill` int(11) NOT NULL DEFAULT '0',
   `batch_id` int(11),
   `queue_position` int(11),
+  `hash_list_id` bigint(20) unsigned NOT NULL,
   PRIMARY KEY (`id`),
-  KEY `batch_id` (`batch_id`)
+  KEY `batch_id` (`batch_id`),
+  KEY `hash_list_id` (`hash_list_id`)
 ) ENGINE=InnoDB  DEFAULT CHARSET=utf8 COLLATE=utf8_bin AUTO_INCREMENT=1 ;
 
 --

server/sql/30_insert_data.sql

@@ -40,11 +40,24 @@ INSERT INTO `fc_dictionary` (`id`, `name`, `path`, `password_distribution`, `key
 (8,	'english.txt',	'english.txt',	'1:26;2:147;3:827;4:2609;5:4822;6:7111;7:8672;8:8486;9:7354;10:5570;11:3764;12:2349;13:1383;14:661;15:317;16:113;17:56;18:27;19:6;20:3;21:2;22:1;28:1;54:1;', 54308,	'2018-08-18 12:00:00',	0,	0);
 
 
+--
+-- Insert default hash lists 
+--
+
+INSERT INTO `fc_hash_list` (`id`, `hash_type`, `name`, `added`, `deleted`) VALUES
+(1, 0, 'BENCH_ALL', '2018-08-18 12:00:00', 1),
+(2, 0, 'MD5 Big Hash List', '2018-08-18 12:00:00', 0), 
+(3, 3200, 'sample-dict-bcrypt hash list', '2018-08-18 12:00:00', 0), 
+(4, 18000, 'sample-mask-sha3 hash list', '2018-08-18 12:00:00', 0),
+(5, 1700, 'sample-combinator-bcrypt hash list', '2018-08-18 12:00:00', 0), 
+(6, 1700, 'sample-pcfg-sha512 hash list', '2018-08-18 12:00:00', 0), 
+(7, 0, 'sample-prince-md5 hash list', '2018-08-18 12:00:00', 0);
+
 --
 -- Insert default hashes
 --
 
-INSERT INTO `fc_hash` (`id`, `job_id`, `hash_type`, `hash`, `result`, `added`, `time_cracked`) VALUES
+INSERT INTO `fc_hash` (`id`, `hash_list_id`, `hash_type`, `hash`, `result`, `added`, `time_cracked`) VALUES
 (1,	2,	0,	'1ffd9e753c8054cc61456ac7fac1ac89',	NULL,	'2018-08-18 12:00:00',	NULL),
 (2,	2,	0,	'79e262a81dd19d40ae008f74eb59edce',	NULL,	'2018-08-18 12:00:00',	NULL),
 (3,	2,	0,	'6517217c0041e7f36e9eae5caca8b69e',	NULL,	'2018-08-18 12:00:00',	NULL),
@@ -6612,27 +6625,27 @@ INSERT INTO `fc_bin` (`name`) VALUES
 -- Insert default BENCH_ALL job and sample jobs
 --
 
-INSERT INTO `fc_job` (`id`, `attack`, `attack_mode`, `attack_submode`, `distribution_mode`, `hash_type`, `status`, `keyspace`, `hc_keyspace`, `indexes_verified`, `current_index`, `current_index_2`, `time`, `name`, `comment`, `time_start`, `time_end`, `workunit_sum_time`, `seconds_per_workunit`, `charset1`, `charset2`, `charset3`, `charset4`, `rules`, `rule_left`, `rule_right`, `markov_hcstat`, `markov_threshold`, `grammar_id`, `min_password_len`, `max_password_len`, `min_elem_in_chain`, `max_elem_in_chain`, `optimized`, `deleted`) VALUES
-(1, 'mask', 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, NOW(), 'BENCH_ALL', 'This is default job for benchmarking all hosts.', NULL, NULL, 0, 3600, '', '', '', '', NULL, '', '', NULL, 0, NULL, 0, 0, 0, 0, 1, 1),
-(2, 'dictionary',	0,	0,	0,  0,	0,	92431,	92431,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-dict-md5-quick',	'Default hashcat MD5 hashlist',	NULL,	NULL,	0,	120, '',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0),
-(3, 'dictionary',	0,	0,	0,  3200,	0,	226082,	226082,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-dict-bcrypt',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0),
-(4, 'mask',	3,	0,	0,  18000,	0,	217180147158,	13270583,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-mask-sha3',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0),
-(5, 'combinator',	1,	0,	0,  3200,	0,	397000,	1000,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-combinator-bcrypt',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0),
-(6,	'pcfg',	9,	0,	0,  1700,	0,	1096,	1096,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-pcfg-sha512',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, 3, 0, 0, 0, 0, 1, 0),
-(7, 'prince',	8,	0,	0,  0,	0,	2306859,	2306859,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-prince-md5',	'',	NULL,	NULL,	0,	120,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 1, 8, 1, 8, 1, 0);
+INSERT INTO `fc_job` (`id`, `attack`, `attack_mode`, `attack_submode`, `distribution_mode`, `hash_type`, `status`, `keyspace`, `hc_keyspace`, `indexes_verified`, `current_index`, `current_index_2`, `time`, `name`, `comment`, `time_start`, `time_end`, `workunit_sum_time`, `seconds_per_workunit`, `charset1`, `charset2`, `charset3`, `charset4`, `rules`, `rule_left`, `rule_right`, `markov_hcstat`, `markov_threshold`, `grammar_id`, `min_password_len`, `max_password_len`, `min_elem_in_chain`, `max_elem_in_chain`, `optimized`, `deleted`, `hash_list_id`) VALUES
+(1, 'mask', 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, NOW(), 'BENCH_ALL', 'This is default job for benchmarking all hosts.', NULL, NULL, 0, 3600, '', '', '', '', NULL, '', '', NULL, 0, NULL, 0, 0, 0, 0, 1, 1, 1),
+(2, 'dictionary',	0,	0,	0,  0,	0,	92431,	92431,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-dict-md5-quick',	'Default hashcat MD5 hash list',	NULL,	NULL,	0,	120, '',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0, 2),
+(3, 'dictionary',	0,	0,	0,  3200,	0,	226082,	226081, 0,	0,	0,	'2018-08-18 12:00:00',	'sample-dict-bcrypt',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0, 3),
+(4, 'mask',	3,	0,	0,  18000,	0,	217180147158,	13270583,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-mask-sha3',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0, 4),
+(5, 'combinator',	1,	0,	0,  3200,	0,	397000,	1000,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-combinator-bcrypt',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 0, 0, 0, 0, 1, 0, 5),
+(6,	'pcfg',	9,	0,	0,  1700,	0,	1096,	1096,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-pcfg-sha512',	'',	NULL,	NULL,	0,	60,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, 3, 0, 0, 0, 0, 1, 0, 6),
+(7, 'prince',	8,	0,	0,  0,	0,	2306859,	2306859,	0,	0,	0,	'2018-08-18 12:00:00',	'sample-prince-md5',	'',	NULL,	NULL,	0,	120,	'',	'',	'',	'',	NULL,	'',	'',	'',	0, NULL, 1, 8, 1, 8, 1, 0, 7);
 
 --
 -- Insert default job dictionaries
 --
 
 INSERT INTO `fc_job_dictionary` (`id`, `job_id`, `dictionary_id`, `current_index`, `current_pos`, `is_left`) VALUES
 (1,	2,	3,	0,	0,	1),
-(2,	2,	2,	0,	0,	1),
+(2,	2,	1, 0,	0,	1),
 (3,	2,	8,	0,	0,	1),
 (4,	3,	1,	0,	0,	1),
-(5,	5,	2,	0,	0,	1),
+(5,	5,	1, 0,	0,	1),
 (6,	5,	6,	0,	0,	0),
-(7,	7,	2,	0,	0,	1);
+(7,	7,	1, 0,	0,	1);
 
 
 --

server/src/headers/Database/SqlLoader.h

@@ -2,6 +2,7 @@
  * @file SqlLoader.h
  * @brief Header file for common MySQL access
  * @authors Lukas Zobal (zobal.lukas(at)gmail.com)
+ * @authors Radek Hranicky (hranicky(at)fit.vut.cz)
  * @date 12. 12. 2018
  * @license MIT, see LICENSE
  */
@@ -240,6 +241,14 @@ class CSqlLoader {
          */
         std::vector<Config::Ptr<CDictionary>> loadJobDictionaries(uint64_t jobId);
 
+
+        /**
+         * @brief Returns ID of the hashlist associated with the job
+         * @param jobId Job ID
+         * @return Hashlist ID
+         */
+        uint64_t getHashlistId(uint64_t jobId);
+
         /**
          * @brief Returns vector of hashes as strings, possibly even binary ones
          * @param jobId Job ID which hashes we search for

server/src/source/Database/SqlLoader.cpp

@@ -2,6 +2,7 @@
  * @file SqlLoader.cpp
  * @brief Source file for common MySQL access
  * @authors Lukas Zobal (zobal.lukas(at)gmail.com)
+ * @authors Radek Hranicky (hranicky(at)fit.vut.cz)
  * @date 12. 12. 2018
  * @license MIT, see LICENSE
  */
@@ -304,11 +305,56 @@ CSqlLoader::loadJobDictionaries(uint64_t jobId) {
       Config::tableNameJobDictionary.c_str()));
 }
 
+uint64_t CSqlLoader::getHashlistId(uint64_t jobId)
+{
+    std::vector<std::string> result;
+    uint64_t hashlistId;
+
+    updateSql(formatQuery("SELECT hash_list_id FROM `%s` WHERE `id` = %" PRIu64 ";",
+                          Config::tableNameJob.c_str(), jobId));
+    MYSQL_RES* sqlResult;
+    sqlResult = mysql_store_result(boinc_db.mysql);
+    if (!sqlResult)
+    {
+        Tools::printDebugTimestamp("Problem with DB query.\nShutting down now.\n");
+        boinc_db.close();
+        exit(1);
+    }
+
+    MYSQL_ROW row;
+    if ((row = mysql_fetch_row(sqlResult)))
+    {
+        if (row[0])
+        {
+            hashlistId = std::stoull(row[0]); // Convert string with ID to uint64_t
+        } else {
+            Tools::printDebugTimestamp("Job with ID %d does not have a hashlist specified.\n", jobId);
+            boinc_db.close();
+            exit(1);
+        }
+    } else {
+        Tools::printDebugTimestamp("Job with ID %d not found.\n", jobId);
+        boinc_db.close();
+        exit(1);
+    }
+
+    mysql_free_result(sqlResult);
+    Tools::printDebugJob(Config::DebugType::Log, jobId,
+                         "Hashlist ID is %d for job %d.\n", hashlistId, jobId);
+
+    return hashlistId;
+}
+
 std::vector<std::string> CSqlLoader::loadJobHashes(uint64_t jobId)
 {
     std::vector<std::string> result;
-    updateSql(formatQuery("SELECT REPLACE(TO_BASE64(`hash`), '\n', '') FROM `%s` WHERE `job_id` = %" PRIu64 " AND `result` IS NULL ;",
-                          Config::tableNameHash.c_str(), jobId));
+
+    /** Get the ID of the hashlist associated with the job */
+    uint64_t hashlistId = this->getHashlistId(jobId);
+
+    /** Get hashes from the hashlist */
+    updateSql(formatQuery("SELECT REPLACE(TO_BASE64(`hash`), '\n', '') FROM `%s` WHERE `hash_list_id` = %" PRIu64 " AND `result` IS NULL ;",
+                          Config::tableNameHash.c_str(), hashlistId));
 
     MYSQL_RES* sqlResult;
     sqlResult = mysql_store_result(boinc_db.mysql);
@@ -439,8 +485,12 @@ std::string CSqlLoader::getDictFileName(uint64_t dictId)
 
 bool CSqlLoader::isAnythingCracked(uint64_t jobId)
 {
-    return getSqlNumber(formatQuery("SELECT COUNT(*) FROM `%s` WHERE `job_id` = %" PRIu64 " AND `result` IS NOT NULL ;",
-            Config::tableNameHash.c_str(), jobId)) != 0;
+    /** Get the ID of the hashlist associated with the job */
+    uint64_t hashlistId = this->getHashlistId(jobId);
+
+    /** Check is there is any cracked hash in the hashlist **/
+    return getSqlNumber(formatQuery("SELECT COUNT(*) FROM `%s` WHERE `hash_list_id` = %" PRIu64 " AND `result` IS NOT NULL ;",
+            Config::tableNameHash.c_str(), hashlistId)) != 0;
 }