3 @author Alexey <developers.hce@gmail.com> 4 @link http://hierarchical-cluster-engine.com/ 5 @copyright Copyright © 2013 IOIX Ukraine 6 @license http://hierarchical-cluster-engine.com/license/ 7 @package HCE project node API 18 DEFAULT_CFG_FILE=
"../ini/crawling-optimizer.ini" 19 APP_NAME =
"crawling-optimizer" 23 MSG_INFO_LOAD_DEFAULT_CONFIG_FILE =
"Loading default config file: " 24 MSG_INFO_LOAD_CONFIG_FILE =
"Loading config file: " 25 MSG_INFO_LOAD_DEFAULT_SITE_ID =
"Load default site id: " 26 MSG_INFO_LOAD_SITE_ID =
"Load site id: " 29 MSG_ERROR_LOAD_CONFIG =
"Can't load config file" 30 MSG_ERROR_LOAD_LOG_CONFIG_FILE =
"Can't load logging config file" 31 MSG_ERROR_PROCESS_GENERAL =
"Can't process query" 32 MSG_ERROR_COLLECT_SITE_DATA =
"Can't collect site's data" 33 MSG_ERROR_STORE_SITE_DATA =
"Can't store site's data" 40 SQL_QUERY_NEW_URLS =
"""SELECT count(*), max(`TcDate`), min(`LastModified`), max(`LastModified`) FROM dc_urls.`urls_%s` 44 (SELECT DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute) FROM dc_sites.`sites` WHERE `Id`='%s') 46 (SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s') 60 SQL_QUERY_RECRAWL_PERIOD_START =
"""SELECT 61 DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute) 67 SQL_QUERY_RECRAWL_END =
"""SELECT max(`TcDate`) FROM dc_urls.`urls_%s` 71 (SELECT DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute) FROM dc_sites.`sites` WHERE `Id`='%s') 73 (SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s')""" 77 SQL_QUERY_RECRAWL_PERIOD_END =
"""SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s'""" 81 SQL_QUERY_NEW_SITE_TABLE =
"""CREATE TABLE IF NOT EXISTS `%s` ( 82 `host` varchar(126) DEFAULT NULL, 83 `Contents` bigint(20) unsigned NOT NULL DEFAULT '0', 84 `RecrawlStart` datetime DEFAULT NULL COMMENT 'Start date of re-crawl', 85 `RecrawlEnd` datetime DEFAULT NULL COMMENT 'End date of re-crawl', 86 `minPDate` datetime DEFAULT NULL COMMENT 'When resource was appeared ', 87 `maxPDate` datetime DEFAULT NULL, 88 `LastAdded` datetime DEFAULT NULL COMMENT 'When content was inserted to the system', 89 `CDate` datetime NOT NULL COMMENT 'Date insert row', 90 UNIQUE KEY `RecrawlEnd` (`RecrawlEnd`) 91 ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 96 SQL_QUERY_INSERT_SITE_DATA =
"""INSERT INTO `%s` VALUES('%s', %s,'%s','%s','%s','%s','%s',NOW()) ON DUPLICATE KEY UPDATE `Contents`=%s, `LastAdded`='%s', `minPDate`='%s', `maxPDate`='%s', `CDate`=NOW()"""