dc_co.Constants Namespace Reference

Variables
int	EXIT_SUCCESS = 0
int	EXIT_FAILURE = 1
int	SITE_ALL = 0
string	DEFAULT_CFG_FILE = "../ini/crawling-optimizer.ini"
string	APP_NAME = "crawling-optimizer"
string	MSG_INFO_LOAD_DEFAULT_CONFIG_FILE = "Loading default config file: "
string	MSG_INFO_LOAD_CONFIG_FILE = "Loading config file: "
string	MSG_INFO_LOAD_DEFAULT_SITE_ID = "Load default site id: "
string	MSG_INFO_LOAD_SITE_ID = "Load site id: "
string	MSG_ERROR_LOAD_CONFIG = "Can't load config file"
string	MSG_ERROR_LOAD_LOG_CONFIG_FILE = "Can't load logging config file"
string	MSG_ERROR_PROCESS_GENERAL = "Can't process query"
string	MSG_ERROR_COLLECT_SITE_DATA = "Can't collect site's data"
string	MSG_ERROR_STORE_SITE_DATA = "Can't store site's data"
string	DB_URLS = "dc_urls"
string	DB_CO = "dc_co"
string	SQL_QUERY_NEW_URLS
string	SQL_QUERY_RECRAWL_PERIOD_START
string	SQL_QUERY_RECRAWL_END
string	SQL_QUERY_RECRAWL_PERIOD_END = """SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s'"""
string	SQL_QUERY_NEW_SITE_TABLE
string	SQL_QUERY_INSERT_SITE_DATA = """INSERT INTO `%s` VALUES('%s', %s,'%s','%s','%s','%s','%s',NOW()) ON DUPLICATE KEY UPDATE `Contents`=%s, `LastAdded`='%s', `minPDate`='%s', `maxPDate`='%s', `CDate`=NOW()"""

Detailed Description

@package docstring
  @file Constants.py
  @author Alexey <developers.hce@gmail.com>
  @link http://hierarchical-cluster-engine.com/
  @copyright Copyright &copy; 2013 IOIX Ukraine
  @license http://hierarchical-cluster-engine.com/license/
  @package HCE project node API
  @since 0.1

Variable Documentation

APP_NAME

string dc_co.Constants.APP_NAME = "crawling-optimizer"

Definition at line 19 of file Constants.py.

DB_CO

string dc_co.Constants.DB_CO = "dc_co"

Definition at line 37 of file Constants.py.

DB_URLS

string dc_co.Constants.DB_URLS = "dc_urls"

Definition at line 36 of file Constants.py.

DEFAULT_CFG_FILE

string dc_co.Constants.DEFAULT_CFG_FILE = "../ini/crawling-optimizer.ini"

Definition at line 18 of file Constants.py.

EXIT_FAILURE

int dc_co.Constants.EXIT_FAILURE = 1

Definition at line 13 of file Constants.py.

EXIT_SUCCESS

int dc_co.Constants.EXIT_SUCCESS = 0

Definition at line 12 of file Constants.py.

MSG_ERROR_COLLECT_SITE_DATA

string dc_co.Constants.MSG_ERROR_COLLECT_SITE_DATA = "Can't collect site's data"

Definition at line 32 of file Constants.py.

MSG_ERROR_LOAD_CONFIG

string dc_co.Constants.MSG_ERROR_LOAD_CONFIG = "Can't load config file"

Definition at line 29 of file Constants.py.

MSG_ERROR_LOAD_LOG_CONFIG_FILE

string dc_co.Constants.MSG_ERROR_LOAD_LOG_CONFIG_FILE = "Can't load logging config file"

Definition at line 30 of file Constants.py.

MSG_ERROR_PROCESS_GENERAL

string dc_co.Constants.MSG_ERROR_PROCESS_GENERAL = "Can't process query"

Definition at line 31 of file Constants.py.

MSG_ERROR_STORE_SITE_DATA

string dc_co.Constants.MSG_ERROR_STORE_SITE_DATA = "Can't store site's data"

Definition at line 33 of file Constants.py.

MSG_INFO_LOAD_CONFIG_FILE

string dc_co.Constants.MSG_INFO_LOAD_CONFIG_FILE = "Loading config file: "

Definition at line 24 of file Constants.py.

MSG_INFO_LOAD_DEFAULT_CONFIG_FILE

string dc_co.Constants.MSG_INFO_LOAD_DEFAULT_CONFIG_FILE = "Loading default config file: "

Definition at line 23 of file Constants.py.

MSG_INFO_LOAD_DEFAULT_SITE_ID

string dc_co.Constants.MSG_INFO_LOAD_DEFAULT_SITE_ID = "Load default site id: "

Definition at line 25 of file Constants.py.

MSG_INFO_LOAD_SITE_ID

string dc_co.Constants.MSG_INFO_LOAD_SITE_ID = "Load site id: "

Definition at line 26 of file Constants.py.

SITE_ALL

int dc_co.Constants.SITE_ALL = 0

Definition at line 15 of file Constants.py.

SQL_QUERY_INSERT_SITE_DATA

string dc_co.Constants.SQL_QUERY_INSERT_SITE_DATA = """INSERT INTO `%s` VALUES('%s', %s,'%s','%s','%s','%s','%s',NOW()) ON DUPLICATE KEY UPDATE `Contents`=%s, `LastAdded`='%s', `minPDate`='%s', `maxPDate`='%s', `CDate`=NOW()"""

Definition at line 96 of file Constants.py.

SQL_QUERY_NEW_SITE_TABLE

string dc_co.Constants.SQL_QUERY_NEW_SITE_TABLE

Initial value:

=  """CREATE TABLE IF NOT EXISTS `%s` (
  `host` varchar(126) DEFAULT NULL,
  `Contents` bigint(20) unsigned NOT NULL DEFAULT '0',
  `RecrawlStart` datetime DEFAULT NULL COMMENT 'Start date of re-crawl',
  `RecrawlEnd` datetime DEFAULT NULL COMMENT 'End date of re-crawl',
  `minPDate` datetime DEFAULT NULL COMMENT 'When resource was appeared ',
  `maxPDate` datetime DEFAULT NULL,
  `LastAdded` datetime DEFAULT NULL COMMENT 'When content was inserted to the system',
  `CDate` datetime NOT NULL COMMENT 'Date insert row',
  UNIQUE KEY `RecrawlEnd` (`RecrawlEnd`)
  ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
"""

Definition at line 81 of file Constants.py.

SQL_QUERY_NEW_URLS

string dc_co.Constants.SQL_QUERY_NEW_URLS

Initial value:

=  """SELECT count(*), max(`TcDate`), min(`LastModified`), max(`LastModified`) FROM dc_urls.`urls_%s`
  WHERE
  `CDate`
  BETWEEN
  (SELECT DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute) FROM dc_sites.`sites` WHERE `Id`='%s')
  AND
  (SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s')
  AND
  `ParentMd5`<>''
  AND
  `Crawled`<>0
  AND
  `Processed`<>0
  AND
  `TagsCount`<>0
  AND
  `Status`=7"""

Definition at line 40 of file Constants.py.

SQL_QUERY_RECRAWL_END

string dc_co.Constants.SQL_QUERY_RECRAWL_END

Initial value:

=  """SELECT max(`TcDate`) FROM dc_urls.`urls_%s`
  WHERE
  `CDate`
  BETWEEN
  (SELECT DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute) FROM dc_sites.`sites` WHERE `Id`='%s')
  AND
  (SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s')"""

Definition at line 67 of file Constants.py.

SQL_QUERY_RECRAWL_PERIOD_END

string dc_co.Constants.SQL_QUERY_RECRAWL_PERIOD_END = """SELECT `RecrawlDate` FROM dc_sites.`sites` WHERE `Id`='%s'"""

Definition at line 77 of file Constants.py.

SQL_QUERY_RECRAWL_PERIOD_START

string dc_co.Constants.SQL_QUERY_RECRAWL_PERIOD_START

Initial value:

=  """SELECT
  DATE_SUB(`RecrawlDate`, INTERVAL `RecrawlPeriod` minute)
  FROM dc_sites.`sites` 
  WHERE `Id`='%s'"""

Definition at line 60 of file Constants.py.