dc_co.CrawlingOptimiser.CrawlingOptimiser Class Reference

Inheritance diagram for dc_co.CrawlingOptimiser.CrawlingOptimiser:

Collaboration diagram for dc_co.CrawlingOptimiser.CrawlingOptimiser:

Classes
class	Meta

Public Member Functions
def	__init__ (self)
def	setup (self)
def	run (self)
def	loadConfig (self)
def	loadLogConfigFile (self)
def	loadOptions (self)
def	process (self)
def	collectSiteData (self)
def	storeSiteData (self)

Public Attributes
	exit_code
	logger
	message_queue
	site_id
	recrawl_dict
	site_features
	local_wrapper
	remote_wrapper
	remote_host
	config

Detailed Description

Definition at line 30 of file CrawlingOptimiser.py.

Constructor & Destructor Documentation

__init__()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.__init__

(

self

)

Definition at line 41 of file CrawlingOptimiser.py.

  def __init__(self):
    # call base class __init__ method
    foundation.CementApp.__init__(self)
    self.exit_code = CONSTS.EXIT_SUCCESS
    self.logger = None
    self.message_queue = []
    # self.url_table = None
    self.site_id = None
    self.recrawl_dict = {}
    self.site_features = {}
    self.local_wrapper = None
    self.remote_wrapper = None
    self.remote_host = None

Member Function Documentation

collectSiteData()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.collectSiteData

(

self

)

Definition at line 172 of file CrawlingOptimiser.py.

  def collectSiteData(self):
    site_data_dict = {}
    if self.site_id is not None:
      try:
        # New Contents
        query = CONSTS.SQL_QUERY_NEW_URLS % (self.site_id, self.site_id, self.site_id)
        response = self.remote_wrapper.customRequest(query, CONSTS.DB_URLS)
        if response is not None:
          self.logger.info("response: %s" % str(response))
          site_data_dict["Contents"] = response[0][0]
          site_data_dict["LastAdded"] = response[0][1]
          site_data_dict["minPDate"] = response[0][2]
          site_data_dict["maxPDate"] = response[0][3]

        # Recrawl start
        query = CONSTS.SQL_QUERY_RECRAWL_PERIOD_START % (self.site_id)
        response = self.remote_wrapper.customRequest(query, CONSTS.DB_URLS)
        if response is not None:
          self.logger.info("response: %s" % str(response))
          site_data_dict["RecrawlStart"] = response[0][0]

        # Recrawl end
        query = CONSTS.SQL_QUERY_RECRAWL_PERIOD_END % (self.site_id)
        response = self.remote_wrapper.customRequest(query, CONSTS.DB_URLS)
        if response is not None:
          self.logger.info("response: %s" % str(response))
          site_data_dict["RecrawlEnd"] = response[0][0]

      except Exception, err:
        self.logger.error(CONSTS.MSG_ERROR_COLLECT_SITE_DATA + ' ' + str(err))
    self.logger.info("site_data_dict: %s" % str(site_data_dict))
    return site_data_dict

Here is the caller graph for this function:

loadConfig()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.loadConfig

(

self

)

Definition at line 89 of file CrawlingOptimiser.py.

  def loadConfig(self):
    try:
      self.config = ConfigParser.ConfigParser()
      self.config.optionxform = str

      # config argument
      if self.pargs.config:
        self.config.read(self.pargs.config)
        self.message_queue.append(CONSTS.MSG_INFO_LOAD_CONFIG_FILE + str(self.pargs.config))
      else:
        self.config.read(CONSTS.DEFAULT_CFG_FILE)
        self.message_queue.append(CONSTS.MSG_INFO_LOAD_DEFAULT_CONFIG_FILE + CONSTS.DEFAULT_CFG_FILE)

      # site argument
      if self.pargs.site:
        self.site_id = self.pargs.site
        self.message_queue.append(CONSTS.MSG_INFO_LOAD_SITE_ID + str(self.pargs.site))
      else:
        self.site_id = CONSTS.SITE_ALL
        self.message_queue.append(CONSTS.MSG_INFO_LOAD_DEFAULT_SITE_ID + str(CONSTS.SITE_ALL))

    except Exception, err:
      print CONSTS.MSG_ERROR_LOAD_CONFIG, err.message
      raise

Here is the caller graph for this function:

loadLogConfigFile()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.loadLogConfigFile

(

self

)

Definition at line 118 of file CrawlingOptimiser.py.

  def loadLogConfigFile(self):
    try:
      # print str(vars(self.config))
      log_conf_file = self.config.get("Application", "log")
      logging.config.fileConfig(log_conf_file)
      self.logger = Utils.MPLogger().getLogger()
    except Exception, err:
      print CONSTS.MSG_ERROR_LOAD_LOG_CONFIG_FILE, err.message
      raise

Here is the call graph for this function:

Here is the caller graph for this function:

loadOptions()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.loadOptions

(

self

)

Definition at line 132 of file CrawlingOptimiser.py.

  def loadOptions(self):
    try:
      # remote host
      remote_db_task_ini = self.config.get(self.__class__.__name__, "db-task_ini_remote")
      remote_cfgParser = ConfigParser.ConfigParser()
      remote_cfgParser.read(remote_db_task_ini)
      self.remote_wrapper = DBTasksWrapper(remote_cfgParser)
      self.remote_host = remote_cfgParser.get("TasksManager", "db_host")
      # local host
      local_db_task_ini = self.config.get(self.__class__.__name__, "db-task_ini_local")
      local_cfgParser = ConfigParser.ConfigParser()
      local_cfgParser.read(local_db_task_ini)
      self.local_wrapper = DBTasksWrapper(local_cfgParser)
    except Exception, err:
      self.logger.error(CONSTS.MSG_ERROR_LOAD_LOG_CONFIG_FILE)
      self.logger.error(str(err.message))
      raise

Here is the caller graph for this function:

process()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.process

(

self

)

Definition at line 154 of file CrawlingOptimiser.py.

  def process(self):
    # log message buffer
    for msg in self.message_queue:
      self.logger.info(msg)

    if self.site_id is not None:
      try:
        # collect site's data
        self.recrawl_dict[self.site_id] = self.collectSiteData()
        self.logger.info("self.recrawl_dict: %s" % str(self.recrawl_dict))
        # store site's data
        self.storeSiteData()
      except Exception, err:
        self.logger.error(CONSTS.MSG_ERROR_PROCESS_GENERAL + ' ' + str(err))

Here is the call graph for this function:

Here is the caller graph for this function:

run()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.run

(

self

)

Definition at line 66 of file CrawlingOptimiser.py.

  def run(self):
    # call base class run method
    foundation.CementApp.run(self)

    # config section
    self.loadConfig()

    # load logger config file
    self.loadLogConfigFile()

    # load mandatory options
    self.loadOptions()

    # make processing
    self.process()

    # Finish logging
    self.logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)

Here is the call graph for this function:

setup()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.setup

(

self

)

Definition at line 57 of file CrawlingOptimiser.py.

  def setup(self):
    # call base class setup method
    foundation.CementApp.setup(self)
    self.args.add_argument('-c', '--config', action='store', metavar='config_file', help='config ini-file')
    self.args.add_argument('-s', '--site', action='store', metavar='site alias', help='site alias')

storeSiteData()

def dc_co.CrawlingOptimiser.CrawlingOptimiser.storeSiteData

(

self

)

Definition at line 208 of file CrawlingOptimiser.py.

  def storeSiteData(self):
    try:
      if True:  # self.recrawl_dict[self.site_id]["Contents"]>0:
        # Create New table if not exists
        query = CONSTS.SQL_QUERY_NEW_SITE_TABLE % (self.site_id)
        response = self.local_wrapper.customRequest(query, CONSTS.DB_CO)
        if response is not None:
          self.logger.info("response: %s" % str(response))

        # Put site's data to the site's table
        query = CONSTS.SQL_QUERY_INSERT_SITE_DATA % \
          (self.site_id, \
          self.remote_host, \
          self.recrawl_dict[self.site_id]["Contents"], \
          self.recrawl_dict[self.site_id]["RecrawlStart"], \
          self.recrawl_dict[self.site_id]["RecrawlEnd"], \
          self.recrawl_dict[self.site_id]["minPDate"], \
          self.recrawl_dict[self.site_id]["maxPDate"], \
          self.recrawl_dict[self.site_id]["LastAdded"], \
          self.recrawl_dict[self.site_id]["Contents"], \
          self.recrawl_dict[self.site_id]["LastAdded"], \
          self.recrawl_dict[self.site_id]["minPDate"], \
          self.recrawl_dict[self.site_id]["maxPDate"])
        response = self.local_wrapper.customRequest(query, CONSTS.DB_CO)
        if response is not None:
          self.logger.info("response: %s" % str(response))
      else:
        self.logger.info("Zero contents.")
    except Exception, err:
      self.logger.error(CONSTS.MSG_ERROR_STORE_SITE_DATA + ' ' + str(err))

Member Data Documentation

config

dc_co.CrawlingOptimiser.CrawlingOptimiser.config

Definition at line 91 of file CrawlingOptimiser.py.

exit_code

dc_co.CrawlingOptimiser.CrawlingOptimiser.exit_code

Definition at line 44 of file CrawlingOptimiser.py.

local_wrapper

dc_co.CrawlingOptimiser.CrawlingOptimiser.local_wrapper

Definition at line 51 of file CrawlingOptimiser.py.

logger

dc_co.CrawlingOptimiser.CrawlingOptimiser.logger

Definition at line 45 of file CrawlingOptimiser.py.

message_queue

dc_co.CrawlingOptimiser.CrawlingOptimiser.message_queue

Definition at line 46 of file CrawlingOptimiser.py.

recrawl_dict

dc_co.CrawlingOptimiser.CrawlingOptimiser.recrawl_dict

Definition at line 49 of file CrawlingOptimiser.py.

remote_host

dc_co.CrawlingOptimiser.CrawlingOptimiser.remote_host

Definition at line 53 of file CrawlingOptimiser.py.

remote_wrapper

dc_co.CrawlingOptimiser.CrawlingOptimiser.remote_wrapper

Definition at line 52 of file CrawlingOptimiser.py.

site_features

dc_co.CrawlingOptimiser.CrawlingOptimiser.site_features

Definition at line 50 of file CrawlingOptimiser.py.

site_id

dc_co.CrawlingOptimiser.CrawlingOptimiser.site_id

Definition at line 48 of file CrawlingOptimiser.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_co/CrawlingOptimiser.py