Inheritance diagram for dc_crawler.RTCFinalizer.RTCFinalizer:

Collaboration diagram for dc_crawler.RTCFinalizer.RTCFinalizer:

Classes
class	Meta

Public Member Functions
def	__init__ (self)

def	setup (self)

def	run (self)

def	getBatchFromInput (self)

def	getURLContent (self)

def	sendURLContent (self)

def	saveBatchToFile (self)

def	getURLContentFromBatch (self)

def	process (self)

def	deleteURLContent (self)

def	selectSiteProperty (self, batchItem, propName)

Public Attributes
	logger

	batch

	items

	exitCode

	urlContentResponse

	rb

	rc

	dbTask

Static Public Attributes
string	MSG_ERROR_PARSE_CMD_PARAMS = "Error parse command line parameters."

string	MSG_ERROR_EMPTY_CONFIG_FILE_NAME = "Config file name is empty."

string	MSG_ERROR_WRONG_CONFIG_FILE_NAME = "Config file name is wrong"

string	MSG_ERROR_LOAD_APP_CONFIG = "Error loading application config file."

string	MSG_ERROR_READ_LOG_CONFIG = "Error read log config file."

string	MSG_ERROR_READ_DB_TASK_CONFIG = "Error read db-task config file."

string	MSG_ERROR_DELETE_URL = "Delete url has failed"

string	MSG_DELETE_URL_OK = "URL was deleted"

string	FINALIZER_OPTION_LOG = "log"

string	FINALIZER_OPTION_DB_TASK_INI = "db_task_ini"

Private Member Functions
def	__initApp (self)

def	__loadAppConfig (self, configName)

def	__loadLogConfig (self, configName)

def	__loadDBTaskConfig (self, configName)

Detailed Description

Definition at line 33 of file RTCFinalizer.py.

Constructor & Destructor Documentation

◆ init()

def dc_crawler.RTCFinalizer.RTCFinalizer.__init__ ( self )

Definition at line 58 of file RTCFinalizer.py.

   def __init__(self):
     # call base class __init__ method
     foundation.CementApp.__init__(self)
 
     self.logger = None
     self.batch = None
     self.items = None
     self.exitCode = APP_CONSTS.EXIT_SUCCESS
     self.urlContentResponse = None
     self.rb = None
     self.rc = None
     self.dbTask = None
 
 

Member Function Documentation

◆ __initApp()

def dc_crawler.RTCFinalizer.RTCFinalizer.__initApp ( self )

private

Definition at line 97 of file RTCFinalizer.py.

   def __initApp(self):
     if self.pargs.config:
       confLogFileName, confDBTaskName = self.__loadAppConfig(self.pargs.config)
       self.__loadLogConfig(confLogFileName)
       self.dbTask = DBTasksManager(self.__loadDBTaskConfig(confDBTaskName))
     else:
       raise Exception(self.MSG_ERROR_LOAD_APP_CONFIG)
     self.rb = self.pargs.rb
     self.rc = int(self.pargs.rc) if self.pargs.rc is not None else None
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ __loadAppConfig()

def dc_crawler.RTCFinalizer.RTCFinalizer.__loadAppConfig	(	self,
		configName
	)

private

Definition at line 112 of file RTCFinalizer.py.

   def __loadAppConfig(self, configName):
     # variable for result
     confLogFileName = ""
     confDBTaskName = ""
 
     try:
       config = ConfigParser.ConfigParser()
       config.optionxform = str
 
       readOk = config.read(configName)
 
       if len(readOk) == 0:
         raise Exception(self.MSG_ERROR_WRONG_CONFIG_FILE_NAME + ": " + configName)
 
       if config.has_section(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME):
         confLogFileName = str(config.get(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME, self.FINALIZER_OPTION_LOG))
         confDBTaskName = str(config.get(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME, \
                                         self.FINALIZER_OPTION_DB_TASK_INI))
 
     except Exception, err:
       raise Exception(self.MSG_ERROR_LOAD_APP_CONFIG + ' ' + str(err))
 
     return confLogFileName, confDBTaskName
 
 

Here is the caller graph for this function:

◆ __loadDBTaskConfig()

def dc_crawler.RTCFinalizer.RTCFinalizer.__loadDBTaskConfig	(	self,
		configName
	)

private

Definition at line 159 of file RTCFinalizer.py.

   def __loadDBTaskConfig(self, configName):
     # return config parser
     config = None
     if isinstance(configName, str) and len(configName) > 0:
       try:
         config = ConfigParser.ConfigParser()
         config.optionxform = str
         config.read(configName)
 
       except Exception, err:
         raise Exception(self.MSG_ERROR_READ_DB_TASK_CONFIG + ' ' + str(err))
 
     return config
 
 

Here is the caller graph for this function:

◆ __loadLogConfig()

def dc_crawler.RTCFinalizer.RTCFinalizer.__loadLogConfig	(	self,
		configName
	)

private

Definition at line 141 of file RTCFinalizer.py.

   def __loadLogConfig(self, configName):
     try:
       if isinstance(configName, str) and len(configName) == 0:
         raise Exception(self.MSG_ERROR_EMPTY_CONFIG_FILE_NAME)
 
       logging.config.fileConfig(configName)
 
       # call rotation log files and initialization logger
       self.logger = Utils.MPLogger().getLogger()
 
     except Exception, err:
       raise Exception(self.MSG_ERROR_READ_LOG_CONFIG + ' ' + str(err))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ deleteURLContent()

def dc_crawler.RTCFinalizer.RTCFinalizer.deleteURLContent ( self )

Definition at line 339 of file RTCFinalizer.py.

   def deleteURLContent(self):
     items = self.batch.items
     urlDeleteRequest = []
     num_of_items = len(items)
     self.logger.debug("Num of items to delete in batch: <<%s>>" % (num_of_items))
     item_no = 1
     for item in self.items:
       if item is None:
         continue
       siteId = item.siteId
       url = item.urlObj.url
       urlId = item.urlId
       self.logger.debug("Delete item #%s: siteId: <<%s>>, urlId: <<%s>>, url: <<%s>>" % (item_no, siteId, urlId, url))
       urlDeleteRequest.append(dc_event.URLDelete(siteId, url, reason=dc_event.URLDelete.REASON_RT_FINALIZER))
       item_no = item_no + 1
     self.dbTask.dbTaskMode = self.batch.dbMode
     drceSyncTasksCoverObj = DC_CONSTS.DRCESyncTasksCover(DC_CONSTS.EVENT_TYPES.URL_DELETE, urlDeleteRequest)
     responseDRCESyncTasksCover = self.dbTask.process(drceSyncTasksCoverObj)
     urlDeleteResponse = responseDRCESyncTasksCover.eventObject
     self.logger.debug("urlDeleteResponse: %s", varDump(urlDeleteResponse))
     for status in urlDeleteResponse.statuses:
       if status:
         self.logger.debug(self.MSG_DELETE_URL_OK)
       else:
         self.exitCode = APP_CONSTS.EXIT_FAILURE
         self.logger.debug(self.MSG_ERROR_DELETE_URL)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getBatchFromInput()

def dc_crawler.RTCFinalizer.RTCFinalizer.getBatchFromInput ( self )

Definition at line 174 of file RTCFinalizer.py.

   def getBatchFromInput(self):
     try:
       # read pickled batch object from stdin and unpickle it
       input_pickled_object = sys.stdin.read()
       # self.logger.debug("input_pickled_object: %s", varDump(input_pickled_object))
 
       # print input_pickled_object
       input_data = (pickle.loads(input_pickled_object))
       # self.logger.debug("input_data: %s", varDump(input_data))
 
       # print("Batch item: siteId: %s, urlId: %s" %(input_data.siteId, input_data.urlId))
       self.batch = input_data
       self.items = self.batch.items
 #       self.logger.debug("Batch: %s", varDump(self.batch, stringifyType=0, maxDepth=10))
     except Exception, err:
       raise Exception('getBatchFromInput error: ' + str(err))
 
 

Here is the caller graph for this function:

◆ getURLContent()

def dc_crawler.RTCFinalizer.RTCFinalizer.getURLContent ( self )

Definition at line 192 of file RTCFinalizer.py.

   def getURLContent(self):
     urlContentRequest = []
     num_of_items = len(self.items)
     self.logger.debug("Num of items in batch: <<%s>>" % (num_of_items))
     item_no = 1
     for item in self.items:
       if not item:
         urlContentRequest.append(None)
         self.logger.debug("Item is None.")
       else:
         siteId = item.siteId
         url = item.urlObj.url
         urlId = item.urlId
         self.logger.debug("Item #%s: siteId: <<%s>>, urlId: <<%s>>, url: <<%s>>" % (item_no, siteId, urlId, url))
         _urlContentRequest = dc_event.URLContentRequest(siteId, url)
         _urlContentRequest.dbFieldsList = ["Status", "Crawled", "Processed", "ContentType", "Charset", "ErrorMask", \
                                            "CrawlingTime", "ProcessingTime", "HTTPCode", "Size", "LinksI", "LinksE", \
                                            "RawContentMd5", "LastModified", "CDate", "UDate", "TagsMask", "TagsCount", \
                                            "PDate", "ContentURLMd5", "Batch_Id"]
         urlContentRequest.append(_urlContentRequest)
       item_no = item_no + 1
     self.dbTask.dbTaskMode = self.batch.dbMode
     drceSyncTasksCoverObj = DC_CONSTS.DRCESyncTasksCover(DC_CONSTS.EVENT_TYPES.URL_CONTENT, urlContentRequest)
     responseDRCESyncTasksCover = self.dbTask.process(drceSyncTasksCoverObj)
     self.urlContentResponse = responseDRCESyncTasksCover.eventObject
     self.logger.debug("urlContentResponse: %s", varDump(obj=self.urlContentResponse, strTypeMaxLen=5000))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getURLContentFromBatch()

def dc_crawler.RTCFinalizer.RTCFinalizer.getURLContentFromBatch ( self )

Definition at line 261 of file RTCFinalizer.py.

   def getURLContentFromBatch(self):
     self.urlContentResponse = []
     attributes = []
     for item in self.items:
       url = item.urlObj.url
       if item.urlPutObj is not None:
         # self.logger.debug("item.urlPutObj.putDict.data: %s", varDump(item.urlPutObj.putDict["data"]))
         try:
           if len(item.urlObj.attributes) > 0:
             self.logger.debug("item.urlPutObj.attributes: %s", varDump(item.urlObj.attributes))
             attributes = item.urlObj.attributes
         except Exception, err:
           self.logger.error("load attributes failed: %s", str(err))
 
         if item.urlPutObj.putDict["cDate"] is not None:
           contents = [dc_event.Content(item.urlPutObj.putDict["data"], item.urlPutObj.putDict["cDate"],
                                        dc_event.Content.CONTENT_PROCESSOR_CONTENT)]
         else:
           contents = [dc_event.Content(item.urlPutObj.putDict["data"],
                                        typeId=dc_event.Content.CONTENT_PROCESSOR_CONTENT)]
       else:
         contents = []
       rawContents = None
       isFetchRawContent = self.selectSiteProperty(item, "FETCH_RAW_CONTENT")
       if item.urlObj.urlPut is not None and isFetchRawContent is not None and int(isFetchRawContent) == 1:
         rawContents = [dc_event.Content(item.urlObj.urlPut.putDict["data"], item.urlObj.urlPut.putDict["cDate"],
                                         dc_event.Content.CONTENT_RAW_CONTENT)]
       urlContentResponse = dc_event.URLContentResponse(url, rawContents, processedContents=contents)
       urlContentResponse.status = 7
       urlContentResponse.urlMd5 = item.urlObj.urlMd5
       urlContentResponse.siteId = item.siteId
       urlContentResponse.contentURLMd5 = item.urlObj.contentURLMd5
       urlContentResponse.rawContentMd5 = item.urlObj.rawContentMd5
       urlContentResponse.attributes = attributes
       urlContentResponse.dbFields = {"Status":item.urlObj.status,
                                      "Crawled":item.urlObj.crawled,
                                      "Processed":item.urlObj.processed,
                                      "ContentType":item.urlObj.contentType,
                                      "Charset":item.urlObj.charset,
                                      "ErrorMask":item.urlObj.errorMask,
                                      "CrawlingTime":item.urlObj.crawlingTime,
                                      "ProcessingTime":item.urlObj.processingTime,
                                      "HttpCode":item.urlObj.httpCode,
                                      "Size":item.urlObj.size,
                                      "LinksI":item.urlObj.linksI,
                                      "LinksE":item.urlObj.linksE,
                                      "RawContentMd5":item.urlObj.rawContentMd5,
                                      "LastModified":item.urlObj.lastModified,
                                      "CDate":item.urlObj.CDate,
                                      "UDate":item.urlObj.UDate,
                                      "TagsMask":item.urlObj.tagsMask,
                                      "TagsCount":item.urlObj.tagsCount,
                                      "PDate":item.urlObj.pDate,
                                      "ContentURLMd5":item.urlObj.contentURLMd5,
                                      "BatchId":item.urlObj.batchId}
 
       if item.urlPutObj is not None and "properties" in item.urlPutObj.putDict:
         urlContentResponse.itemProperties = item.urlPutObj.putDict["properties"]
 
       self.urlContentResponse.append(urlContentResponse)
     self.logger.debug("urlContentResponse: %s", varDump(obj=self.urlContentResponse, strTypeMaxLen=5000))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process()

def dc_crawler.RTCFinalizer.RTCFinalizer.process ( self )

Definition at line 324 of file RTCFinalizer.py.

   def process(self):
     self.getBatchFromInput()
     # Check is Real-Time crawling
     if self.batch.crawlerType == dc_event.Batch.TYPE_REAL_TIME_CRAWLER:
       self.logger.debug("Real-Time crawling batch")
       self.getURLContentFromBatch()
       self.deleteURLContent()
     else:
       self.logger.debug("Regular crawling batch")
       self.getURLContent()
     self.saveBatchToFile()
     if self.exitCode != self.rc:
       self.sendURLContent()
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ run()

def dc_crawler.RTCFinalizer.RTCFinalizer.run ( self )

Definition at line 79 of file RTCFinalizer.py.

   def run(self):
     # call base class run method
     foundation.CementApp.run(self)
 
     # call initialization application
     self.__initApp()
 
     # call internal processing
     self.process()
 
     # Finish logging
     self.logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)
 
 

Here is the call graph for this function:

◆ saveBatchToFile()

def dc_crawler.RTCFinalizer.RTCFinalizer.saveBatchToFile ( self )

Definition at line 225 of file RTCFinalizer.py.

   def saveBatchToFile(self):
     if self.rb is not None:
       self.logger.debug("batchSaveFile is = " + str(self.rb))
       urlCleanupList = []
       contentCheck = ContentCheck()
       for item in self.items:
         if item.siteObj is not None and item.siteObj.fetchType == BaseFetcher.TYP_AUTO:
           if item.urlPutObj is not None and contentCheck.lookMetricsinContent(item.urlPutObj):
             self.logger.debug(">>> start checkUrlPutObj")
             metricsApplying = self.selectSiteProperty(item, "FINALIZER_METRICS")
             toRecrawl = contentCheck.checkUrlPutObj(item.urlPutObj, contentCheck.CHECK_TYPE_SIMPLE, metricsApplying)
           else:
             self.logger.debug(">>> start urlObj")
             toRecrawl = contentCheck.checkUrlObj(item.urlObj)
           if not toRecrawl:
             urlCleanup = dc_event.URLCleanup(item.urlObj.siteId, item.urlObj.url)
             urlCleanup.urlType = dc_event.URLStatus.URL_TYPE_MD5
             urlCleanup.url = item.urlObj.urlMd5
             urlCleanupList.append(urlCleanup)
             item.siteObj.fetchType = BaseFetcher.TYP_DYNAMIC
             item.urlObj.status = dc_event.URL.STATUS_SELECTED_CRAWLING
             item.urlObj.crawled = 0
             item.urlObj.urlPut = None
             item.urlPutObj = None
             if self.rc is not None:
               self.exitCode = self.rc
       if len(urlCleanupList) > 0:
         drceSyncTasksCoverObj = DC_CONSTS.DRCESyncTasksCover(DC_CONSTS.EVENT_TYPES.URL_CLEANUP, urlCleanupList)
         self.dbTask.process(drceSyncTasksCoverObj)
       fd = open(self.rb, "w")
       if fd is not None:
         pickleObj = pickle.dumps(self.batch)
         fd.write(pickleObj)
         fd.close()
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ selectSiteProperty()

def dc_crawler.RTCFinalizer.RTCFinalizer.selectSiteProperty	(	self,
		batchItem,
		propName
	)

Definition at line 372 of file RTCFinalizer.py.

   def selectSiteProperty(self, batchItem, propName):
     ret = None
     if batchItem.properties is not None and propName in batchItem.properties:
       ret = batchItem.properties[propName]
     elif batchItem.siteObj is not None and batchItem.siteObj.properties is not None:
       for elem in batchItem.siteObj.properties:
         if elem["name"] == propName:
           ret = elem["value"]
           break
     return ret
 

Here is the caller graph for this function:

◆ sendURLContent()

def dc_crawler.RTCFinalizer.RTCFinalizer.sendURLContent ( self )

Definition at line 220 of file RTCFinalizer.py.

   def sendURLContent(self):
     print pickle.dumps(self.urlContentResponse)
     sys.stdout.flush()
 
 

Here is the caller graph for this function:

◆ setup()

def dc_crawler.RTCFinalizer.RTCFinalizer.setup ( self )

Definition at line 73 of file RTCFinalizer.py.

   def setup(self):
     # call base class setup method
     foundation.CementApp.setup(self)
 
 

Member Data Documentation

◆ batch

dc_crawler.RTCFinalizer.RTCFinalizer.batch

Definition at line 63 of file RTCFinalizer.py.

◆ dbTask

dc_crawler.RTCFinalizer.RTCFinalizer.dbTask

Definition at line 69 of file RTCFinalizer.py.

◆ exitCode

dc_crawler.RTCFinalizer.RTCFinalizer.exitCode

Definition at line 65 of file RTCFinalizer.py.

◆ FINALIZER_OPTION_DB_TASK_INI

string dc_crawler.RTCFinalizer.RTCFinalizer.FINALIZER_OPTION_DB_TASK_INI = "db_task_ini"

static

Definition at line 49 of file RTCFinalizer.py.

◆ FINALIZER_OPTION_LOG

string dc_crawler.RTCFinalizer.RTCFinalizer.FINALIZER_OPTION_LOG = "log"

static

Definition at line 48 of file RTCFinalizer.py.

◆ items

dc_crawler.RTCFinalizer.RTCFinalizer.items

Definition at line 64 of file RTCFinalizer.py.

◆ logger

dc_crawler.RTCFinalizer.RTCFinalizer.logger

Definition at line 62 of file RTCFinalizer.py.

◆ MSG_DELETE_URL_OK

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_DELETE_URL_OK = "URL was deleted"

static

Definition at line 45 of file RTCFinalizer.py.

◆ MSG_ERROR_DELETE_URL

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_DELETE_URL = "Delete url has failed"

static

Definition at line 44 of file RTCFinalizer.py.

◆ MSG_ERROR_EMPTY_CONFIG_FILE_NAME

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_EMPTY_CONFIG_FILE_NAME = "Config file name is empty."

static

Definition at line 37 of file RTCFinalizer.py.

◆ MSG_ERROR_LOAD_APP_CONFIG

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_LOAD_APP_CONFIG = "Error loading application config file."

static

Definition at line 39 of file RTCFinalizer.py.

◆ MSG_ERROR_PARSE_CMD_PARAMS

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_PARSE_CMD_PARAMS = "Error parse command line parameters."

static

Definition at line 36 of file RTCFinalizer.py.

◆ MSG_ERROR_READ_DB_TASK_CONFIG

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_READ_DB_TASK_CONFIG = "Error read db-task config file."

static

Definition at line 41 of file RTCFinalizer.py.

◆ MSG_ERROR_READ_LOG_CONFIG

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_READ_LOG_CONFIG = "Error read log config file."

static

Definition at line 40 of file RTCFinalizer.py.

◆ MSG_ERROR_WRONG_CONFIG_FILE_NAME

string dc_crawler.RTCFinalizer.RTCFinalizer.MSG_ERROR_WRONG_CONFIG_FILE_NAME = "Config file name is wrong"

static

Definition at line 38 of file RTCFinalizer.py.

◆ rb

dc_crawler.RTCFinalizer.RTCFinalizer.rb

Definition at line 67 of file RTCFinalizer.py.

◆ rc

dc_crawler.RTCFinalizer.RTCFinalizer.rc

Definition at line 68 of file RTCFinalizer.py.

◆ urlContentResponse

dc_crawler.RTCFinalizer.RTCFinalizer.urlContentResponse

Definition at line 66 of file RTCFinalizer.py.

The documentation for this class was generated from the following file:

sources/hce/dc_crawler/RTCFinalizer.py

Classes

Public Member Functions

Public Attributes

Static Public Attributes

Private Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ __initApp()

◆ __loadAppConfig()

◆ __loadDBTaskConfig()

◆ __loadLogConfig()

◆ deleteURLContent()

◆ getBatchFromInput()

◆ getURLContent()

◆ getURLContentFromBatch()

◆ process()

◆ run()

◆ saveBatchToFile()

◆ selectSiteProperty()

◆ sendURLContent()

◆ setup()

Member Data Documentation

◆ batch

◆ dbTask

◆ exitCode

◆ FINALIZER_OPTION_DB_TASK_INI

◆ FINALIZER_OPTION_LOG

◆ items

◆ logger

◆ MSG_DELETE_URL_OK

◆ MSG_ERROR_DELETE_URL

◆ MSG_ERROR_EMPTY_CONFIG_FILE_NAME

◆ MSG_ERROR_LOAD_APP_CONFIG

◆ MSG_ERROR_PARSE_CMD_PARAMS

◆ MSG_ERROR_READ_DB_TASK_CONFIG

◆ MSG_ERROR_READ_LOG_CONFIG

◆ MSG_ERROR_WRONG_CONFIG_FILE_NAME

◆ rb

◆ rc

◆ urlContentResponse

◆ init()