Inheritance diagram for dc.ClientInterfaceService.ClientInterfaceService:

Collaboration diagram for dc.ClientInterfaceService.ClientInterfaceService:

Public Member Functions
def	__init__ (self, configParser, connectBuilderLight)

def	onSitesManagerRoute (self, event)

def	onBatchTasksManagerRealTimeRoute (self, event)

def	onDCClientRoute (self, event)

def	registerEvent (self, event)

def	getRequestEvent (self, event)

def	unregisterEvent (self, event)

def	mergeResultsData (self, eventType, eventObj, eventCookie)

def	sortURLContentResults (self, itemObject, criterion)

def	sortURLFetchResults (self, itemObject, criterion)

def	mergeResultsURLContent (self, newItemsList, clientResponseItem)

def	mergeResultsSiteStatus (self, mergedSite, clientResponseItem)

def	mergeResultsSiteFields (self, siteToMerge, siteMergeWith)

def	mergeResultsSiteFind (self, newItemsList, clientResponseItem)

def	mergeResultsURLFetch (self, newItemsList, clientResponseItem)

def	mergeResultsGeneralResponse (self, mergedGeneralResponse, clientResponseItem)

def	mergeResultsBatch (self, newItemsList, clientResponseItem)

def	getUnixTimeFromString (self, buf, dateFormat='%Y-%m-%d %H:%M:%S', valueType=0)

Public Member Functions inherited from app.BaseServerManager.BaseServerManager
def	__init__ (self, poller_manager=None, admin_connection=None, conectionLightBuilder=None, exceptionForward=False, dumpStatVars=True)
	constructor More...

def	addConnection (self, name, connection)

def	setEventHandler (self, eventType, handler)
	set event handler rewrite the current handler for eventType More...

def	send (self, connect_name, event)
	send event More...

def	reply (self, event, reply_event)
	wrapper for sending event in reply for event More...

def	poll (self)
	poll function polling connections receive as multipart msg, the second argument is pickled pyobj More...

def	process (self, event)
	process event call the event handler method that was set by user or on_unhandled_event method if not set More...

def	run (self)

def	is_connection_registered (self, name)
	check is a connection was registered in a instance of BaseServerManager i object More...

def	on_poll_timeout (self)
	function will call every time when ConnectionTimeout exception arrive More...

def	on_unhandled_event (self, event)
	function will call every time when arrive doesn't set handler for event type of event.evenType More...

def	build_poller_list (self)

def	clear_poller (self)

def	onAdminState (self, event)
	onAdminState event handler process admin SHUTDOWN command More...

def	onAdminFetchStatData (self, event)
	onAdminState event handler process admin command More...

def	onAdminSuspend (self, event)
	onAdminState event handler process admin command More...

def	getStatDataFields (self, fields)
	getStatDataFields returns stat data from storage More...

def	getSystemStat (self)
	getSystemStat returns stat data for system indicators: RAMV, RAMR and CPU More...

def	getConfigVarsFields (self, fields)
	getConfigVarsFields returns config vars from storage More...

def	onAdminGetConfigVars (self, event)
	onAdminGetConfigVars event handler process getConfigVars admin command, fill and return config vars array from internal storage More...

def	onAdminSetConfigVars (self, event)
	onAdminSetConfigVars event handler process setConfigVars admin command More...

def	setConfigVars (self, setConfigVars)
	processSetConfigVars sets config vars in storage More...

def	sendAdminReadyEvent (self)
	send ready event to notify adminInterfaceService More...

def	createLogMsg (self, event)
	from string message from event object More...

def	initStatFields (self, connect_name)
	add record in statFields More...

def	updateStatField (self, field_name, value, operation=STAT_FIELDS_OPERATION_ADD)
	update values of stat field - default sum More...

def	processSpecialConfigVars (self, name, value)
	send ready event to notify adminInterfaceService More...

def	getLogLevel (self)
	Get log level from first of existing loggers. More...

def	setLogLevel (self, level)
	Set log level for all loggers. More...

def	saveStatVarsDump (self)
	Save stat vars in json file. More...

def	loadStatVarsDump (self)
	Load stat vars in json file. More...

def	getStatVarsDumpFileName (self)
	Get stat vars file name. More...

def	createDBIDict (self, configParser)

Public Attributes
	sitesManager

	batchTasksManagerRealTime

	processEvents

Public Attributes inherited from app.BaseServerManager.BaseServerManager
	dumpStatVars

	poller_manager

	eventBuilder

	exit_flag

	pollTimeout

	connections

	event_handlers

	statFields
	stat fields container More...

	configVars

	exceptionForward

Static Public Attributes
string	CONFIG_SECTION = "ClientInterfaceService"

string	CONFIG_SERVER_HOST = "serverHost"

string	CONFIG_SERVER_PORT = "serverPort"

string	CONFIG_SITES_MANAGER = "clientSitesManager"

string	SERVER_CONNECTION_NAME = "server"

string	CONNECTION_PREFIX = "Connection"

string	CONFIG_BATCH_TASKS_MANAGER_REALTIME = "clientBatchTasksManagerRealTime"

string	CONFIG_DRCE_NODES = "DRCENodes"

Static Public Attributes inherited from app.BaseServerManager.BaseServerManager
string	ADMIN_CONNECT_ENDPOINT = "Admin"

string	ADMIN_CONNECT_CLIENT = "Admin"

int	POLL_TIMEOUT_DEFAULT = 3000

int	STAT_FIELDS_OPERATION_ADD = 0

int	STAT_FIELDS_OPERATION_SUB = 1

int	STAT_FIELDS_OPERATION_SET = 2

int	STAT_FIELDS_OPERATION_INIT = 3

string	POLL_TIMEOUT_CONFIG_VAR_NAME = "POLL_TIMEOUT"

string	LOG_LEVEL_CONFIG_VAR_NAME = "LOG_LEVEL"

string	STAT_DUMPS_DEFAULT_DIR = "/tmp/"

string	STAT_DUMPS_DEFAULT_NAME = "%APP_NAME%_%CLASS_NAME%_stat_vars.dump"

dictionary	LOGGERS_NAMES = {APP_CONSTS.LOGGER_NAME, "dc", "dtm", "root", ""}

Detailed Description

Definition at line 29 of file ClientInterfaceService.py.

Constructor & Destructor Documentation

◆ init()

def dc.ClientInterfaceService.ClientInterfaceService.__init__	(	self,
		configParser,
		connectBuilderLight
	)

Constructor

Definition at line 45 of file ClientInterfaceService.py.

   def __init__(self, configParser, connectBuilderLight):
     '''
     Constructor
     '''
     BaseServerManager.__init__(self)
 
     serverHost = configParser.get(self.CONFIG_SECTION, self.CONFIG_SERVER_HOST)
     serverPort = configParser.get(self.CONFIG_SECTION, self.CONFIG_SERVER_PORT)
     server = serverHost + ":" + str(serverPort)
     self.sitesManager = configParser.get(self.CONFIG_SECTION, self.CONFIG_SITES_MANAGER)
     self.batchTasksManagerRealTime = configParser.get(self.CONFIG_SECTION, self.CONFIG_BATCH_TASKS_MANAGER_REALTIME)
 
     try:
       self.configVars[self.CONFIG_DRCE_NODES] = configParser.get(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME,
                                                                  self.CONFIG_DRCE_NODES)
     except ConfigParser.NoOptionError:
       self.configVars[self.CONFIG_DRCE_NODES] = 1
 
     serverConnection = connectBuilderLight.build(consts.SERVER_CONNECT, server, consts.TCP_TYPE)
     sitesManagerConnection = connectBuilderLight.build(consts.CLIENT_CONNECT, self.sitesManager)
     batchTasksManagerRealTimeConnection = connectBuilderLight.build(consts.CLIENT_CONNECT,
                                                                     self.batchTasksManagerRealTime)
 
     self.addConnection(self.SERVER_CONNECTION_NAME + self.CONNECTION_PREFIX, serverConnection)
     self.addConnection(str(self.sitesManager) + self.CONNECTION_PREFIX, sitesManagerConnection)
     self.addConnection(str(self.batchTasksManagerRealTime) + self.CONNECTION_PREFIX,
                        batchTasksManagerRealTimeConnection)
 
     self.setEventHandler(EVENT_TYPES.SITE_NEW, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SITE_UPDATE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SITE_STATUS, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SITE_DELETE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SITE_CLEANUP, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SITE_FIND, self.onSitesManagerRoute)
 
     self.setEventHandler(EVENT_TYPES.URL_NEW, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_STATUS, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_UPDATE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_FETCH, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_DELETE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_CLEANUP, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_CONTENT, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.SQL_CUSTOM, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_PUT, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_HISTORY, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.URL_STATS, self.onSitesManagerRoute)
 
     self.setEventHandler(EVENT_TYPES.PROXY_NEW, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_UPDATE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_DELETE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_STATUS, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_FIND, self.onSitesManagerRoute)
 
     self.setEventHandler(EVENT_TYPES.ATTR_SET, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_UPDATE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_DELETE, self.onSitesManagerRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_FETCH, self.onSitesManagerRoute)
 
     self.setEventHandler(EVENT_TYPES.SITE_NEW_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SITE_UPDATE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SITE_STATUS_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SITE_DELETE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SITE_CLEANUP_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SITE_FIND_RESPONSE, self.onDCClientRoute)
 
     self.setEventHandler(EVENT_TYPES.URL_NEW_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_STATUS_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_UPDATE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_FETCH_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_DELETE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_CLEANUP_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_CONTENT_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.SQL_CUSTOM_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_PUT_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_HISTORY_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.URL_STATS_RESPONSE, self.onDCClientRoute)
 
     self.setEventHandler(EVENT_TYPES.PROXY_NEW_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_UPDATE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_DELETE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_STATUS_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.PROXY_FIND_RESPONSE, self.onDCClientRoute)
 
     self.setEventHandler(EVENT_TYPES.ATTR_SET_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_UPDATE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_DELETE_RESPONSE, self.onDCClientRoute)
     self.setEventHandler(EVENT_TYPES.ATTR_FETCH_RESPONSE, self.onDCClientRoute)
 
     self.setEventHandler(EVENT_TYPES.BATCH, self.onBatchTasksManagerRealTimeRoute)
     self.setEventHandler(EVENT_TYPES.BATCH_RESPONSE, self.onDCClientRoute)
 
     # map of incoming event, which are in processing
     # event.uid => event without eventObj field
     self.processEvents = dict()
 
 
 

Member Function Documentation

◆ getRequestEvent()

def dc.ClientInterfaceService.ClientInterfaceService.getRequestEvent	(	self,
		event
	)

Definition at line 204 of file ClientInterfaceService.py.

   def getRequestEvent(self, event):
     return self.processEvents[event.uid]

Here is the caller graph for this function:

◆ getUnixTimeFromString()

def dc.ClientInterfaceService.ClientInterfaceService.getUnixTimeFromString	(	self,
		buf,
		dateFormat = `'%Y-%m-%d %H:%M:%S'`,
		valueType = `0`
	)

Definition at line 749 of file ClientInterfaceService.py.

   def getUnixTimeFromString(self, buf, dateFormat='%Y-%m-%d %H:%M:%S', valueType=0):
     ret = 0
 
     try:
       ret = time.mktime(time.strptime(str(buf), dateFormat))
       if valueType == 0:
         ret = int(ret)
     except Exception as err:
       logger.error("Error get date from: `%s` with format: `%s` : %s", str(buf), dateFormat, str(err))
 
     return ret
 
 

Here is the caller graph for this function:

◆ mergeResultsBatch()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsBatch	(	self,
		newItemsList,
		clientResponseItem
	)

Definition at line 715 of file ClientInterfaceService.py.

   def mergeResultsBatch(self, newItemsList, clientResponseItem):
     itemsList = clientResponseItem.itemObject
 
     # If this is first item in results list, init itemObject with empty list
     if newItemsList is None:
       newItemsList = []
 
     if isinstance(itemsList, list):
       # For each URLContentResponse object in the clientResponseItem.itemObject
       for urlContentResponse in itemsList:
         # If raw or processed content exists
         # if len(urlContentResponse.rawContents) > 0 or len(urlContentResponse.processedContents) > 0:
         # Add content to list
         if urlContentResponse is not None:
           urlContentResponse.host = clientResponseItem.host
           newItemsList.append(urlContentResponse)
           logger.debug("Added urlContentResponse:\n" + Utils.varDump(urlContentResponse))
         else:
           logger.debug("Rejected urlContentResponse")
         # else:
         #  logger.debug("Empty contents lists in urlContentResponse:\n" + Utils.varDump(urlContentResponse))
     else:
       # Object is null or wrong type
       logger.error("Wrong type of clientResponseItem.itemObject\n" + Utils.varDump(itemsList))
 
     return newItemsList
 
 

Here is the caller graph for this function:

◆ mergeResultsData()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsData	(	self,
		eventType,
		eventObj,
		eventCookie
	)

Definition at line 223 of file ClientInterfaceService.py.

   def mergeResultsData(self, eventType, eventObj, eventCookie):
     if isinstance(eventObj, EventObjects.ClientResponse):
       # If number of client response items grater than one perform the merging, else leave untouched
       if len(eventObj.itemsList) > 1:
         if isinstance(eventObj.itemsList[0], EventObjects.ClientResponseItem):
           # Create new list of ClientResponseItem items and fill it with first item from response
           newItemObject = None
           newHost = ""
           newPort = ""
           newNode = ""
           newTime = ""
           newErrorMessage = ""
           newErrorCode = ""
           logger.debug("Merging, response items: %s", str(len(eventObj.itemsList)))
           mergedCounter = 0
           # Cycle response items
           for clientResponseItem in eventObj.itemsList:
             # If response item exists
             if clientResponseItem.itemObject is not None:
               mergedCounter = mergedCounter + 1
               logger.debug("clientResponseItem:\n" + Utils.varDump(clientResponseItem))
               newHost += clientResponseItem.host + ";"
               newPort += str(clientResponseItem.port) + ";"
               newNode += clientResponseItem.node + ";"
               newTime += str(clientResponseItem.time) + ";"
               newErrorMessage += clientResponseItem.errorMessage
               newErrorCode += str(clientResponseItem.errorCode) + ";"
               if eventType == EVENT_TYPES.SITE_FIND_RESPONSE:
                 # Merge SITE_FIND operation results response
                 newItemObject = self.mergeResultsSiteFind(newItemObject, clientResponseItem)
               elif eventType == EVENT_TYPES.URL_FETCH_RESPONSE or eventType == EVENT_TYPES.URL_STATUS_RESPONSE:
                 # Merge URL_FETCH operation results response
                 newItemObject = self.mergeResultsURLFetch(newItemObject, clientResponseItem)
               elif eventType == EVENT_TYPES.URL_CONTENT_RESPONSE:
                 # Merge URL_CONTENT operation results response
                 newItemObject = self.mergeResultsURLContent(newItemObject, clientResponseItem)
               elif eventType == EVENT_TYPES.SITE_STATUS_RESPONSE:
                 # Merge SITE_STATUS operation results response
                 newItemObject = self.mergeResultsSiteStatus(newItemObject, clientResponseItem)
               # elif eventType == EVENT_TYPES.SITE_NEW_RESPONSE:
               #  #Merge SITE_NEW operation results response
               #  newItemObject = self.mergeResultsGeneralResponse(newItemObject, clientResponseItem.itemObject)
               #  #Merge BATCH_RESPONSE operation results response
               elif eventType == EVENT_TYPES.BATCH_RESPONSE:
                 # Merge URL_CONTENT operation results response
                 newItemObject = self.mergeResultsBatch(newItemObject, clientResponseItem)
               else:
                 l = {EVENT_TYPES.SITE_NEW_RESPONSE, EVENT_TYPES.SITE_UPDATE_RESPONSE, EVENT_TYPES.SITE_DELETE_RESPONSE,
                      EVENT_TYPES.SITE_CLEANUP_RESPONSE, EVENT_TYPES.URL_NEW_RESPONSE, EVENT_TYPES.URL_UPDATE_RESPONSE,
                      EVENT_TYPES.URL_DELETE_RESPONSE, EVENT_TYPES.URL_CLEANUP_RESPONSE, EVENT_TYPES.URL_UPDATE_RESPONSE,
                      EVENT_TYPES.URL_DELETE_RESPONSE, EVENT_TYPES.URL_CLEANUP_RESPONSE}  # pylint: disable=unused-import
                 if eventType in l:
                   # Merge another result types that no need special processing of fields but check and leave just one
                   newItemObject = self.mergeResultsGeneralResponse(newItemObject, clientResponseItem)
             else:
               # Object is null or wrong type
               logger.error("The clientResponseItem.itemObject is None!")
           # Replace items list with newly created and merged if itemObject is set
           if newItemObject is not None:
             eventObj.itemsList = [eventObj.itemsList[0]]
             eventObj.itemsList[0].errorCode = newErrorCode.rstrip(";")
             eventObj.itemsList[0].errorMessage = newErrorMessage.rstrip(";")
             eventObj.itemsList[0].itemObject = newItemObject
             eventObj.itemsList[0].host = newHost.rstrip(";")
             eventObj.itemsList[0].port = newPort.rstrip(";")
             eventObj.itemsList[0].node = newNode.rstrip(";")
             eventObj.itemsList[0].time = newTime.rstrip(";")
             if isinstance(newItemObject, list):
               itemsNumber = len(newItemObject)
             else:
               itemsNumber = 1
             logger.debug("Merged with " + str(mergedCounter) + " response objects, " + \
                          str(itemsNumber) + " merged items.")
             # Does event is URL_CONTENT_RESPONSE and sort criterion defined
             if eventType == EVENT_TYPES.URL_CONTENT_RESPONSE and eventCookie is not None\
               and isinstance(eventCookie, dict) and EventObjects.URLFetch.CRITERION_ORDER in eventCookie and\
               len(eventCookie[EventObjects.URLFetch.CRITERION_ORDER]) > 0 and\
               eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0].strip() != '':
               # TODO: now use only criterion of the first item of request items list
               eventObj.itemsList[0].itemObject = self.sortURLContentResults(newItemObject,
                                                  eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0])  #  pylint: disable=C0330
               logger.debug("URL_CONTENT results sorted by the: %s",
                            str(eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0]))
             # Does event is URL_FETCH_RESPONSE and sort criterion defined
             elif eventType == EVENT_TYPES.URL_FETCH_RESPONSE and eventCookie is not None\
               and isinstance(eventCookie, dict) and EventObjects.URLFetch.CRITERION_ORDER in eventCookie and\
               len(eventCookie[EventObjects.URLFetch.CRITERION_ORDER]) > 0 and\
               eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0].strip() != '':
               # TODO: now use only criterion of the first item of request items list
               eventObj.itemsList[0].itemObject = self.sortURLFetchResults(newItemObject,
                                                  eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0])  #  pylint: disable=C0330
               logger.debug("URL_CONTENT results sorted by the: %s",
                            str(eventCookie[EventObjects.URLFetch.CRITERION_ORDER][0]))
             else:
               logger.debug("Sort conditions are not satisfied, results not sorted.")
           else:
             logger.debug("No items collected while merge procedure, merge skipped.")
         else:
           # Wrong type
           logger.error("Wrong eventObj.itemsList[0] type " + str(type(eventObj.itemsList[0])) + \
                          " expected EventObjects.ClientResponseItem\n" + Utils.varDump(eventObj.itemsList[0]))
     else:
       logger.error("Wrong eventObj type " + str(type(eventObj)) + " expected EventObjects.ClientResponse\n" + \
                    Utils.varDump(eventObj))
 
     return eventObj
 
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ mergeResultsGeneralResponse()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsGeneralResponse	(	self,
		mergedGeneralResponse,
		clientResponseItem
	)

Definition at line 684 of file ClientInterfaceService.py.

   def mergeResultsGeneralResponse(self, mergedGeneralResponse, clientResponseItem):
     currentGeneralResponse = clientResponseItem.itemObject
 
     if isinstance(currentGeneralResponse, dtm.EventObjects.GeneralResponse):
       if mergedGeneralResponse is None:
         # If this is first time call for results list init. with this Site object
         mergedGeneralResponse = currentGeneralResponse
         mergedGeneralResponse.host = clientResponseItem.host
         logger.debug("Merge init GeneralResponse object:\n" + Utils.varDump(currentGeneralResponse))
       else:
         logger.debug("Merge with GeneralResponse object:\n" + Utils.varDump(currentGeneralResponse))
         # Merge fields values
         mergedGeneralResponse.errorCode = str(mergedGeneralResponse.errorCode) + ";" + \
                                              str(currentGeneralResponse.errorCode)
         mergedGeneralResponse.errorMessage = str(mergedGeneralResponse.errorMessage) + ";" + \
                                              str(currentGeneralResponse.errorMessage)
         mergedGeneralResponse.statuses.extend(currentGeneralResponse.statuses)
     else:
       # Object is null or wrong type
       logger.error("Wrong type of currentGeneralResponse object: " + str(type(currentGeneralResponse)) + \
                    ", dtm.GeneralResponse expected\n" + Utils.varDump(currentGeneralResponse))
 
     return mergedGeneralResponse
 
 
 

Here is the caller graph for this function:

◆ mergeResultsSiteFields()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsSiteFields	(	self,
		siteToMerge,
		siteMergeWith
	)

Definition at line 556 of file ClientInterfaceService.py.

   def mergeResultsSiteFields(self, siteToMerge, siteMergeWith):
     # Set merged values for correspondent fields of the Site objects
     siteToMerge.resources += siteMergeWith.resources
     siteToMerge.contents += siteMergeWith.contents
     siteToMerge.collectedURLs += siteMergeWith.collectedURLs
     siteToMerge.newURLs += siteMergeWith.newURLs
     siteToMerge.deletedURLs += siteMergeWith.deletedURLs
     siteToMerge.iterations = max([siteToMerge.iterations, siteMergeWith.iterations])
     siteToMerge.errors += siteMergeWith.errors
     siteToMerge.errorMask |= siteMergeWith.errorMask
     siteToMerge.size += siteMergeWith.size
     siteToMerge.avgSpeed = min([siteToMerge.avgSpeed, siteMergeWith.avgSpeed])
     siteToMerge.avgSpeedCounter = min([siteToMerge.avgSpeedCounter, siteMergeWith.avgSpeedCounter])
 
     if self.configVars[self.CONFIG_DRCE_NODES] > 1:
       siteToMerge.maxURLs += siteMergeWith.maxURLs
       siteToMerge.maxResources += siteMergeWith.maxResources
       siteToMerge.maxErrors += siteMergeWith.maxErrors
 
     return siteToMerge
 
 
 

Here is the caller graph for this function:

◆ mergeResultsSiteFind()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsSiteFind	(	self,
		newItemsList,
		clientResponseItem
	)

Definition at line 584 of file ClientInterfaceService.py.

   def mergeResultsSiteFind(self, newItemsList, clientResponseItem):
     itemsList = clientResponseItem.itemObject
 
     # If this is first item in results list, init itemObject with empty list
     if newItemsList is None:
       newItemsList = []
 
     if isinstance(itemsList, list):
       # For each Site object in the clientResponseItem.itemObject
       for site in itemsList:
         # Check presence of this site in the newItemsList
         present = False
         for addedSite in newItemsList:
           if addedSite.id == site.id:
             # Merge site's data
             addedSite = self.mergeResultsSiteFields(addedSite, site)
             addedSite.host = clientResponseItem.host
             present = True
             break
         if not present:
           # Add content to list
           site.host = clientResponseItem.host
           newItemsList.append(site)
           logger.debug("Added Site:\n" + Utils.varDump(site))
     else:
       # Object is null or wrong type
       logger.error("Wrong type of clientResponseItem.itemObject\n" + Utils.varDump(itemsList))
 
     return newItemsList
 
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ mergeResultsSiteStatus()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsSiteStatus	(	self,
		mergedSite,
		clientResponseItem
	)

Definition at line 529 of file ClientInterfaceService.py.

   def mergeResultsSiteStatus(self, mergedSite, clientResponseItem):
     currentSite = clientResponseItem.itemObject
 
     if isinstance(currentSite, EventObjects.Site):
       # If this is first time call for results list init. with this Site object
       if mergedSite is None:
         mergedSite = currentSite
       else:
         logger.debug("Merge with Site object:\n" + Utils.varDump(currentSite))
         # Merge site's data
         mergedSite = self.mergeResultsSiteFields(mergedSite, currentSite)
 
       mergedSite.host = clientResponseItem.host
     else:
       # Object is null or wrong type
       logger.error("Wrong type of currentSite object: " + str(type(currentSite)) + ", Site expected\n" + \
                    Utils.varDump(currentSite))
 
     return mergedSite
 
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ mergeResultsURLContent()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsURLContent	(	self,
		newItemsList,
		clientResponseItem
	)

Definition at line 456 of file ClientInterfaceService.py.

   def mergeResultsURLContent(self, newItemsList, clientResponseItem):
     itemsList = clientResponseItem.itemObject
 
     # If this is first item in results list, init itemObject with empty list
     if newItemsList is None:
       newItemsList = []
 
     if isinstance(itemsList, list):
       # For each URLContentResponse object in the clientResponseItem.itemObject
       for urlContentResponse in itemsList:
         # If raw or processed content exists
         if len(urlContentResponse.rawContents) > 0 or len(urlContentResponse.processedContents) > 0:
           # Check is candidate valid
           for i, itemObject in enumerate(newItemsList):
             # Is item already exists in accumulated list by urlMd5 from the same site or by the rawContentMd5 or
             # the contentURLMd5 from another
             if itemObject.siteId == urlContentResponse.siteId and\
               (itemObject.urlMd5 == urlContentResponse.urlMd5 or\
                (urlContentResponse.rawContentMd5 != '' and urlContentResponse.rawContentMd5 != '""' and\
                 itemObject.rawContentMd5 == urlContentResponse.rawContentMd5) or\
                (urlContentResponse.contentURLMd5 != '' and urlContentResponse.contentURLMd5 != '""' and\
                 itemObject.contentURLMd5 == urlContentResponse.contentURLMd5)):
               # Is item that exists is not processed or migrated from another host and candidate is not
               # or status greater
               if 'Batch_Id' not in itemObject.dbFields:
                 itemObject.dbFields['Batch_Id'] = 0
               if 'Batch_Id' not in urlContentResponse.dbFields:
                 urlContentResponse.dbFields['Batch_Id'] = 0
               # Is both items have crawled and processed but some one is older by UDate (experimental).
               itemObjectUDate = 0
               urlContentResponseUDate = 0
               if 'UDate' in itemObject.dbFields:
                 itemObjectUDate = self.getUnixTimeFromString(itemObject.dbFields['UDate'])
               if 'UDate' in urlContentResponse.dbFields:
                 urlContentResponseUDate = self.getUnixTimeFromString(urlContentResponse.dbFields['UDate'])
               # logger.debug("urlContentResponseUDate: %s, itemObjectUDate: %s",
               #              str(urlContentResponseUDate), str(itemObjectUDate))
               if (int(itemObject.dbFields['Batch_Id']) == 0 and int(urlContentResponse.dbFields['Batch_Id']) != 0) or\
                (urlContentResponse.dbFields['Status'] > itemObject.dbFields['Status']) or\
                (urlContentResponseUDate > itemObjectUDate):
                 # Item already exists and candidate is better and replaces it
                 logger.debug("Already exists itemObject:\n" + Utils.varDump(itemObject.dbFields) + \
                              "\nand replaced urlContentResponse:\n" + Utils.varDump(urlContentResponse.dbFields))
                 urlContentResponse.host = clientResponseItem.host
                 # itemObject = urlContentResponse
                 newItemsList[i] = urlContentResponse
               else:
                 # Item already exists and better than candidate
                 logger.debug("Already exists urlContentResponse:\n" + Utils.varDump(urlContentResponse))
               urlContentResponse = None
               break
           # Add content to list
           if urlContentResponse is not None:
             urlContentResponse.host = clientResponseItem.host
             newItemsList.append(urlContentResponse)
             logger.debug("Added urlContentResponse:\n" + Utils.varDump(urlContentResponse))
           else:
             logger.debug("Rejected urlContentResponse")
         else:
           logger.debug("Empty contents lists in urlContentResponse:\n" + Utils.varDump(urlContentResponse))
     else:
       # Object is null or wrong type
       logger.error("Wrong type of clientResponseItem.itemObject\n" + Utils.varDump(itemsList))
 
     return newItemsList
 
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ mergeResultsURLFetch()

def dc.ClientInterfaceService.ClientInterfaceService.mergeResultsURLFetch	(	self,
		newItemsList,
		clientResponseItem
	)

Definition at line 621 of file ClientInterfaceService.py.

   def mergeResultsURLFetch(self, newItemsList, clientResponseItem):
     itemsList = clientResponseItem.itemObject
 
     logger.debug("Merging object of URLFEtch, host: %s, items: %s", str(clientResponseItem.host), str(len(itemsList)))
     # If this is first item in results list, init itemObject with empty list
     if newItemsList is None:
       newItemsList = []
 
     replacements = 0
     insertions = 0
 
     if isinstance(itemsList, list):
       # For each URL object in the clientResponseItem.itemObject
       for url in itemsList:
         # Check presence of this url in the newItemsList
         present = False
         for i, addedURL in enumerate(newItemsList):
           if addedURL.urlMd5 == url.urlMd5:
             logger.debug("URL found in list: %s", url.urlMd5)
             # For case if both are valid - to get newer (experimental)
             addedURLUDate = self.getUnixTimeFromString(addedURL.UDate)
             urUDate = self.getUnixTimeFromString(url.UDate)
             # logger.debug("addedURLUDate: %s, urUDate: %s", str(addedURLUDate), str(urUDate))
             # Replace with the URL object with higher status value and not migrated from another host
             if (addedURL.status <= url.status and (url.crawled > 0 and url.processed > 0 and url.batchId > 0)) or\
                (addedURL.status == url.status and addedURL.crawled == 0 and url.crawled > 0) or\
                (addedURL.status == url.status and addedURL.crawled > 0 and url.crawled > 0 and urUDate > addedURLUDate):
               logger.debug("URL replaced in list with best fields values: %s, old.status=%s, new.status=%s, " + \
                            "old.crawled=%s, new.crawled=%s, old.processed=%s, new.processed=%s, old.batchId=%s, " + \
                            "new.batchId=%s, old.UDate=%s, new.UDate=%s",
                            url.urlMd5, str(addedURL.status), str(url.status), str(addedURL.crawled), str(url.crawled),
                            str(addedURL.processed), str(url.processed), str(addedURL.batchId), str(url.batchId),
                            str(addedURLUDate), str(urUDate))
               url.host = clientResponseItem.host
               newItemsList[i] = url
               replacements += 1
             else:
               logger.debug("URL not replaced cause conditions not matched, old url:\n%s\ncandidate url:\n%s",
                            str(addedURL), str(url))
             present = True
             break
         if not present:
           # Add content to list
           url.host = clientResponseItem.host
           newItemsList.append(url)
           logger.debug("Added URL: %s", str(url.urlMd5))
           insertions += 1
     else:
       # Object is null or wrong type
       logger.error("Wrong type of clientResponseItem.itemObject\n%s", Utils.varDump(itemsList))
 
     logger.debug("Merging object of URLFEtch finished, replacements: %s, insertions: %s", str(replacements),
                  str(insertions))
 
     return newItemsList
 
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ onBatchTasksManagerRealTimeRoute()

def dc.ClientInterfaceService.ClientInterfaceService.onBatchTasksManagerRealTimeRoute	(	self,
		event
	)

Definition at line 160 of file ClientInterfaceService.py.

   def onBatchTasksManagerRealTimeRoute(self, event):
     try:
       logger.debug("Received event: " + Utils.varDump(event))
       self.send(str(self.batchTasksManagerRealTime) + self.CONNECTION_PREFIX, event)
       self.registerEvent(event)
     except KeyError as err:
       logger.error(err.message)
     except Exception, err:
       logger.error("Error `%s`", str(err))
 
 
 

Here is the call graph for this function:

◆ onDCClientRoute()

def dc.ClientInterfaceService.ClientInterfaceService.onDCClientRoute	(	self,
		event
	)

Definition at line 175 of file ClientInterfaceService.py.

   def onDCClientRoute(self, event):
     try:
       request_event = self.getRequestEvent(event)
       if event.cookie is not None and isinstance(event.cookie, dict) and\
          DC_CONSTANTS.MERGE_PARAM_NAME in event.cookie and bool(event.cookie[DC_CONSTANTS.MERGE_PARAM_NAME]) is False:
         logger.debug("No merge results specified in Event.cookie!")
       else:
         logger.debug("Results merge try by default.")
         event.eventObj = self.mergeResultsData(event.eventType, event.eventObj, event.cookie)
       self.reply(request_event, event)
       logger.debug("Results sent to DCC.")
       self.unregisterEvent(request_event)
     except KeyError as err:
       logger.error(err.message)
 
 
 

Here is the call graph for this function:

◆ onSitesManagerRoute()

def dc.ClientInterfaceService.ClientInterfaceService.onSitesManagerRoute	(	self,
		event
	)

Definition at line 145 of file ClientInterfaceService.py.

   def onSitesManagerRoute(self, event):
     try:
       logger.debug("Received event: " + Utils.varDump(event))
       self.send(str(self.sitesManager) + self.CONNECTION_PREFIX, event)
       self.registerEvent(event)
     except KeyError as err:
       logger.error(err.message)
     except Exception, err:
       logger.error("Error `%s`", str(err))
 
 
 

Here is the call graph for this function:

◆ registerEvent()

def dc.ClientInterfaceService.ClientInterfaceService.registerEvent	(	self,
		event
	)

Definition at line 195 of file ClientInterfaceService.py.

   def registerEvent(self, event):
     event.eventObj = None
     self.processEvents[event.uid] = event

Here is the caller graph for this function:

◆ sortURLContentResults()

def dc.ClientInterfaceService.ClientInterfaceService.sortURLContentResults	(	self,
		itemObject,
		criterion
	)

Definition at line 337 of file ClientInterfaceService.py.

   def sortURLContentResults(self, itemObject, criterion):
     ret = itemObject
 
     try:
       crits = criterion.split(',')
       crits = crits[0]
       crits = criterion.split(' ')
       if len(crits) == 0:
         crits = ('CDate', False)
       elif len(crits) == 1:
         crits = (crits[0], False)
       else:
         if crits[1] == 'ASC':
           crits[1] = False
         else:
           crits[1] = True
 
       if crits[0] == 'PDate':
         def orderKey(itemObject):
           if itemObject.dbFields['PDate'] is not None:
             return parse(itemObject.dbFields['PDate']).strftime('%s')
           else:
             return -1
       elif crits[0] == 'CDate':
         def orderKey(itemObject):
           return parse(itemObject.dbFields['CDate']).strftime('%s')
       elif crits[0] == 'UDate':
         def orderKey(itemObject):
           return parse(itemObject.dbFields['UDate']).strftime('%s')
       elif crits[0] == 'TcDate':
         def orderKey(itemObject):
           return parse(itemObject.dbFields['TcDate']).strftime('%s')
       elif crits[0] == 'Size':
         def orderKey(itemObject):
           return itemObject.dbFields['Size']
       elif crits[0] == 'Depth':
         def orderKey(itemObject):
           return itemObject.dbFields['Depth']
       elif crits[0] == 'TagsCount':
         def orderKey(itemObject):
           return itemObject.dbFields['TagsCount']
       elif crits[0] == 'ContentURLMd5':
         def orderKey(itemObject):
           return itemObject.dbFields['ContentURLMd5']
       else:
         def orderKey(itemObject):
           return itemObject.dbFields['CDate']
 
       ret = sorted(ret, key=orderKey, reverse=crits[1])
 
     except Exception, err:
       logger.error("Exception: '%s', criterion: '%s'", str(err), str(criterion))
 
     return ret
 
 

Here is the caller graph for this function:

◆ sortURLFetchResults()

def dc.ClientInterfaceService.ClientInterfaceService.sortURLFetchResults	(	self,
		itemObject,
		criterion
	)

Definition at line 398 of file ClientInterfaceService.py.

   def sortURLFetchResults(self, itemObject, criterion):
     ret = itemObject
 
     try:
       crits = criterion.split(',')
       crits = crits[0]
       crits = criterion.split(' ')
       if len(crits) == 0:
         crits = ('CDate', False)
       elif len(crits) == 1:
         crits = (crits[0], False)
       else:
         if crits[1] == 'ASC':
           crits[1] = False
         else:
           crits[1] = True
 
       if crits[0] == 'PDate':
         def orderKey(itemObject):
           return parse(itemObject.pDate).strftime('%s')
       elif crits[0] == 'CDate':
         def orderKey(itemObject):
           return parse(itemObject.CDate).strftime('%s')
       elif crits[0] == 'UDate':
         def orderKey(itemObject):
           return parse(itemObject.UDate).strftime('%s')
       elif crits[0] == 'TcDate':
         def orderKey(itemObject):
           return parse(itemObject.tcDate).strftime('%s')
       elif crits[0] == 'Size':
         def orderKey(itemObject):
           return itemObject.size
       elif crits[0] == 'Depth':
         def orderKey(itemObject):
           return itemObject.depth
       elif crits[0] == 'TagsCount':
         def orderKey(itemObject):
           return itemObject.tagsCount
       elif crits[0] == 'ContentURLMd5':
         def orderKey(itemObject):
           return itemObject.contentURLMd5
       else:
         def orderKey(itemObject):
           return itemObject.CDate
 
       ret = sorted(ret, key=orderKey, reverse=crits[1])
     except Exception as err:
       logger.error("Exception: %s", str(err))
 
     return ret
 
 
 

Here is the caller graph for this function:

◆ unregisterEvent()

def dc.ClientInterfaceService.ClientInterfaceService.unregisterEvent	(	self,
		event
	)

Definition at line 212 of file ClientInterfaceService.py.

   def unregisterEvent(self, event):
     del self.processEvents[event.uid]

Here is the caller graph for this function:

Member Data Documentation

◆ batchTasksManagerRealTime

dc.ClientInterfaceService.ClientInterfaceService.batchTasksManagerRealTime

Definition at line 55 of file ClientInterfaceService.py.

◆ CONFIG_BATCH_TASKS_MANAGER_REALTIME

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_BATCH_TASKS_MANAGER_REALTIME = "clientBatchTasksManagerRealTime"

static

Definition at line 37 of file ClientInterfaceService.py.

◆ CONFIG_DRCE_NODES

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_DRCE_NODES = "DRCENodes"

static

Definition at line 38 of file ClientInterfaceService.py.

◆ CONFIG_SECTION

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_SECTION = "ClientInterfaceService"

static

Definition at line 31 of file ClientInterfaceService.py.

◆ CONFIG_SERVER_HOST

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_SERVER_HOST = "serverHost"

static

Definition at line 32 of file ClientInterfaceService.py.

◆ CONFIG_SERVER_PORT

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_SERVER_PORT = "serverPort"

static

Definition at line 33 of file ClientInterfaceService.py.

◆ CONFIG_SITES_MANAGER

string dc.ClientInterfaceService.ClientInterfaceService.CONFIG_SITES_MANAGER = "clientSitesManager"

static

Definition at line 34 of file ClientInterfaceService.py.

◆ CONNECTION_PREFIX

string dc.ClientInterfaceService.ClientInterfaceService.CONNECTION_PREFIX = "Connection"

static

Definition at line 36 of file ClientInterfaceService.py.

◆ processEvents

dc.ClientInterfaceService.ClientInterfaceService.processEvents

Definition at line 138 of file ClientInterfaceService.py.

◆ SERVER_CONNECTION_NAME

string dc.ClientInterfaceService.ClientInterfaceService.SERVER_CONNECTION_NAME = "server"

static

Definition at line 35 of file ClientInterfaceService.py.

◆ sitesManager

dc.ClientInterfaceService.ClientInterfaceService.sitesManager

Definition at line 54 of file ClientInterfaceService.py.

The documentation for this class was generated from the following file:

sources/hce/dc/ClientInterfaceService.py

Public Member Functions

Public Attributes

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ getRequestEvent()

◆ getUnixTimeFromString()

◆ mergeResultsBatch()

◆ mergeResultsData()

◆ mergeResultsGeneralResponse()

◆ mergeResultsSiteFields()

◆ mergeResultsSiteFind()

◆ mergeResultsSiteStatus()

◆ mergeResultsURLContent()

◆ mergeResultsURLFetch()

◆ onBatchTasksManagerRealTimeRoute()

◆ onDCClientRoute()

◆ onSitesManagerRoute()

◆ registerEvent()

◆ sortURLContentResults()

◆ sortURLFetchResults()

◆ unregisterEvent()

Member Data Documentation

◆ batchTasksManagerRealTime

◆ CONFIG_BATCH_TASKS_MANAGER_REALTIME

◆ CONFIG_DRCE_NODES

◆ CONFIG_SECTION

◆ CONFIG_SERVER_HOST

◆ CONFIG_SERVER_PORT

◆ CONFIG_SITES_MANAGER

◆ CONNECTION_PREFIX

◆ processEvents

◆ SERVER_CONNECTION_NAME

◆ sitesManager

◆ init()