Inheritance diagram for dc_postprocessor.SocialModule.SocialModule:

Collaboration diagram for dc_postprocessor.SocialModule.SocialModule:

Public Member Functions
def	__init__ (self, getConfigOption=None, log=None)

def	init (self)

def	executeCommand (self, cmd, inputStream='')

def	processBatch (self, batch)

def	executeSocialTask (self, inputBatch)

Public Member Functions inherited from dc_postprocessor.PostProcessingModuleClass.PostProcessingModuleClass
def	__init__ (self, getConfigOption=None, log=None)

def	init (self)

def	processBatch (self, batchObj)

def	processBatchItem (self, batchItemObj)

Public Attributes
	cmd

	dbWrapper

Public Attributes inherited from dc_postprocessor.PostProcessingModuleClass.PostProcessingModuleClass
	getConfigOption

	logger

Static Public Attributes
string	SOCIAL_RATE_PROPERTY_NAME = 'SOCIAL_RATE'

string	USER_PROXY_PROPERTY_NAME = 'USER_PROXY'

string	PARAM_USER_PROXY = 'user_proxy'

string	OPTION_EXECUTION_LOCAL = 'executionLocal'

string	OPTION_EXECUTION_REMOTE = 'executionRemote'

string	OPTION_EXECUTION_TYPE = 'executionType'

string	OPTION_DB_TASK_INI = 'db_task_ini'

int	EXECUTION_TYPE_LOCAL = 0

int	EXECUTION_TYPE_REMOTE = 1

int	EXECUTION_TYPE_DEFAULT = EXECUTION_TYPE_LOCAL

string	TMP_INPUT_FILE_NAME = 'in'

string	TMP_OUTPUT_FILE_NAME = 'out'

list	TMP_FILE_NAMES_LIST = [TMP_INPUT_FILE_NAME, TMP_OUTPUT_FILE_NAME]

string	MACRO_INPUT_FILE = '%INPUT_FILE%'

string	MACRO_OUTPUT_FILE = '%OUTPUT_FILE%'

string	MACRO_USER_NAME = '%USER_NAME%'

string	ERROR_MSG_INITIALIZATION_CALLBACK = "Error initialization of callback function for get config options."

string	ERROR_MSG_INITIALIZATION_LOGGER = "Error initialization of self.logger."

string	ERROR_MSG_EXECUTION_TYPE = "Wrong execution type ( %s ) was got from config file."

string	ERROR_MSG_EXECUTION_CMD_EMPTY = "Execution command line is empty."

string	ERROR_MSG_CREATION_DBTASK_WRAPPER = "Creation DBTaskWrapper failed. Error: %s"

string	ERROR_MSG_LOAD_USER_PROXY = "Load parameter '" + PARAM_USER_PROXY + "' from site property failed. Error: %s"

Private Member Functions
def	__getDBWrapper (self)

def	__getCmd (self)

def	__createTemporaryFiles (self)

def	__removeTemporaryFiles (self, tempFiles)

def	__makeInputFile (self, tempFiles, inputBatch)

def	__readOutputFile (self, tempFiles)

def	__makeCmdLine (self, tempFiles, templateCmdLine)

def	__fillUserProxyData (self, batchItem)

Detailed Description

Definition at line 37 of file SocialModule.py.

Constructor & Destructor Documentation

◆ init()

def dc_postprocessor.SocialModule.SocialModule.__init__	(	self,
		getConfigOption = `None`,
		log = `None`
	)

Definition at line 74 of file SocialModule.py.

   def __init__(self, getConfigOption=None, log=None):
     PostProcessingModuleClass.__init__(self, getConfigOption, log)
 
     self.cmd = None
     self.dbWrapper = None

Member Function Documentation

◆ __createTemporaryFiles()

def dc_postprocessor.SocialModule.SocialModule.__createTemporaryFiles ( self )

private

Definition at line 165 of file SocialModule.py.

   def __createTemporaryFiles(self):
     # variable for result
     files = {}
     for name in self.TMP_FILE_NAMES_LIST:
       files[name] = tempfile.NamedTemporaryFile(delete=False)
 
     return files
 
 

Here is the caller graph for this function:

◆ __fillUserProxyData()

def dc_postprocessor.SocialModule.SocialModule.__fillUserProxyData	(	self,
		batchItem
	)

private

Definition at line 248 of file SocialModule.py.

   def __fillUserProxyData(self, batchItem):
 
     if self.PARAM_USER_PROXY in batchItem.properties[self.SOCIAL_RATE_PROPERTY_NAME]:
       try:
         socialRateProperties = json.loads(batchItem.properties[self.SOCIAL_RATE_PROPERTY_NAME])
 
         if self.PARAM_USER_PROXY in socialRateProperties:
           self.logger.debug("!!! user_proxy: %s", str(socialRateProperties[self.PARAM_USER_PROXY]))
           userProxyJsonWrapper = UserProxyJsonWrapper(socialRateProperties[self.PARAM_USER_PROXY])
           self.logger.debug("!!! source: %s", str(userProxyJsonWrapper.getSource()))
           self.logger.debug("!!! proxies: %s", str(userProxyJsonWrapper.getProxies()))
 
           if userProxyJsonWrapper.getSource() == UserProxyJsonWrapper.SOURCE_DATABASE:
             self.logger.debug("Getting proxies list from DB.")
 
             self.logger.debug("!!! batchItem.siteId: %s", str(batchItem.siteId))
 
             proxyWrapper = DBProxyWrapper(self.dbWrapper)
             proxiesList = proxyWrapper.getEnaibledProxies(batchItem.siteId)
             self.logger.debug("!!! type: %s, proxiesList: %s", str(type(proxiesList)), str(proxiesList))
 
             userProxyJsonWrapper.addProxyList(proxiesList)
             userProxyJsonWrapper.setSource(UserProxyJsonWrapper.SOURCE_PROPERTY)
             self.logger.debug("!!! userProxyJsonWrapper.getProxies(): %s", str(userProxyJsonWrapper.getProxies()))
             self.logger.debug("!!! userProxyJsonWrapper.getSource(): %s", str(userProxyJsonWrapper.getSource()))
 
             batchItem.properties[self.SOCIAL_RATE_PROPERTY_NAME] = json.dumps(socialRateProperties)
 
       except Exception, err:
         self.logger.error(self.ERROR_MSG_LOAD_USER_PROXY, str(err))
 
       self.logger.debug("!!! batchItem.properties: %s", str(batchItem.properties))
 
 
     return batchItem
 
 

Here is the caller graph for this function:

◆ __getCmd()

def dc_postprocessor.SocialModule.SocialModule.__getCmd ( self )

private

Definition at line 117 of file SocialModule.py.

   def __getCmd(self):
     # variable for result
     ret = None
     executionType = int(self.getConfigOption(sectionName=self.__class__.__name__,
                                              optionName=self.OPTION_EXECUTION_TYPE,
                                              defaultValue=self.EXECUTION_TYPE_DEFAULT))
 
     if executionType == self.EXECUTION_TYPE_LOCAL:
       ret = self.getConfigOption(sectionName=self.__class__.__name__,
                                  optionName=self.OPTION_EXECUTION_LOCAL,
                                  defaultValue='')
 
     elif executionType == self.EXECUTION_TYPE_REMOTE:
       ret = self.getConfigOption(sectionName=self.__class__.__name__,
                                  optionName=self.OPTION_EXECUTION_REMOTE,
                                  defaultValue='')
 
     else:
       raise Exception(self.ERROR_MSG_EXECUTION_TYPE % str(executionType))
 
     if ret == "":
       raise Exception(self.ERROR_MSG_EXECUTION_CMD_EMPTY)
 
     return ret
 
 

Here is the caller graph for this function:

◆ __getDBWrapper()

def dc_postprocessor.SocialModule.SocialModule.__getDBWrapper ( self )

private

Definition at line 100 of file SocialModule.py.

   def __getDBWrapper(self):
     # variable for result
     ret = None
     try:
       configParser = ConfigParser.ConfigParser()
       configParser.read(self.getConfigOption(sectionName=self.__class__.__name__, optionName=self.OPTION_DB_TASK_INI))
       ret = DBTasksWrapper(configParser)
     except Exception, err:
       raise Exception(self.ERROR_MSG_CREATION_DBTASK_WRAPPER % str(err))
 
     return ret
 
 

Here is the caller graph for this function:

◆ __makeCmdLine()

def dc_postprocessor.SocialModule.SocialModule.__makeCmdLine	(	self,
		tempFiles,
		templateCmdLine
	)

private

Definition at line 216 of file SocialModule.py.

   def __makeCmdLine(self, tempFiles, templateCmdLine):
     # variable for result
     ret = templateCmdLine
 
     # set temporary file names
     if isinstance(tempFiles, dict):
 
       if self.MACRO_INPUT_FILE.upper() in ret and tempFiles[self.TMP_INPUT_FILE_NAME] is not None:
         ret = ret.replace(self.MACRO_INPUT_FILE.upper(), tempFiles[self.TMP_INPUT_FILE_NAME].name)
 
       if self.MACRO_INPUT_FILE.lower() in ret and tempFiles[self.TMP_INPUT_FILE_NAME] is not None:
         ret = ret.replace(self.MACRO_INPUT_FILE.lower(), tempFiles[self.TMP_INPUT_FILE_NAME].name)
 
       if self.MACRO_OUTPUT_FILE.upper() in ret and tempFiles[self.TMP_OUTPUT_FILE_NAME] is not None:
         ret = ret.replace(self.MACRO_OUTPUT_FILE.upper(), tempFiles[self.TMP_OUTPUT_FILE_NAME].name)
 
       if self.MACRO_OUTPUT_FILE.lower() in ret and tempFiles[self.TMP_OUTPUT_FILE_NAME] is not None:
         ret = ret.replace(self.MACRO_OUTPUT_FILE.lower(), tempFiles[self.TMP_OUTPUT_FILE_NAME].name)
 
       if self.MACRO_USER_NAME .upper() in ret:
         ret = ret.replace(self.MACRO_USER_NAME.upper(), getpass.getuser())
 
       if self.MACRO_USER_NAME .lower() in ret:
         ret = ret.replace(self.MACRO_USER_NAME.lower(), getpass.getuser())
 
     return ret
 
 

Here is the caller graph for this function:

◆ __makeInputFile()

def dc_postprocessor.SocialModule.SocialModule.__makeInputFile	(	self,
		tempFiles,
		inputBatch
	)

private

Definition at line 190 of file SocialModule.py.

   def __makeInputFile(self, tempFiles, inputBatch):
 
     if isinstance(tempFiles, dict) and self.TMP_INPUT_FILE_NAME in tempFiles:
       tempFiles[self.TMP_INPUT_FILE_NAME].write(pickle.dumps(inputBatch))
       tempFiles[self.TMP_INPUT_FILE_NAME].close()

Here is the caller graph for this function:

◆ __readOutputFile()

def dc_postprocessor.SocialModule.SocialModule.__readOutputFile	(	self,
		tempFiles
	)

private

Definition at line 201 of file SocialModule.py.

   def __readOutputFile(self, tempFiles):
     # variable for result
     ret = None
     if tempFiles[self.TMP_OUTPUT_FILE_NAME] is not None:
       ret = pickle.loads(tempFiles[self.TMP_OUTPUT_FILE_NAME].read())
       tempFiles[self.TMP_OUTPUT_FILE_NAME].close()
 
     return ret
 
 

Here is the caller graph for this function:

◆ __removeTemporaryFiles()

def dc_postprocessor.SocialModule.SocialModule.__removeTemporaryFiles	(	self,
		tempFiles
	)

private

Definition at line 178 of file SocialModule.py.

   def __removeTemporaryFiles(self, tempFiles):
     if isinstance(tempFiles, dict):
       for f in tempFiles.values():
         if f is not None and os.path.isfile(f.name):
           os.unlink(f.name)
 
 

Here is the caller graph for this function:

◆ executeCommand()

def dc_postprocessor.SocialModule.SocialModule.executeCommand	(	self,
		cmd,
		inputStream = `''`
	)

Definition at line 148 of file SocialModule.py.

   def executeCommand(self, cmd, inputStream=''):
     self.logger.debug("Popen: %s", str(cmd))
     process = Popen(cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True, close_fds=True, executable='/bin/bash')
     self.logger.debug("process.communicate(), len(inputStream)=" + str(len(inputStream)))
     (output, err) = process.communicate(input=inputStream)
     self.logger.debug("Process std_error=: %s", str(err))
     self.logger.debug("Process output len=:" + str(len(output)))
     exitCode = process.wait()
     self.logger.debug("Process response exitCode: %s", str(exitCode))
 
     return output, exitCode
 
 

Here is the caller graph for this function:

◆ executeSocialTask()

def dc_postprocessor.SocialModule.SocialModule.executeSocialTask	(	self,
		inputBatch
	)

Definition at line 329 of file SocialModule.py.

   def executeSocialTask(self, inputBatch):
     # variable for result
     ret = inputBatch
     tempFiles = self.__createTemporaryFiles()
     self.logger.debug("!!! tempFiles: %s", str(tempFiles))
     try:
       if self.cmd is None or self.cmd == "":
         raise Exception(self.ERROR_MSG_EXECUTION_CMD_EMPTY)
 
       self.__makeInputFile(tempFiles, inputBatch)
 
       self.logger.debug("!!! template cmd: %s", str(self.cmd))
       cmd = self.__makeCmdLine(tempFiles, self.cmd)
       self.logger.debug("!!! execute cmd: %s", str(cmd))
 
       output, exitCode = self.executeCommand(cmd)
       self.logger.debug("!!! output: %s", str(output))
 
       if int(exitCode) == 0:
         ret = self.__readOutputFile(tempFiles)
 
     except Exception, err:
       self.logger.error(str(err))
     finally:
       self.__removeTemporaryFiles(tempFiles)
 
     return ret

Here is the call graph for this function:








Here is the caller graph for this function:

◆ init()

def dc_postprocessor.SocialModule.SocialModule.init ( self )

Definition at line 85 of file SocialModule.py.

   def init(self):
     if self.getConfigOption is None:
       raise Exception(self.ERROR_MSG_INITIALIZATION_CALLBACK)
 
     if self.logger is None:
       raise Exception(self.ERROR_MSG_INITIALIZATION_LOGGER)
 
     self.cmd = self.__getCmd()
     self.dbWrapper = self.__getDBWrapper()
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ processBatch()

def dc_postprocessor.SocialModule.SocialModule.processBatch	(	self,
		batch
	)

Definition at line 289 of file SocialModule.py.

   def processBatch(self, batch):
 
     if isinstance(batch, Batch):
       localBatchItems = []
       # accumulate batch items for send to social task processing
       for i in xrange(len(batch.items)):
         if self.SOCIAL_RATE_PROPERTY_NAME in batch.items[i].properties:
           localBatchItems.append(self.__fillUserProxyData(batch.items[i]))
 
       if len(localBatchItems) > 0:
         localBatch = Batch(batchId=batch.id)
         localBatch.items = localBatchItems
         self.logger.debug("Accumulated %s items from %s total for send to SocialTask",
                           str(len(localBatchItems)), str(len(batch.items)))
 
         localBatch = self.executeSocialTask(localBatch)
         self.logger.debug("Recived %s items from SocialTask", str(len(localBatch.items)))
 
         foundCount = 0
         # update batch items after processing of the social task
         for i in xrange(len(batch.items)):
           for batchItem in localBatch.items:
             if batch.items[i].urlId == batchItem.urlId and batch.items[i].siteId == batchItem.siteId:
               batch.items[i] = batchItem
               self.logger.debug("Found result for %s", str(batch.items[i].urlId))
               foundCount += 1
               break
 
         self.logger.debug("Found results for %s items", str(foundCount))
 
     else:
       self.logger.error("Input object has type: %s", str(type(batch)))
 
     return batch
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ cmd

dc_postprocessor.SocialModule.SocialModule.cmd

Definition at line 77 of file SocialModule.py.

◆ dbWrapper

dc_postprocessor.SocialModule.SocialModule.dbWrapper

Definition at line 78 of file SocialModule.py.

◆ ERROR_MSG_CREATION_DBTASK_WRAPPER

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_CREATION_DBTASK_WRAPPER = "Creation DBTaskWrapper failed. Error: %s"

static

Definition at line 69 of file SocialModule.py.

◆ ERROR_MSG_EXECUTION_CMD_EMPTY

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_EXECUTION_CMD_EMPTY = "Execution command line is empty."

static

Definition at line 68 of file SocialModule.py.

◆ ERROR_MSG_EXECUTION_TYPE

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_EXECUTION_TYPE = "Wrong execution type ( %s ) was got from config file."

static

Definition at line 67 of file SocialModule.py.

◆ ERROR_MSG_INITIALIZATION_CALLBACK

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_INITIALIZATION_CALLBACK = "Error initialization of callback function for get config options."

static

Definition at line 65 of file SocialModule.py.

◆ ERROR_MSG_INITIALIZATION_LOGGER

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_INITIALIZATION_LOGGER = "Error initialization of self.logger."

static

Definition at line 66 of file SocialModule.py.

◆ ERROR_MSG_LOAD_USER_PROXY

string dc_postprocessor.SocialModule.SocialModule.ERROR_MSG_LOAD_USER_PROXY = "Load parameter '" + PARAM_USER_PROXY + "' from site property failed. Error: %s"

static

Definition at line 70 of file SocialModule.py.

◆ EXECUTION_TYPE_DEFAULT

int dc_postprocessor.SocialModule.SocialModule.EXECUTION_TYPE_DEFAULT = EXECUTION_TYPE_LOCAL

static

Definition at line 52 of file SocialModule.py.

◆ EXECUTION_TYPE_LOCAL

int dc_postprocessor.SocialModule.SocialModule.EXECUTION_TYPE_LOCAL = 0

static

Definition at line 50 of file SocialModule.py.

◆ EXECUTION_TYPE_REMOTE

int dc_postprocessor.SocialModule.SocialModule.EXECUTION_TYPE_REMOTE = 1

static

Definition at line 51 of file SocialModule.py.

◆ MACRO_INPUT_FILE

string dc_postprocessor.SocialModule.SocialModule.MACRO_INPUT_FILE = '%INPUT_FILE%'

static

Definition at line 60 of file SocialModule.py.

◆ MACRO_OUTPUT_FILE

string dc_postprocessor.SocialModule.SocialModule.MACRO_OUTPUT_FILE = '%OUTPUT_FILE%'

static

Definition at line 61 of file SocialModule.py.

◆ MACRO_USER_NAME

string dc_postprocessor.SocialModule.SocialModule.MACRO_USER_NAME = '%USER_NAME%'

static

Definition at line 62 of file SocialModule.py.

◆ OPTION_DB_TASK_INI

string dc_postprocessor.SocialModule.SocialModule.OPTION_DB_TASK_INI = 'db_task_ini'

static

Definition at line 48 of file SocialModule.py.

◆ OPTION_EXECUTION_LOCAL

string dc_postprocessor.SocialModule.SocialModule.OPTION_EXECUTION_LOCAL = 'executionLocal'

static

Definition at line 45 of file SocialModule.py.

◆ OPTION_EXECUTION_REMOTE

string dc_postprocessor.SocialModule.SocialModule.OPTION_EXECUTION_REMOTE = 'executionRemote'

static

Definition at line 46 of file SocialModule.py.

◆ OPTION_EXECUTION_TYPE

string dc_postprocessor.SocialModule.SocialModule.OPTION_EXECUTION_TYPE = 'executionType'

static

Definition at line 47 of file SocialModule.py.

◆ PARAM_USER_PROXY

string dc_postprocessor.SocialModule.SocialModule.PARAM_USER_PROXY = 'user_proxy'

static

Definition at line 42 of file SocialModule.py.

◆ SOCIAL_RATE_PROPERTY_NAME

string dc_postprocessor.SocialModule.SocialModule.SOCIAL_RATE_PROPERTY_NAME = 'SOCIAL_RATE'

static

Definition at line 40 of file SocialModule.py.

◆ TMP_FILE_NAMES_LIST

list dc_postprocessor.SocialModule.SocialModule.TMP_FILE_NAMES_LIST = [TMP_INPUT_FILE_NAME, TMP_OUTPUT_FILE_NAME]

static

Definition at line 57 of file SocialModule.py.

◆ TMP_INPUT_FILE_NAME

string dc_postprocessor.SocialModule.SocialModule.TMP_INPUT_FILE_NAME = 'in'

static

Definition at line 55 of file SocialModule.py.

◆ TMP_OUTPUT_FILE_NAME

string dc_postprocessor.SocialModule.SocialModule.TMP_OUTPUT_FILE_NAME = 'out'

static

Definition at line 56 of file SocialModule.py.

◆ USER_PROXY_PROPERTY_NAME

string dc_postprocessor.SocialModule.SocialModule.USER_PROXY_PROPERTY_NAME = 'USER_PROXY'

static

Definition at line 41 of file SocialModule.py.

The documentation for this class was generated from the following file:

sources/hce/dc_postprocessor/SocialModule.py

Public Member Functions

Public Attributes

Static Public Attributes

Private Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ __createTemporaryFiles()

◆ __fillUserProxyData()

◆ __getCmd()

◆ __getDBWrapper()

◆ __makeCmdLine()

◆ __makeInputFile()

◆ __readOutputFile()

◆ __removeTemporaryFiles()

◆ executeCommand()

◆ executeSocialTask()

◆ init()

◆ processBatch()

Member Data Documentation

◆ cmd

◆ dbWrapper

◆ ERROR_MSG_CREATION_DBTASK_WRAPPER

◆ ERROR_MSG_EXECUTION_CMD_EMPTY

◆ ERROR_MSG_EXECUTION_TYPE

◆ ERROR_MSG_INITIALIZATION_CALLBACK

◆ ERROR_MSG_INITIALIZATION_LOGGER

◆ ERROR_MSG_LOAD_USER_PROXY

◆ EXECUTION_TYPE_DEFAULT

◆ EXECUTION_TYPE_LOCAL

◆ EXECUTION_TYPE_REMOTE

◆ MACRO_INPUT_FILE

◆ MACRO_OUTPUT_FILE

◆ MACRO_USER_NAME

◆ OPTION_DB_TASK_INI

◆ OPTION_EXECUTION_LOCAL

◆ OPTION_EXECUTION_REMOTE

◆ OPTION_EXECUTION_TYPE

◆ PARAM_USER_PROXY

◆ SOCIAL_RATE_PROPERTY_NAME

◆ TMP_FILE_NAMES_LIST

◆ TMP_INPUT_FILE_NAME

◆ TMP_OUTPUT_FILE_NAME

◆ USER_PROXY_PROPERTY_NAME

◆ init()