Inheritance diagram for app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor:

Collaboration diagram for app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor:

Classes
class	Meta

Public Member Functions
def	__init__ (self)

def	setup (self)

def	run (self)

def	getURLFetchJson (self)

def	createOutputPickle (self, inputUrlFetchJson)

def	sendToDbTask (self, outputPickle)

def	sendPickle (self, output_pickle)

def	process (self)

Public Attributes
	logger

	exitCode

	cmd

Static Public Attributes
string	CMD_DEFAULT = "cd ~/hce-node-bundle/api/python/bin && ./db-task.py --c=../ini/db-task.ini"

string	MSG_ERROR_EMPTY_CONFIG_FILE_NAME = "Config file name is empty."

string	MSG_ERROR_WRONG_CONFIG_FILE_NAME = "Config file name is wrong"

string	MSG_ERROR_LOAD_APP_CONFIG = "Error loading application config file."

string	MSG_ERROR_READ_LOG_CONFIG = "Error read log config file."

string	MSG_ERROR_EXIT_STATUS = "Exit failure. "

string	MSG_DEBUG_INPUT_PICKLE = "Input pickle: "

string	MSG_DEBUG_OUTPUT_PICKLE = "Output pickle: "

string	MSG_DEBUG_SEND_PICKLE = "Send pickle. Done."

string	MSG_INFO_PROCESSOR_EXIT_CODE = "Scraper exit_code: "

string	MSG_INFO_PROCESSOR_OUTPUT = "Scraper output: "

string	MSG_INFO_PROCESSOR_ERROR = "Scraper err: "

string	URLS_FETCH_JSON_TO_DBTASK_OPTION_LOG = "log"

string	URLS_FETCH_JSON_TO_DBTASK_OPTION_CMD = "cmd"

Private Member Functions
def	__initApp (self)

def	__loadAppConfig (self, configName)

def	__loadLogConfig (self, configName)

Detailed Description

Definition at line 37 of file UrlFetchJsonToDBTaskConvertor.py.

Constructor & Destructor Documentation

◆ init()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.__init__ ( self )

Definition at line 70 of file UrlFetchJsonToDBTaskConvertor.py.

   def __init__(self):
     # call base class __init__ method
     foundation.CementApp.__init__(self)
 
     self.logger = None
     self.exitCode = APP_CONSTS.EXIT_SUCCESS
     self.cmd = self.CMD_DEFAULT
 
 

Member Function Documentation

◆ __initApp()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.__initApp ( self )

private

Definition at line 104 of file UrlFetchJsonToDBTaskConvertor.py.

   def __initApp(self):
     if self.pargs.config:
       self.__loadLogConfig(self.__loadAppConfig(self.pargs.config))
     else:
       raise Exception(self.MSG_ERROR_LOAD_APP_CONFIG)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ __loadAppConfig()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.__loadAppConfig	(	self,
		configName
	)

private

Definition at line 115 of file UrlFetchJsonToDBTaskConvertor.py.

   def __loadAppConfig(self, configName):
     # variable for result
     confLogFileName = ""
 
     try:
       config = ConfigParser.ConfigParser()
       config.optionxform = str
 
       readOk = config.read(configName)
 
       if len(readOk) == 0:
         raise Exception(self.MSG_ERROR_WRONG_CONFIG_FILE_NAME + ": " + configName)
 
       if config.has_section(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME):
         confLogFileName = str(config.get(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME,
                                          self.URLS_FETCH_JSON_TO_DBTASK_OPTION_LOG))
         self.cmd = str(config.get(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME,
                                   self.URLS_FETCH_JSON_TO_DBTASK_OPTION_CMD))
 
     except Exception, err:
       raise Exception(self.MSG_ERROR_LOAD_APP_CONFIG + ' ' + str(err))
 
     return confLogFileName
 
 

Here is the caller graph for this function:

◆ __loadLogConfig()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.__loadLogConfig	(	self,
		configName
	)

private

Definition at line 144 of file UrlFetchJsonToDBTaskConvertor.py.

   def __loadLogConfig(self, configName):
     try:
       if isinstance(configName, str) and len(configName) == 0:
         raise Exception(self.MSG_ERROR_EMPTY_CONFIG_FILE_NAME)
 
       logging.config.fileConfig(configName)
 
       # call rotation log files and initialization logger
       self.logger = Utils.MPLogger().getLogger()
 
     except Exception, err:
       raise Exception(self.MSG_ERROR_READ_LOG_CONFIG + ' ' + str(err))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ createOutputPickle()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.createOutputPickle	(	self,
		inputUrlFetchJson
	)

Definition at line 165 of file UrlFetchJsonToDBTaskConvertor.py.

   def createOutputPickle(self, inputUrlFetchJson):
     eventType = DC_CONSTS.EVENT_TYPES.URL_FETCH
     convertor = DCCObjectsSerializator()
     eventObj = convertor.URLFetchDeserialize(inputUrlFetchJson)
     self.logger.debug(self.MSG_DEBUG_OUTPUT_PICKLE + '\n' + str(eventObj))
     drceSyncTasksCoverObj = DC_CONSTS.DRCESyncTasksCover(eventType, eventObj)
     outputPickle = pickle.dumps(drceSyncTasksCoverObj)
     self.logger.debug(self.MSG_DEBUG_OUTPUT_PICKLE + '\n' + str(outputPickle))
 
     return outputPickle
 
 

Here is the caller graph for this function:

◆ getURLFetchJson()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.getURLFetchJson ( self )

Definition at line 158 of file UrlFetchJsonToDBTaskConvertor.py.

   def getURLFetchJson(self):
     inputUrlFetchJson = sys.stdin.read()
     self.logger.debug(self.MSG_DEBUG_INPUT_PICKLE + '\n' + varDump(inputUrlFetchJson))
 
     return json.loads(inputUrlFetchJson)

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.process ( self )

Definition at line 193 of file UrlFetchJsonToDBTaskConvertor.py.

193 def process(self):

194

195 self.logger.info('self.cmd: ' + str(self.cmd))

196 try:

197 # chain of necessary calls for processing

198 self.sendPickle(self.sendToDbTask(self.createOutputPickle(self.getURLFetchJson())))

199 except Exception, err:

200 self.logger.error(self.MSG_ERROR_EXIT_STATUS + str(err))

201 self.exitCode = APP_CONSTS.EXIT_FAILURE

202

ftests.ftest_popen.process

process

Definition: ftest_popen.py:37

error

-mask-info

Here is the call graph for this function:

Here is the caller graph for this function:

◆ run()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.run ( self )

Definition at line 86 of file UrlFetchJsonToDBTaskConvertor.py.

   def run(self):
     # call base class run method
     foundation.CementApp.run(self)
 
     # call initialization application
     self.__initApp()
 
     # call internal processing
     self.process()
 
     # Finish logging
     self.logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)
 
 

Here is the call graph for this function:

◆ sendPickle()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.sendPickle	(	self,
		output_pickle
	)

Definition at line 188 of file UrlFetchJsonToDBTaskConvertor.py.

   def sendPickle(self, output_pickle):
     sys.stdout.write(output_pickle)
     self.logger.debug(self.MSG_DEBUG_SEND_PICKLE)
 
 

Here is the caller graph for this function:

◆ sendToDbTask()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.sendToDbTask	(	self,
		outputPickle
	)

Definition at line 177 of file UrlFetchJsonToDBTaskConvertor.py.

   def sendToDbTask(self, outputPickle):
     process = Popen(self.cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
     (output, err) = process.communicate(input=outputPickle)
     self.exitCode = process.wait()
     self.logger.info(self.MSG_INFO_PROCESSOR_EXIT_CODE + str(self.exitCode) + '\n' + \
                 self.MSG_INFO_PROCESSOR_OUTPUT + str(output) + '\n' + \
                 self.MSG_INFO_PROCESSOR_ERROR + str(err) + '\n' + str(pickle.loads(output)))
 
     return output
 
 

Here is the caller graph for this function:

◆ setup()

def app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.setup ( self )

Definition at line 80 of file UrlFetchJsonToDBTaskConvertor.py.

   def setup(self):
     # call base class setup method
     foundation.CementApp.setup(self)
 
 

Member Data Documentation

◆ cmd

app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.cmd

Definition at line 76 of file UrlFetchJsonToDBTaskConvertor.py.

◆ CMD_DEFAULT

string app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor.CMD_DEFAULT = "cd ~/hce-node-bundle/api/python/bin && ./db-task.py --c=../ini/db-task.ini"

static

Definition at line 40 of file UrlFetchJsonToDBTaskConvertor.py.