Inheritance diagram for dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB:

Collaboration diagram for dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB:

Classes
class	Meta

Public Member Functions
def	__init__ (self)

def	setup (self)

def	run (self)

def	process (self)

def	putContentToDB (self)

def	processBatch (self)

def	loadConfig (self)

def	loadLogConfigFile (self)

def	loadOptions (self)

def	getExitCode (self)

Public Attributes
	exit_code

	logger

	config_db_dir

	sqliteTimeout

	input_data

	raw_contents_tbl

	config

Detailed Description

Definition at line 42 of file ProcessorStoreContentKVDB.py.

Constructor & Destructor Documentation

◆ init()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.__init__ ( self )

Definition at line 54 of file ProcessorStoreContentKVDB.py.

   def __init__(self):
     # call base class __init__ method
     foundation.CementApp.__init__(self)
     self.exit_code = CONSTS.EXIT_SUCCESS
     self.logger = None
     self.config_db_dir = None
     self.sqliteTimeout = SQLITE_TIMEOUT
     self.input_data = None
     self.raw_contents_tbl = None
 
 

Member Function Documentation

◆ getExitCode()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.getExitCode ( self )

Definition at line 193 of file ProcessorStoreContentKVDB.py.

   def getExitCode(self):
     return self.exit_code

◆ loadConfig()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.loadConfig ( self )

Definition at line 153 of file ProcessorStoreContentKVDB.py.

   def loadConfig(self):
     try:
       self.config = ConfigParser.ConfigParser()
       self.config.optionxform = str
       if self.pargs.config:
         self.config.read(self.pargs.config)
     except Exception as err:
       print MSG_ERROR_LOAD_CONFIG + err.message
       raise
 
 

Here is the caller graph for this function:

◆ loadLogConfigFile()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.loadLogConfigFile ( self )

Definition at line 167 of file ProcessorStoreContentKVDB.py.

   def loadLogConfigFile(self):
     try:
       log_conf_file = self.config.get("Application", "log")
       logging.config.fileConfig(log_conf_file)
       self.logger = Utils.MPLogger().getLogger()
     except Exception as err:
       print MSG_ERROR_LOAD_LOG_CONFIG_FILE + err.message
       raise
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ loadOptions()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.loadOptions ( self )

Definition at line 180 of file ProcessorStoreContentKVDB.py.

   def loadOptions(self):
     try:
       self.config_db_dir = self.config.get(self.__class__.__name__, "config_db_dir")
       self.raw_contents_tbl = self.config.get("sqlite", "raw_contents_tbl")
       self.sqliteTimeout = self.config.getint("sqlite", "timeout")
     except Exception as err:
       print MSG_ERROR_LOAD_OPTIONS + err.message
       raise
 
 

Here is the caller graph for this function:

◆ process()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.process ( self )

Definition at line 98 of file ProcessorStoreContentKVDB.py.

   def process(self):
     self.putContentToDB()
     # return response.get()
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ processBatch()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.processBatch ( self )

Definition at line 134 of file ProcessorStoreContentKVDB.py.

   def processBatch(self):
     try:
       # read pickled batch object from stdin and unpickle it
       input_pickled_object = sys.stdin.read()
       stored_in_data = pickle.loads(input_pickled_object)
       self.input_data = stored_in_data
       # self.logger.info("input scraper object: " + str(vars(stored_in_data)))
       # TODO main processing over every url from list of urls in the batch object
       self.process()
       # self.logger.info("output : " + str(output))
       # send response to the stdout
       # print input_pickled_object
     except Exception as err:
       ExceptionLog.handler(self.logger, err, MSG_ERROR_PROCESS, (err))
       self.exit_code = CONSTS.EXIT_FAILURE
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ putContentToDB()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.putContentToDB ( self )

Definition at line 103 of file ProcessorStoreContentKVDB.py.

   def putContentToDB(self):
     # get appropriate db name, depending on siteId
     if len(self.input_data.siteId):
       db_name = self.config_db_dir + "/" + self.input_data.siteId + ".db"
     else:
       db_name = self.config_db_dir + "/0.db"
     self.logger.info("db_name: " + db_name)
     connector = None
     try:
       # put parsed resource to the db
       connector = sqlite3.connect(db_name, timeout=self.sqliteTimeout)  # @UndefinedVariable
       connector.text_factory = str
       with connector:
         cur = connector.cursor()
         query = "CREATE TABLE IF NOT EXISTS \
                 %s(id VARCHAR(32) PRIMARY KEY UNIQUE, data TEXT, CDate DATETIME DEFAULT CURRENT_TIMESTAMP)" \
                 % (self.raw_contents_tbl)
         cur.execute(query)
         cur.execute("INSERT OR REPLACE INTO raw_contents VALUES(?,?,datetime('now','localtime'))",
                     (self.input_data.urlId, self.input_data.raw_content))
 
     except Exception as err:
       # Connection objects can be used as context managers that automatically commit or rollback transactions.
       # In the event of an exception, the transaction is rolled back; otherwise, the transaction is committed:
       # connector.rollback()
       ExceptionLog.handler(self.logger, err, 'putContentToDB')
       raise
 
 

Here is the caller graph for this function:

◆ run()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.run ( self )

Definition at line 75 of file ProcessorStoreContentKVDB.py.

   def run(self):
     # call base class run method
     foundation.CementApp.run(self)
 
     # config section
     self.loadConfig()
 
     # load logger config file
     self.loadLogConfigFile()
 
     # load sqlite db backend
     # self.loadSqliteDBBackend()
 
     # sqlite
     # self.loadDBBackend()
 
     # options
     self.loadOptions()
 
 

Here is the call graph for this function:

◆ setup()

def dc_processor.ProcessorStoreContentKVDB.ProcessorStoreContentKVDB.setup ( self )

Definition at line 67 of file ProcessorStoreContentKVDB.py.

   def setup(self):
     # call base class setup method
     foundation.CementApp.setup(self)
     self.args.add_argument('-c', '--config', action='store', metavar='config_file', help='config ini-file')
 
 