dc_processor.ProcessorFeedParser.ProcessorFeedParser Class Reference

Inheritance diagram for dc_processor.ProcessorFeedParser.ProcessorFeedParser:

Collaboration diagram for dc_processor.ProcessorFeedParser.ProcessorFeedParser:

Classes
class	Meta

Public Member Functions
def	__init__ (self, usageModel=APP_CONSTS.APP_USAGE_MODEL_PROCESS, configFile=None, logger=None, inputData=None)
def	setup (self)
def	run (self)
def	createArticle (self)
def	parseFeed (self)
def	process (self)
def	getQueryPrefix (self)
	For. More...
def	getDataBuffer (self, data)
	getDataBuffer prepare data buffer More...
def	putArticleToDB (self, result)
def	processBatch (self)
def	storeXPath (self)
	storeXpath More...
def	loadConfig (self)
def	loadLogConfigFile (self)
def	loadOptions (self)
def	getExitCode (self)

Public Attributes
	exit_code
	logger
	config_db_dir
	sqliteTimeout
	entry
	tagsCount
	tagsMask
	pubdate
	processedContent
	store_xpath
	db_engine
	article
	input_data
	articles_tbl
	PRAGMA_synchronous
	PRAGMA_journal_mode
	PRAGMA_temp_store
	usageModel
	configFile
	output_data
	config

Detailed Description

Definition at line 31 of file ProcessorFeedParser.py.

Constructor & Destructor Documentation

__init__()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.__init__	(		self,
			usageModel = APP_CONSTS.APP_USAGE_MODEL_PROCESS,
			configFile = None,
			logger = None,
			inputData = None
	)

Definition at line 43 of file ProcessorFeedParser.py.

  def __init__(self, usageModel=APP_CONSTS.APP_USAGE_MODEL_PROCESS, configFile=None, logger=None, inputData=None):
    if usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
      # call base class __init__ method
      foundation.CementApp.__init__(self)

    self.exit_code = CONSTS.EXIT_SUCCESS
    self.logger = logger
    self.config_db_dir = None
    self.sqliteTimeout = CONSTS.SQLITE_TIMEOUT
    self.entry = None
    self.tagsCount = None
    self.tagsMask = None
    self.pubdate = None
    self.processedContent = None
    self.store_xpath = False
    self.db_engine = CONSTS.MYSQL_ENGINE
    self.article = None
    self.input_data = inputData
    self.articles_tbl = None
    self.PRAGMA_synchronous = None
    self.PRAGMA_journal_mode = None
    self.PRAGMA_temp_store = None
    self.usageModel = usageModel
    self.configFile = configFile
    self.output_data = None

Member Function Documentation

createArticle()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.createArticle

(

self

)

Definition at line 113 of file ProcessorFeedParser.py.

  def createArticle(self):
    #self.logger.debug("self.entry: %s" % varDump(self.entry))

    article = Result(self.config, self.entry["urlMd5"])
    extractor = self.__class__.__name__

    for tag in self.entry["entry"]:
      tagValue = self.entry["entry"][tag]
      if isinstance(tagValue, (str, unicode)):
        if tag == 'link':
          tagValue = Utils.UrlNormalizator.entitiesEncode(tagValue)
        else:
          tagValue = tagValue
      elif isinstance(tagValue, int) or isinstance(tagValue, bool) or isinstance(tagValue, float):
        tagValue = str(tagValue)
      elif isinstance(tagValue, dict):
        names = {"url", "name", "value"}
        tagValueNew = None
        for name in names:
          if name in tagValue and isinstance(tagValue[name], (str, unicode)):
            if name == 'url':
              tagValueNew = Utils.UrlNormalizator.entitiesEncode(tagValue[name])
            else:
              tagValueNew = tagValue[name]
            break
        if tagValueNew is None:
          tagValue = ""
        else:
          tagValue = tagValueNew
      elif isinstance(tagValue, list) and len(tagValue) > 0:
        tagValueNew = None
        if isinstance(tagValue[0], dict):
          names = {"href":",", "url":",", "name":",", "term":",", "value":" "}  # pylint: disable=R0204
          for name in names:
            if name in tagValue[0]:
              tv = []
              for item in tagValue:
                if name in item and isinstance(item[name], (str, unicode)):
                  if name == 'href':
                    tv.append(Utils.UrlNormalizator.entitiesEncode(item[name].strip()))
                  else:
                    tv.append(item[name].strip())
              tagValueNew = names[name].join(tv)
              break
        if tagValueNew is None:
          tagValue = ""
        else:
          tagValue = tagValueNew
      else:
        self.logger.debug("Unsupported tag '%s' value type: %s", str(tag), varDump(tagValue))
        tagValue = ""

      article.tags[tag] = {"data":[tagValue], "name":tag, "xpath":"", "extractor":extractor}

    #parent rss feed tag
    if "parent_rss_feed" in self.entry:
      parent_rss_feed = json.dumps(self.entry["parent_rss_feed"])
      if parent_rss_feed[0] == '"':
        parent_rss_feed = parent_rss_feed[1:]
      if parent_rss_feed[-1] == '"':
        parent_rss_feed = parent_rss_feed[:-1]
      article.tags["parent_rss_feed"] = {"data":[parent_rss_feed],
                                         "name":"parent_rss_feed", "xpath":"", "extractor":extractor}

    #parent rss feed urlMd5 tag
    if "parent_rss_feed_urlMd5" in self.entry:
      article.tags["parent_rss_feed_urlMd5"] = {"data":[self.entry["parent_rss_feed_urlMd5"]],
                                                "name":"parent_rss_feed_urlMd5", "xpath":"", "extractor":extractor}

    #tags count
    article.tagsCount = len(article.tags.keys())

    # Finish time
    article.finish = time.time()

    # Set final article
    self.article = article
    #self.logger.debug("article: %s" % varDump(article))

Here is the call graph for this function:

Here is the caller graph for this function:

getDataBuffer()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.getDataBuffer	(		self,
			data
	)

getDataBuffer prepare data buffer

Definition at line 253 of file ProcessorFeedParser.py.

  def getDataBuffer(self, data):
    buf = {}
    for key in data.keys():
      data[key].get()
      buf[key] = data[key].data
    ret = encode(json.dumps(buf))
    self.logger.info("Result buffer: %s" % ret)
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

getExitCode()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.getExitCode

(

self

)

Definition at line 417 of file ProcessorFeedParser.py.

  def getExitCode(self):
    return self.exit_code

getQueryPrefix()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.getQueryPrefix

(

self

)

For.

Definition at line 240 of file ProcessorFeedParser.py.

  def getQueryPrefix(self):
    query_prefix = None
    if self.db_engine == CONSTS.MYSQL_ENGINE:
      query_prefix = "REPLACE INTO `contents_" + str(self.input_data.siteId if len(self.input_data.siteId) else 0)
    else:
      self.logger.info("DB Backend %s not supported" % str(self.db_engine))
    self.logger.info("db_name: " + query_prefix)
    return query_prefix

Here is the caller graph for this function:

loadConfig()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.loadConfig

(

self

)

Definition at line 362 of file ProcessorFeedParser.py.

  def loadConfig(self):
    try:
      self.config = ConfigParser.ConfigParser()
      self.config.optionxform = str
      if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
        if self.pargs.config:
          self.config.read(self.pargs.config)
      else:
        self.config.read(self.configFile)
    except Exception as err:
      raise Exception(CONSTS.MSG_ERROR_LOAD_CONFIG + " : " + str(err))

Here is the caller graph for this function:

loadLogConfigFile()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.loadLogConfigFile

(

self

)

Definition at line 379 of file ProcessorFeedParser.py.

  def loadLogConfigFile(self):
    try:
      if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
        log_conf_file = self.config.get("Application", "log")
        logging.config.fileConfig(log_conf_file)
        self.logger = Utils.MPLogger().getLogger()
    except Exception as err:
      raise Exception(CONSTS.MSG_ERROR_LOAD_CONFIG + " : " + str(err))

Here is the call graph for this function:

Here is the caller graph for this function:

loadOptions()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.loadOptions

(

self

)

Definition at line 393 of file ProcessorFeedParser.py.

  def loadOptions(self):
    class_name = self.__class__.__name__
    try:
      self.config_db_dir = self.config.get(class_name, "config_db_dir")
      self.articles_tbl = self.config.get("sqlite", "articles_tbl")
      self.sqliteTimeout = self.config.getint("sqlite", "timeout")

      # support sqlite optimizer options
      if self.config.has_option("sqlite", "PRAGMA_synchronous"):
        self.PRAGMA_synchronous = self.config.get("sqlite", "PRAGMA_synchronous")
      if self.config.has_option("sqlite", "PRAGMA_journal_mode"):
        self.PRAGMA_journal_mode = self.config.get("sqlite", "PRAGMA_journal_mode")
      if self.config.has_option("sqlite", "PRAGMA_temp_store"):
        self.PRAGMA_temp_store = self.config.get("sqlite", "PRAGMA_temp_store")

      # support db backend
      if self.config.has_option(class_name, "db_engine"):
        self.db_engine = self.config.get(class_name, "db_engine")

    except Exception as err:
      print CONSTS.MSG_ERROR_LOAD_OPTIONS + err.message
      raise

Here is the caller graph for this function:

parseFeed()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.parseFeed

(

self

)

Definition at line 197 of file ProcessorFeedParser.py.

  def parseFeed(self):
    ret = True
    try:
      self.entry = json.loads(self.input_data.raw_content)
      self.createArticle()
      #self.putArticleToDB({"default":self.article})
    except ValueError, err:
      ExceptionLog.handler(self.logger, err, 'Bad raw content:', (self.input_data.raw_content), \
                           {ExceptionLog.LEVEL_NAME_ERROR:ExceptionLog.LEVEL_VALUE_DEBUG})
      ret = False
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

process()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.process

(

self

)

Definition at line 213 of file ProcessorFeedParser.py.

  def process(self):
    self.logger.debug("URL: %s" % str(self.input_data.url))
    self.logger.debug("URLMd5: %s" % str(self.input_data.urlId))
    self.logger.debug("SiteId: %s" % str(self.input_data.siteId))
    if self.parseFeed():
      self.tagsCount = self.article.tagsCount
      self.tagsMask = self.article.tagsMask
      # TODO: strange level "default", possible need to be removed and new common object need to be used by all scrapers
      self.article.get()
      self.processedContent = {"default":self.article}
      self.processedContent["internal"] = [self.article]
      self.processedContent["custom"] = []
      # correct pubdate
      if CONSTS.PUBLISHED in self.article.tags:
        from dateutil.parser import parse
        self.pubdate = parse(self.article.tags[CONSTS.PUBLISHED]["data"][0]).strftime('%Y-%m-%d %H:%M:%S')
      else:
        self.logger.debug("Resource %s hasn't publish date" % str(self.article.tags[CONSTS.TAG_LINK]["data"]))
      # for ml
      if self.store_xpath:
        self.storeXPath()
    else:
      self.logger.debug("Resource hasn't raw content. Exit.")

Here is the call graph for this function:

Here is the caller graph for this function:

processBatch()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.processBatch

(

self

)

Definition at line 304 of file ProcessorFeedParser.py.

  def processBatch(self):
    try:
      if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
        # read pickled batch object from stdin and unpickle it
        self.input_data = pickle.loads(sys.stdin.read())

      msgStr = "siteId=" + str(self.input_data.siteId) + ", urlId=" + str(self.input_data.urlId) + \
               ", url=" + str(self.input_data.url)
      self.logger.info("Incoming data:%s", msgStr)
      app.Profiler.messagesList.append(msgStr)
      self.logger.debug("self.input_data:%s\n", varDump(self.input_data))


      self.process()

      self.logger.info("Resulted tagsCount=%s, tagsMask=%s, pubdate=%s", str(self.tagsCount), str(self.tagsMask),
                       str(self.pubdate))

      scraperResponse = ScraperResponse(self.tagsCount, self.tagsMask, self.pubdate, self.processedContent)
      self.logger.debug("scraperResponse:%s\n", varDump(scraperResponse))

      if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
        print pickle.dumps(scraperResponse)
        sys.stdout.flush()
      else:
        self.output_data = scraperResponse
    except Exception as err:
      ExceptionLog.handler(self.logger, err, CONSTS.MSG_ERROR_PROCESS)
      self.exit_code = CONSTS.EXIT_FAILURE
      raise Exception(err)

Here is the call graph for this function:

Here is the caller graph for this function:

putArticleToDB()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.putArticleToDB	(		self,
			result
	)

Definition at line 264 of file ProcessorFeedParser.py.

  def putArticleToDB(self, result):
    # Check if Content extracted
    self.logger.info("Result: %s" % varDump(result))
    tags_count = int(result["default"].tagsCount)
    self.logger.info("Tags count: %s" % str(tags_count))
    if int(tags_count) > 0:
      self.logger.info("Tags count OK")

      # Get DB table name
      query_prefix = self.getQueryPrefix()

      # Prepare query
      # Firstly, prepare data buffer
      data = self.getDataBuffer(result)
      # Then, insert data buffer to the query
      #query = query_prefix + "`(`id`,`data`, CDate) VALUES('" \
      #  + result.resId + "', '" + encode(result.get()) + "', strftime('%Y-%m-%d %H:%M:%f', 'now', 'localtime'))"
      query = query_prefix + "`(`id`,`data`, CDate) VALUES('" + result["default"].resId + "', '" + data + "', NOW())"

      # Initialize options for DB Backend
      options = {}
      # Genaral options
      options["query"] = query
      self.logger.info("DB Backend %s" % str(self.db_engine))

      if self.db_engine == CONSTS.MYSQL_ENGINE:
        # MYSQL options
        options["dbHost"] = self.dbHost  # pylint: disable=E1101
        options["dbPort"] = self.dbPort  # pylint: disable=E1101
        options["dbUser"] = self.dbUser  # pylint: disable=E1101
        options["dbPWD"] = self.dbPWD  # pylint: disable=E1101
        options["MYSQLDBName"] = self.dc_contents_db  # pylint: disable=E1101
        #TODO: remove obsolete functionality
        #Utils.putContentToMYSQLDB(result, options)
      else:
        self.logger.info("DB Backend %s not supported" % str(self.db_engine))

Here is the call graph for this function:

Here is the caller graph for this function:

run()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.run

(

self

)

Definition at line 81 of file ProcessorFeedParser.py.

  def run(self):
    if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
      # call base class run method
      foundation.CementApp.run(self)

    # config section
    self.loadConfig()

    # load logger config file
    self.loadLogConfigFile()

    # load sqlite db backend
    # self.loadSqliteDBBackend()

    # sqlite
    # self.loadDBBackend()

    # options
    self.loadOptions()

    #Do applied algorithm's job
    self.processBatch()

    if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
      # Finish logging
      self.logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)

Here is the call graph for this function:

setup()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.setup

(

self

)

Definition at line 73 of file ProcessorFeedParser.py.

  def setup(self):
    if self.usageModel == APP_CONSTS.APP_USAGE_MODEL_PROCESS:
      # call base class setup method
      foundation.CementApp.setup(self)

storeXPath()

def dc_processor.ProcessorFeedParser.ProcessorFeedParser.storeXPath

(

self

)

storeXpath

Definition at line 338 of file ProcessorFeedParser.py.

  def storeXPath(self):
    ret = False
    '''
    from dc_processor.base_extractor import BaseExtractor
    # check content's presence in response
    if self.scraper_response.tagsMask & BaseExtractor.tagsMask[CONSTS.SUMMARY_DETAIL]:
      content = None
      tags = json.loads(self.scraper_response.processedContent)["data"]["tagList"]
      for tag in tags:
        if tag["name"]==CONSTS.SUMMARY_DETAIL:
          content = tag["data"]["value"]
          self.logger.debug("content: %s" % str(content))
          from scrapy.selector import Selector
          sel = Selector(text=self.raw_content)
          xpath_list = sel.xpath("//*[contains(., '" + content + "')]").extract()
          self.logger.debug("xpath_list: %s" % str(xpath_list))
    '''  # pylint: disable=W0105

    return ret

Here is the caller graph for this function:

Member Data Documentation

article

dc_processor.ProcessorFeedParser.ProcessorFeedParser.article

Definition at line 59 of file ProcessorFeedParser.py.

articles_tbl

dc_processor.ProcessorFeedParser.ProcessorFeedParser.articles_tbl

Definition at line 61 of file ProcessorFeedParser.py.

config

dc_processor.ProcessorFeedParser.ProcessorFeedParser.config

Definition at line 364 of file ProcessorFeedParser.py.

config_db_dir

dc_processor.ProcessorFeedParser.ProcessorFeedParser.config_db_dir

Definition at line 50 of file ProcessorFeedParser.py.

configFile

dc_processor.ProcessorFeedParser.ProcessorFeedParser.configFile

Definition at line 66 of file ProcessorFeedParser.py.

db_engine

dc_processor.ProcessorFeedParser.ProcessorFeedParser.db_engine

Definition at line 58 of file ProcessorFeedParser.py.

entry

dc_processor.ProcessorFeedParser.ProcessorFeedParser.entry

Definition at line 52 of file ProcessorFeedParser.py.

exit_code

dc_processor.ProcessorFeedParser.ProcessorFeedParser.exit_code

Definition at line 48 of file ProcessorFeedParser.py.

input_data

dc_processor.ProcessorFeedParser.ProcessorFeedParser.input_data

Definition at line 60 of file ProcessorFeedParser.py.

logger

dc_processor.ProcessorFeedParser.ProcessorFeedParser.logger

Definition at line 49 of file ProcessorFeedParser.py.

output_data

dc_processor.ProcessorFeedParser.ProcessorFeedParser.output_data

Definition at line 67 of file ProcessorFeedParser.py.

PRAGMA_journal_mode

dc_processor.ProcessorFeedParser.ProcessorFeedParser.PRAGMA_journal_mode

Definition at line 63 of file ProcessorFeedParser.py.

PRAGMA_synchronous

dc_processor.ProcessorFeedParser.ProcessorFeedParser.PRAGMA_synchronous

Definition at line 62 of file ProcessorFeedParser.py.

PRAGMA_temp_store

dc_processor.ProcessorFeedParser.ProcessorFeedParser.PRAGMA_temp_store

Definition at line 64 of file ProcessorFeedParser.py.

processedContent

dc_processor.ProcessorFeedParser.ProcessorFeedParser.processedContent

Definition at line 56 of file ProcessorFeedParser.py.

pubdate

dc_processor.ProcessorFeedParser.ProcessorFeedParser.pubdate

Definition at line 55 of file ProcessorFeedParser.py.

sqliteTimeout

dc_processor.ProcessorFeedParser.ProcessorFeedParser.sqliteTimeout

Definition at line 51 of file ProcessorFeedParser.py.

store_xpath

dc_processor.ProcessorFeedParser.ProcessorFeedParser.store_xpath

Definition at line 57 of file ProcessorFeedParser.py.

tagsCount

dc_processor.ProcessorFeedParser.ProcessorFeedParser.tagsCount

Definition at line 53 of file ProcessorFeedParser.py.

tagsMask

dc_processor.ProcessorFeedParser.ProcessorFeedParser.tagsMask

Definition at line 54 of file ProcessorFeedParser.py.

usageModel

dc_processor.ProcessorFeedParser.ProcessorFeedParser.usageModel

Definition at line 65 of file ProcessorFeedParser.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_processor/ProcessorFeedParser.py