dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing Class Reference

Inheritance diagram for dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing:

Collaboration diagram for dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing:

Public Member Functions
def	__init__ (self, innerTextTagReplacers=None, attrConditions=None)
def	resolveDelimiter (self, path, properties, defautlDelimiter=" ")
def	resolveInnerDelimiter (self, path, properties, defautlDelimiter=" ")
def	process (self, path, sel, delimiter=' ', innerDelimiter=' ', innerTextFunc=Utils.innerText)
def	getXpathValueForDTime (self, initXpath, sel, innerTextFunc=Utils.innerText)
def	extractXpathFromSelectorList (self, sList, localXpath, lambdaCall)

Public Attributes
	innerTextTagReplacers
	attrConditions

Detailed Description

Definition at line 16 of file TemplateExtractorXPathPreparing.py.

Constructor & Destructor Documentation

__init__()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.__init__	(		self,
			innerTextTagReplacers = None,
			attrConditions = None
	)

Definition at line 18 of file TemplateExtractorXPathPreparing.py.

  def __init__(self, innerTextTagReplacers=None, attrConditions=None):
    self.innerTextTagReplacers = innerTextTagReplacers
    self.attrConditions = attrConditions

Member Function Documentation

extractXpathFromSelectorList()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.extractXpathFromSelectorList	(		self,
			sList,
			localXpath,
			lambdaCall
	)

Definition at line 183 of file TemplateExtractorXPathPreparing.py.

  def extractXpathFromSelectorList(self, sList, localXpath, lambdaCall):
    ret = []
    for elem in sList:
      if lambdaCall(elem):
        ret = elem.xpath(localXpath).extract()
        if len(ret) > 1 and any(True for ch in ret if ch >= '0' and ch <= '9'):
          ret = [ret[0]]
          break
    return ret

getXpathValueForDTime()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.getXpathValueForDTime	(		self,
			initXpath,
			sel,
			innerTextFunc = Utils.innerText
	)

Definition at line 159 of file TemplateExtractorXPathPreparing.py.

  def getXpathValueForDTime(self, initXpath, sel, innerTextFunc=Utils.innerText):
    xpath = initXpath
    localXpath = sel.xpath(xpath)
    logger.info(">>> Datetime | Meta extraction")
    xpathValue = self.extractXpathFromSelectorList(localXpath, "@content", \
                                                   lambda elem: elem.extract().find("<meta") == 0)
    if len(xpathValue) == 0:
      logger.info(">>> Datetime | any tag @datetime argument extraction")
      xpathValue = self.extractXpathFromSelectorList(localXpath, "@datetime", \
                                                     lambda elem: elem.extract().find("<time") == 0)
      if len(xpathValue) == 0:
        logger.info(">>> Datetime | inner Text Extraction")
        localStr = innerTextFunc(localXpath, ' ', ' ', self.innerTextTagReplacers, None, self.attrConditions)
        if localStr != '':
          xpathValue = [localStr]
    return xpath, xpathValue

Here is the call graph for this function:

Here is the caller graph for this function:

process()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.process	(		self,
			path,
			sel,
			delimiter = ' ',
			innerDelimiter = ' ',
			innerTextFunc = Utils.innerText
	)

Definition at line 67 of file TemplateExtractorXPathPreparing.py.

  def process(self, path, sel, delimiter=' ', innerDelimiter=' ', innerTextFunc=Utils.innerText):
    xpath = None
    xpathValue = None
    # Added new template type specification
    #---> "text" rule type.
    if path["type"] == "text":
      localXpath = sel.xpath(path["target"])
      xpathValue = innerTextFunc(localXpath, delimiter, innerDelimiter, self.innerTextTagReplacers, None,
                                 self.attrConditions)
      xpath = path["target"]
    #---> "datetime" rule type.
    elif path["type"] == "datetime":
      xpath, xpathValue = self.getXpathValueForDTime(path["target"], sel, innerTextFunc)
      logger.info(">>> final XPath = " + str(xpathValue))
    #---> "image" rule type.
    elif path["type"] == "image":
      logger.info(">>> img format = " + str(path["format"]))
      if path["target"][0] not in SelectorWrapper.CSS_DETECT_SYMBOLS:
        if path["format"] == "URL":
          if path["target"].find('/@') == -1:
            localXPathPattern = '/@%s'
            subXPathesList = ['src', 'srcset', 'image-src']
            xpath, xpathValue = Utils.getFirstNotEmptySubXPath(path["target"], sel, localXPathPattern, subXPathesList)
        elif path["format"] == "DATA":
          pass
        elif path["format"] == "ALT":
          xpath = path["target"] + "/@alt"
        elif path["format"] == "TITLE":
          xpath = path["target"] + "/@title"
    #---> "html" rule type.
    elif path["type"] == "html":
      pass
    #---> "link" rule type.
    elif path["type"] == "link":
      if path["target"][0] not in SelectorWrapper.CSS_DETECT_SYMBOLS:
        formatName = path["format"]
        if len(formatName.split(',')) > 1:
          formatName = formatName.split(',')[1]
        if formatName == "email-text":
          isEmail = False
          if not Utils.isTailSubstr(path["target"], "/@href"):
            localXpath = path["target"] + "/@href"
            xpathValue = sel.xpath(localXpath).extract()
            for xpathValueElem in xpathValue:
              if isinstance(xpathValueElem, basestring) and xpathValueElem.find("mailto:") >= 0:
                isEmail = True
                break
          if isEmail:
            localXpath = sel.xpath(path["target"])
            xpathValue = innerTextFunc(localXpath, delimiter, innerDelimiter, self.innerTextTagReplacers, None,
                                       self.attrConditions)
          else:
            xpathValue = []
          xpath = path["target"]
        else:
          if not Utils.isTailSubstr(path["target"], "/@href"):
            xpath = path["target"] + "/@href"
    #---> "attribute" rule type.
    elif path["type"] == "attribute":
      if path["format"] == "":
        xpathValue = []
      else:
        splittedFormatString = path["format"].split(',')
        attrName = None
        if len(splittedFormatString) >= 2:
          attrName = splittedFormatString[1]
        else:
          attrName = splittedFormatString[0]
        if path["target"].rfind(attrName) == -1 or \
        (len(path["target"]) - len(attrName)) != path["target"].rfind(attrName):
          xpath = path["target"]
          xpath += "/@"
          xpath += attrName
    if xpath is None:
      xpath = path["target"]

    if xpathValue is None:
      try:
        xpathValue = sel.xpath(xpath).extract()
      except Exception as excp:
        logger.info(">>> Common xPath extractor exception=" + str(excp))
        xpathValue = []

    return xpath, xpathValue

Here is the call graph for this function:

Here is the caller graph for this function:

resolveDelimiter()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.resolveDelimiter	(		self,
			path,
			properties,
			defautlDelimiter = " "
	)

Definition at line 28 of file TemplateExtractorXPathPreparing.py.

  def resolveDelimiter(self, path, properties, defautlDelimiter=" "):
    ret = defautlDelimiter
    if "delimiter" in path and path["delimiter"]:
      ret = path["delimiter"]
    elif "SCRAPER_TAG_ITEMS_DELIMITER" in properties:
      ret = properties["SCRAPER_TAG_ITEMS_DELIMITER"]
    elif path["type"] == "text" or path["type"] == "html":
      ret = ' '
    else:
      ret = ','
    return ret

resolveInnerDelimiter()

def dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.resolveInnerDelimiter	(		self,
			path,
			properties,
			defautlDelimiter = " "
	)

Definition at line 46 of file TemplateExtractorXPathPreparing.py.

  def resolveInnerDelimiter(self, path, properties, defautlDelimiter=" "):
    ret = defautlDelimiter
    if "delimiter_sub_items" in path and path["delimiter_sub_items"]:
      ret = path["delimiter_sub_items"]
    elif "SCRAPER_TAG_ITEMS_INNER_DELIMITER" in properties:
      ret = properties["SCRAPER_TAG_ITEMS_INNER_DELIMITER"]
    elif path["type"] == "text" or path["type"] == "html":
      ret = ' '
    else:
      ret = ','
    return ret

Member Data Documentation

attrConditions

dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.attrConditions

Definition at line 20 of file TemplateExtractorXPathPreparing.py.

innerTextTagReplacers

dc_processor.TemplateExtractorXPathPreparing.TemplateExtractorXPathPreparing.innerTextTagReplacers

Definition at line 19 of file TemplateExtractorXPathPreparing.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_processor/TemplateExtractorXPathPreparing.py