dc_processor.ScraperMultiItemsTask.ScraperResultDocuments Class Reference

Inheritance diagram for dc_processor.ScraperMultiItemsTask.ScraperResultDocuments:

Collaboration diagram for dc_processor.ScraperMultiItemsTask.ScraperResultDocuments:

Public Member Functions
def	__init__ (self, keys, urlId)
def	addEtree (self, key, value)
def	addDoc (self, key, value, join, isExtract, mandatory)
def	getMaxCount (self, inDict)
def	getTagNamesExistAllDocs (self)
def	getCommonPath (self, lhs, rhs, logger=None)
def	calculateIndexPath (self, etree, logger=None)
def	getIndexNumberOfPath (self, indexPath, elemPath, logger=None)
def	getAllTags (self, mandatoryTags, logger=None)
def	updateTagValue (self, result, tags, tag_name)
def	getAllDocs (self, mandatoryTags, logger=None)

Public Attributes
	urlId
	docs
	join
	isExtract
	mandatory
	etree

Detailed Description

Definition at line 65 of file ScraperMultiItemsTask.py.

Constructor & Destructor Documentation

__init__()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.__init__	(		self,
			keys,
			urlId
	)

Definition at line 70 of file ScraperMultiItemsTask.py.

  def __init__(self, keys, urlId):
    self.urlId = urlId
    self.docs = {}
    self.join = {}
    self.isExtract = {}
    self.mandatory = {}
    self.etree = {}
    for key in keys:
      self.docs[key] = []
      self.join[key] = []
      self.isExtract[key] = []
      self.mandatory[key] = []
      self.etree[key] = []

Member Function Documentation

addDoc()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.addDoc	(		self,
			key,
			value,
			join,
			isExtract,
			mandatory
	)

Definition at line 105 of file ScraperMultiItemsTask.py.

  def addDoc(self, key, value, join, isExtract, mandatory):
    if not self.docs.has_key(key):
      self.docs[key] = []
      self.join[key] = []
      self.isExtract[key] = []
      self.mandatory[key] = []

    self.docs.get(key).append(copy.deepcopy(value))
    self.join.get(key).append(copy.deepcopy(join))
    self.isExtract.get(key).append(copy.deepcopy(isExtract))
    self.mandatory.get(key).append(copy.deepcopy(mandatory))

addEtree()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.addEtree	(		self,
			key,
			value
	)

Definition at line 90 of file ScraperMultiItemsTask.py.

  def addEtree(self, key, value):
    if not self.etree.has_key(key):
      self.docs[key] = []
      self.join[key] = []
      self.isExtract[key] = []
      self.mandatory[key] = []
      self.etree[key] = []

    self.etree.get(key).append(copy.deepcopy(value))

calculateIndexPath()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.calculateIndexPath	(		self,
			etree,
			logger = None
	)

Definition at line 188 of file ScraperMultiItemsTask.py.

  def calculateIndexPath(self, etree, logger=None):
    # variable for result
    ret = []
    pathDict = {}
    pathList = []

    for key in etree.keys():
      pathList.extend(etree.get(key))

    for index in range(len(pathList) - 1):
      commonPath = self.getCommonPath(pathList[index], pathList[index + 1], logger)
      commonPathCount = 0
      if pathDict.has_key(str(commonPath)):
        commonPathCount = int(pathDict.get(str(commonPath))[1])

      pathDict[str(commonPath)] = (commonPath, commonPathCount + 1)

    localpathList = []
    for elem in pathDict.values():
      localpathList.append(elem)

    localpathList.sort(key=lambda tup: tup[1], reverse=True)
    if len(localpathList) > 0:
      ret = (localpathList[0])[0]

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

getAllDocs()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getAllDocs	(		self,
			mandatoryTags,
			logger = None
	)

Definition at line 358 of file ScraperMultiItemsTask.py.

  def getAllDocs(self, mandatoryTags, logger=None):
    # variable for result
    resDocs = []

    resTags = self.getAllTags(mandatoryTags, logger)
    count = len(resTags)

    tagsNames = self.getTagNamesExistAllDocs()

    if len(tagsNames) > 0:
      key = tagsNames[0]

      for index in range(count):
        if len(self.join.get(key)) > index and \
        len(self.isExtract.get(key)) > index and \
        len(self.mandatory.get(key)) > index:
          resDocs.append({"obj": resTags[index],
                          "join": self.join.get(key)[index],
                          "isExtract": self.isExtract.get(key)[index],
                          "mandatory": self.mandatory.get(key)[index],
                          CONSTS.TAG_ORDER_NUMBER: len(resDocs) + 1})

    return resDocs



# # ScraperMultiItemsTask Class content main functional scrapering for multi items,
# class inherits from foundation.CementApp
#

Here is the call graph for this function:

getAllTags()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getAllTags	(		self,
			mandatoryTags,
			logger = None
	)

Definition at line 264 of file ScraperMultiItemsTask.py.

  def getAllTags(self, mandatoryTags, logger=None):
    # variable for result
    resTags = []
    count = self.getMaxCount(self.docs)

    # #Calculate index block
    indexPath = self.calculateIndexPath(self.etree, logger)
    if logger is not None:
      logger.info('Calculated indexPath: ' + str(indexPath))

    if logger is not None:
      for key in self.etree:
        logger.debug('len(self.etree.get(' + str(key) + ') = ' + str(len(self.etree.get(key))))
      for key in self.docs:
        logger.debug('len(self.docs.get(' + str(key) + ') = ' + str(len(self.docs.get(key))))

    resultList = []
    for index in range(self.getMaxCount(self.etree)):
      localRes = Result(None, self.urlId)
      resultList.append(localRes)

    if logger is not None:
      logger.debug('count = ' + str(count))
      logger.debug('len(resultList) = ' + str(len(resultList)))

    for key in self.docs.keys():
      for index in range(len(self.docs.get(key))):
        if logger is not None:
          logger.debug('==== key: ' + str(key) + ' index: ' + str(index) + ' ====')

        if len(self.etree.get(key)) > index:
          number = int(self.getIndexNumberOfPath(indexPath, self.etree.get(key)[index], logger))
          if logger is not None:
            logger.debug('number = ' + str(number) + ' self.docs.get(' + str(key) + ')[' + str(index) + '].tags: ' + \
                         varDump(self.docs.get(key)[index].tags))

          if int(number) > 0 and int(number) <= len(self.docs.get(key)):
            if resultList[int(number) - 1].tags.has_key(key):
              result = self.updateTagValue(resultList[int(number) - 1], self.docs.get(key)[index].tags, key)
              resultList[int(number) - 1].tags.update(result.tags)
            else:
              resultList[int(number) - 1].tags.update({key:self.docs.get(key)[index].tags[key]})

            if logger is not None:
              logger.debug("resultList[" + str(int(number) - 1) + "].tags.update({" + str(key) + ":self.docs.get(" + \
                         str(key) + ")[" + str(index) + "].tags[" + str(key) + "]})")

    for index in range(0, len(resultList)):
      isMandatory = True
      countSelected = 0
      for key in self.docs.keys():
        if not resultList[index].tags.has_key(key) and bool(mandatoryTags[key]) is True:
          isMandatory = False
          break

        if resultList[index].tags.has_key(key):
          countSelected = countSelected + 1

      if countSelected == 0:
        isMandatory = False

      if isMandatory:
        resTags.append(resultList[index])

    if len(resTags) == 0:
      resTags.append(Result(None, self.urlId))

    return resTags

Here is the call graph for this function:

Here is the caller graph for this function:

getCommonPath()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getCommonPath	(		self,
			lhs,
			rhs,
			logger = None
	)

Definition at line 151 of file ScraperMultiItemsTask.py.

  def getCommonPath(self, lhs, rhs, logger=None):  # pylint: disable=W0612,W0613
    # variable for result
    ret = []
    length = min(len(lhs), len(rhs))

    # if logger is not None:
      # logger.debug('>>> lhs: ' + str(lhs))
      # logger.debug('>>> rhs: ' + str(rhs))

    for i in range(length):
      if isinstance(lhs[i], str) and isinstance(rhs[i], str) and lhs[i] != rhs[i]:
        if i > 0:
          ret = lhs[:i]
        return ret

      # logger.info('len(lhs[' + str(i) + ']) = ' + str(len(lhs[i])) + \
      # ' len(rhs[' + str(i) + ']) = ' + str(len(rhs[i])))

      if isinstance(lhs[i], tuple) and isinstance(rhs[i], tuple) and len(lhs[i]) == len(rhs[i]):
        for j in range(len(lhs[i])):
          # logger.info('lhs[' + str(j) + '] = ' + str(lhs[i][j] + ' rhs[' + str(j) + '] = ' + str(rhs[i][j])))
          if lhs[i][j] != rhs[i][j]:

            # logger.info('lhs[:i] = ' + str(lhs[:i]))
            if i > 0:
              ret = lhs[:i]

            # logger.debug('ret = ' + str(ret))
            return ret

    return ret

Here is the caller graph for this function:

getIndexNumberOfPath()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getIndexNumberOfPath	(		self,
			indexPath,
			elemPath,
			logger = None
	)

Definition at line 221 of file ScraperMultiItemsTask.py.

  def getIndexNumberOfPath(self, indexPath, elemPath, logger=None):
    elementPath = copy.deepcopy(elemPath)
    length = min(len(indexPath), len(elementPath))

    if logger is not None:
      logger.debug('\n>>> indexPath: ' + str(indexPath))
      logger.debug('\n>>> elementPath: ' + str(elementPath))

    for i in range(length):
      if isinstance(indexPath[i], str) and isinstance(elementPath[i], str) and indexPath[i] != elementPath[i]:
        if logger is not None:
          logger.debug("Both have type 'str' and indexPath[" + str(i) + "] != elementPath[" + str(i) + "]")
        return -1

      if isinstance(indexPath[i], tuple) and isinstance(elementPath[i], tuple):
        size = min(len(indexPath[i]), len(elementPath[i]))
        for j in range(size):
          if indexPath[i][j] != elementPath[i][j]:
            if logger is not None:
              logger.debug("Both have type 'tuple' and indexPath[" + str(i) + "][" + str(j) + "] != elementPath[" + \
                           str(i) + "][" + str(j) + "]")
            return -1

    if len(elementPath) > len(indexPath):
      if logger is not None:
        logger.debug('type(elementPath[len(indexPath)])) = ' + str(type(elementPath[len(indexPath)])) + \
                     ' elementPath[' + str(len(indexPath)) + ']: ' + str(elementPath[len(indexPath)]))

      if isinstance(elementPath[len(indexPath)], tuple):
        if len(elementPath[len(indexPath)]) > 1:
          if logger is not None:
            logger.debug('>>> elementPath[' + str(len(indexPath)) + '][1] = ' + str(elementPath[len(indexPath)][1]))

          return elementPath[len(indexPath)][1]

    return -1

Here is the caller graph for this function:

getMaxCount()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getMaxCount	(		self,
			inDict
	)

Definition at line 122 of file ScraperMultiItemsTask.py.

  def getMaxCount(self, inDict):
    # variable for result
    count = 0
    for key in inDict.keys():
      count = max(count, len(inDict.get(key)))

    return count

Here is the caller graph for this function:

getTagNamesExistAllDocs()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.getTagNamesExistAllDocs

(

self

)

Definition at line 135 of file ScraperMultiItemsTask.py.

  def getTagNamesExistAllDocs(self):
    # variable for result
    tagNames = []
    count = self.getMaxCount(self.docs)
    for key in self.docs.keys():
      size = len(self.docs.get(key))
      if count == size:
        tagNames.append(key)

    return tagNames

Here is the call graph for this function:

Here is the caller graph for this function:

updateTagValue()

def dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.updateTagValue	(		self,
			result,
			tags,
			tag_name
	)

Definition at line 340 of file ScraperMultiItemsTask.py.

  def updateTagValue(self, result, tags, tag_name):

    data = {"extractor":"Base extractor", "data":"", "name":""}
    data["data"] = [result.tags[tag_name]["data"][0] + tags[tag_name]["data"][0]]
    data["name"] = result.tags[tag_name]["name"]
    data["xpath"] = result.tags[tag_name]["xpath"]
    data["type"] = result.tags[tag_name]["type"]
    data["extractor"] = result.tags[tag_name]["extractor"]
    result.tags[tag_name] = data

    return result

Here is the caller graph for this function:

Member Data Documentation

docs

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.docs

Definition at line 72 of file ScraperMultiItemsTask.py.

etree

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.etree

Definition at line 76 of file ScraperMultiItemsTask.py.

isExtract

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.isExtract

Definition at line 74 of file ScraperMultiItemsTask.py.

join

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.join

Definition at line 73 of file ScraperMultiItemsTask.py.

mandatory

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.mandatory

Definition at line 75 of file ScraperMultiItemsTask.py.

urlId

dc_processor.ScraperMultiItemsTask.ScraperResultDocuments.urlId

Definition at line 71 of file ScraperMultiItemsTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_processor/ScraperMultiItemsTask.py