app.ExtendInnerText.ExtendInnerText Class Reference

Inheritance diagram for app.ExtendInnerText.ExtendInnerText:

Collaboration diagram for app.ExtendInnerText.ExtendInnerText:

Public Member Functions
def	__init__ (self, tagReplacers=None, delimiter=' ', innerDelimiter=' ', REconditions=None, attrConditions=None, keepAttributes=None, baseUrl=None, closeVoid=None, excludeNodes=None)
def	nodeCallbackOpenHandler (self, nodeElem, level)
def	nodeCallbackCloseHandler (self, nodeElem, level)
def	textCallbackHandler (self, nodeElem, level, excludeTags)
def	innerText (self, contentBuf, xPath, tagRemoves=None)
def	innerTextToList (self, contentBuf, xPath, tagRemoves=None)
def	extractAttributes (self, nodeElem, tagName, keepAttributes, baseUrl)
def	applyCloseVoid (self, nodeElem, tagName)

Static Public Member Functions
def	checkElemAttributes (attrConditions, elem)
def	traversalNodes (elemList, level=0, nodeCallbackOpen=None, nodeCallbackClose=None, textCallback=None, excludeTags=None, attrConditions=None, excludeNodes=None)
def	isExcludeNode (excludeNodes, elem)
	check is exlude node More...

Public Attributes
	stripHtml
	stripHtmlList
	errorString
	delimiter
	innerDelimiter
	REconditions
	attrConditions
	tagReplacers
	keepAttributes
	baseUrl
	closeVoid
	excludeNodes

Static Public Attributes
list	NONE_CLOSED_HTML_TAGS
list	CANONIZATION_TAGS = ['href', 'src']
string	MACRO_ATTRIBUTES = '%ATTRIBUTES%'
string	PATTERN_CLOSE_VOID = r"<%s.*?(/)>"
int	CLOSE_VOID_NOT_CLOSE = 0
int	CLOSE_VOID_CLOSE = 1
int	CLOSE_VOID_AUTO = 2
	values = elem.xpath('@' + attrName).extract()
bool	found = False
bool	ret = True

Detailed Description

Definition at line 20 of file ExtendInnerText.py.

Constructor & Destructor Documentation

__init__()

def app.ExtendInnerText.ExtendInnerText.__init__	(		self,
			tagReplacers = None,
			delimiter = ' ',
			innerDelimiter = ' ',
			REconditions = None,
			attrConditions = None,
			keepAttributes = None,
			baseUrl = None,
			closeVoid = None,
			excludeNodes = None
	)

Definition at line 36 of file ExtendInnerText.py.

               keepAttributes=None, baseUrl=None, closeVoid=None, excludeNodes=None):
    self.stripHtml = ''
    self.stripHtmlList = []
    self.errorString = ''
    self.delimiter = delimiter
    self.innerDelimiter = innerDelimiter
    self.REconditions = REconditions
    self.attrConditions = attrConditions
    self.tagReplacers = tagReplacers 
    self.keepAttributes = keepAttributes
    self.baseUrl = baseUrl
    self.closeVoid = closeVoid 
    self.excludeNodes = excludeNodes

Member Function Documentation

applyCloseVoid()

def app.ExtendInnerText.ExtendInnerText.applyCloseVoid	(		self,
			nodeElem,
			tagName
	)

Definition at line 266 of file ExtendInnerText.py.

  def applyCloseVoid(self, nodeElem, tagName):
    # variable for result
    ret = ''
    
    if tagName in self.NONE_CLOSED_HTML_TAGS:
      closeVoid = self.CLOSE_VOID_NOT_CLOSE
      if self.closeVoid is not None:
        closeVoid = int(self.closeVoid) 
      
      if closeVoid == self.CLOSE_VOID_NOT_CLOSE:
        ret = ''
      elif closeVoid == self.CLOSE_VOID_CLOSE:   
        ret = '/'
      elif closeVoid == self.CLOSE_VOID_AUTO: 
#         logger.info("!!!!! BEFORE nodeElem.select()")
#         for sel in nodeElem.select('//*'):
#           logger.info("!!!!! sel: '%s'", str(sel.extract()))  
        
        logger.info("!!!!! nodeElem.extract(): '%s'", str(nodeElem.extract()))  
        pattern = self.PATTERN_CLOSE_VOID % str(tagName)
        logger.info("!!!!! pattern: '%s'", str(pattern)) 
        res = nodeElem.re(pattern)
        logger.info("!!!!! nodeElem.re(pattern): '%s'", str(res))
        if len(res) > 0:
          ret = '/'
    
    return ret    
    
    

Here is the caller graph for this function:

checkElemAttributes()

def app.ExtendInnerText.ExtendInnerText.checkElemAttributes (   attrConditions,   elem  )

static

Definition at line 145 of file ExtendInnerText.py.

  def checkElemAttributes(attrConditions, elem):
    ret = True
    if attrConditions is not None:
      if attrConditions["TYPE"] == "include":
        ret = False
      i = 1
      attrList = elem.xpath("@*")
      if len(attrList) > 0:
        for internalElem in attrList:
          for key in attrConditions:
            attrName = "".join(elem.xpath("name(@*[%s])" % str(i)).extract())
            if key != "type" and (key == "*" or key == attrName) and \
            re.compile(attrConditions[key]).match(internalElem.extract()):
              ret = not ret
              break
          i += 1
      elif "NO_ATTRIBUTES" in attrConditions:
        ret = True
    return ret

Here is the call graph for this function:

extractAttributes()

def app.ExtendInnerText.ExtendInnerText.extractAttributes	(		self,
			nodeElem,
			tagName,
			keepAttributes,
			baseUrl
	)

Definition at line 236 of file ExtendInnerText.py.

  def extractAttributes(self, nodeElem, tagName, keepAttributes, baseUrl):
    # variable for result
    ret = ''

#     logger.info("!!! tagName = %s", str(tagName))
    import app.Utils
    if keepAttributes is not None and tagName in keepAttributes.keys():
      attrList = keepAttributes[tagName]
      values = []
      for attrName in attrList:
        value = nodeElem.xpath('@' + attrName).extract()
#         logger.info("!!! for %s extracted: %s", str(attrName), str(value))
        if len(value) > 0 and value[0] != "":
          if attrName in self.CANONIZATION_TAGS:
            value[0] = app.Utils.urlNormalization(baseUrl, value[0])

          values.append(attrName + '="' + value[0].replace('\n', ' ').replace('"', '\\\"') + '"')

      # ret = "<" + tagName + ' ' + ' '.join(values) + '>'
      ret = ' ' + ' '.join(values)

#     logger.debug("!!! return: '%s'", str(ret))
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

innerText()

def app.ExtendInnerText.ExtendInnerText.innerText	(		self,
			contentBuf,
			xPath,
			tagRemoves = None
	)

Definition at line 109 of file ExtendInnerText.py.

  def innerText(self, contentBuf, xPath, tagRemoves=None):
    self.stripHtml = ''
    self.errorString = ''
    if xPath is not None:
      if tagRemoves is None:
        tagRemoves = ['script', 'style', '']
      try:
        if isinstance(xPath, basestring):
          sel = SelectorWrapper(text=contentBuf)
          selectorElem = sel.xpath(xPath)
        else:
          selectorElem = xPath
        localBuf = ''

        for elem in selectorElem:
          if self.REconditions is not None:
            if (self.REconditions["type"] == "include" and re.compile(self.REconditions["RE"]).match(elem) is None) or \
            (self.REconditions["type"] == "exclude" and re.compile(self.REconditions["RE"]).match(elem) is not None):
              continue
          self.stripHtml = ''
          elemList = []
          elemList.append(elem)
          ExtendInnerText.traversalNodes(elemList, 0, self.nodeCallbackOpenHandler, self.nodeCallbackCloseHandler,
                                         self.textCallbackHandler, tagRemoves, self.attrConditions, self.excludeNodes)
          localBuf += self.stripHtml.strip(self.innerDelimiter)
          localBuf += self.delimiter

        self.stripHtml = localBuf.strip(self.delimiter)
      except Exception as excp:
        self.errorString = str(excp)
        logger.error("!!! Exception: %s", str(self.errorString))
        import app.Utils as Utils
        logger.info(Utils.getTracebackInfo())

Here is the call graph for this function:

innerTextToList()

def app.ExtendInnerText.ExtendInnerText.innerTextToList	(		self,
			contentBuf,
			xPath,
			tagRemoves = None
	)

Definition at line 198 of file ExtendInnerText.py.

  def innerTextToList(self, contentBuf, xPath, tagRemoves=None):
    stripHtmlList = []
    self.stripHtml = ''
    self.errorString = ''
    if xPath is not None:
      if tagRemoves is None:
        tagRemoves = ['script', 'style', '']
      try:
        if isinstance(xPath, types.StringTypes):
          sel = SelectorWrapper(text=contentBuf)
          selectorElem = sel.xpath(xPath)
        else:
          selectorElem = xPath

        for elem in selectorElem:
          if self.REconditions is not None:
            if (self.REconditions["type"] == "include" and re.compile(self.REconditions["RE"]).match(elem) is None) or \
            (self.REconditions["type"] == "exclude" and re.compile(self.REconditions["RE"]).match(elem) is not None):
              continue
          self.stripHtml = ''
          elemList = []
          elemList.append(elem)
          ExtendInnerText.traversalNodes(elemList, 0, self.nodeCallbackOpenHandler, self.nodeCallbackCloseHandler,
                                         self.textCallbackHandler, tagRemoves, self.attrConditions, self.excludeNodes)

          stripHtmlList.append((self.stripHtml.strip(self.innerDelimiter) + self.delimiter).strip(self.delimiter))

        self.stripHtmlList = stripHtmlList
      except Exception as excp:
        self.errorString = str(excp)

Here is the call graph for this function:

isExcludeNode()

def app.ExtendInnerText.ExtendInnerText.isExcludeNode (   excludeNodes,   elem  )

static

check is exlude node

Parameters

excludeNodes	- dictionary with criterion for exclude
elem	- element for check

Returns

True if necessary exclude or False otherwise

Definition at line 301 of file ExtendInnerText.py.

  def isExcludeNode(excludeNodes, elem):
    # variable for result
    ret = False
#     logger.debug("!!! excludeNodes: %s, type: %s", str(excludeNodes), str(type(excludeNodes)))

    if len(elem.xpath("name()")) > 0:
      nodeName = str(elem.xpath("name()")[0].extract())
#       logger.debug("!!! nodeName: %s, type: %s", str(nodeName), str(type(nodeName)))

      import app.Utils as Utils

      if isinstance(excludeNodes, list):
        for excludeNode in excludeNodes:
          if isinstance(excludeNode, dict):
            for tagName, attributes in excludeNode.items():
#               logger.debug("!!! tagName: %s, attributes: %s", str(tagName), str(attributes))
              if Utils.reMatch(tagName, nodeName, logger):
#                 logger.debug("tagName: %s == nodeName: %s", str(tagName), str(nodeName))
                if attributes is None:
                  logger.debug("Found exclude node rule for '%s' with attributes: %s", str(tagName), str(attributes))
                  ret = True
                  break

                if isinstance(attributes, dict):
                  for attrName, attrValue in attributes.items():

nodeCallbackCloseHandler()

def app.ExtendInnerText.ExtendInnerText.nodeCallbackCloseHandler	(		self,
			nodeElem,
			level
	)

Definition at line 78 of file ExtendInnerText.py.

  def nodeCallbackCloseHandler(self, nodeElem, level):  # pylint: disable=W0613
    closeTagName = str(nodeElem.xpath("name()")[0].extract())
#     logger.info("closeTagName: %s", str(closeTagName))
    if self.tagReplacers is None and closeTagName not in self.NONE_CLOSED_HTML_TAGS and closeTagName != "":
      self.stripHtml += '</' + closeTagName + '>'
    else:
      closeTag = "</" + closeTagName + ">"
      if (len(str(nodeElem.extract())) >= len(closeTag)) and \
      str(nodeElem.extract()).rfind(closeTag) == (len(str(nodeElem.extract())) - len(closeTag)):
        closeTagName = '/' + closeTagName
        if self.tagReplacers is not None and closeTagName in self.tagReplacers:
          self.stripHtml += self.tagReplacers[closeTagName]

Here is the caller graph for this function:

nodeCallbackOpenHandler()

def app.ExtendInnerText.ExtendInnerText.nodeCallbackOpenHandler	(		self,
			nodeElem,
			level
	)

Definition at line 51 of file ExtendInnerText.py.

  def nodeCallbackOpenHandler(self, nodeElem, level):  # pylint: disable=W0613
    openTagName = str(nodeElem.xpath("name()")[0].extract())

#     logger.info("self.tagReplacers: %s", str(self.tagReplacers))
#     logger.info("openTagName: %s", str(openTagName))
       
    if self.tagReplacers is None:

      if self.MACRO_ATTRIBUTES in openTagName:
        self.stripHtml += '<' + openTagName.replace(self.MACRO_ATTRIBUTES, self.extractAttributes(nodeElem, \
                          openTagName, self.keepAttributes, self.baseUrl)) + self.applyCloseVoid(nodeElem, openTagName) + '>'
      elif openTagName != "":
        self.stripHtml += '<' + openTagName + self.extractAttributes(nodeElem, openTagName, self.keepAttributes, \
                          self.baseUrl) + self.applyCloseVoid(nodeElem, openTagName) + '>'

#       logger.info("self.stripHtml1: %s", str(self.stripHtml))
    else:
      if openTagName in self.tagReplacers:
        self.stripHtml += self.tagReplacers[openTagName].replace(self.MACRO_ATTRIBUTES, \
                          self.extractAttributes(nodeElem, openTagName, self.keepAttributes, self.baseUrl))

#       logger.info("self.stripHtml2: %s", str(self.stripHtml))

#     logger.info("!!! nodeElem.xpath('name(@href)').extract() = %s", str(nodeElem.xpath('name(@href)').extract()))
#     logger.info("!!! nodeElem.xpath('@href').extract() = %s", str(nodeElem.xpath('@href').extract()))

Here is the call graph for this function:

Here is the caller graph for this function:

textCallbackHandler()

def app.ExtendInnerText.ExtendInnerText.textCallbackHandler	(		self,
			nodeElem,
			level,
			excludeTags
	)

Definition at line 92 of file ExtendInnerText.py.

  def textCallbackHandler(self, nodeElem, level, excludeTags): # pylint: disable=W0613
#     logger.debug("excludeTags: %s", str(excludeTags))

    buff = str(nodeElem.extract())
    if buff.strip() != "":
      for excludeTag in excludeTags:
        if excludeTag != "":
          pattern = '<' + excludeTag + '.*>'
          buff = re.sub(pattern=pattern, repl='', string=buff, flags=re.I + re.U + re.M)

      if self.tagReplacers is None:
        self.stripHtml += buff
      else:
        self.stripHtml += buff + self.innerDelimiter
#       self.stripHtml += str(nodeElem.extract()) + self.innerDelimiter

Here is the caller graph for this function:

traversalNodes()

def app.ExtendInnerText.ExtendInnerText.traversalNodes (   elemList,   level = 0,   nodeCallbackOpen = None,   nodeCallbackClose = None,   textCallback = None,   excludeTags = None,   attrConditions = None,   excludeNodes = None  )

static

Definition at line 168 of file ExtendInnerText.py.

                     excludeTags=None, attrConditions=None, excludeNodes=None):
    if excludeTags is None:
      excludeTags = ['script', 'style', '']
    # print str(level) + " " + str(len(elemList))

#     logger.debug("elemList: %s", str(elemList))
#     logger.debug("excludeNodes: %s", str(excludeNodes))
    for elem in elemList:

      if not ExtendInnerText.checkElemAttributes(attrConditions, elem):
        continue
      
      if ExtendInnerText.isExcludeNode(excludeNodes, elem):
        continue

      if len(elem.xpath("name()")) > 0:
        if nodeCallbackOpen is not None and str(elem.xpath("name()")[0].extract()) not in excludeTags:
          nodeCallbackOpen(elem, level)
          
          
        if str(elem.xpath("name()")[0].extract()) not in excludeTags:
          ExtendInnerText.traversalNodes(elem.xpath("node()"), level + 1, nodeCallbackOpen, nodeCallbackClose,
                                         textCallback, excludeTags, attrConditions, excludeNodes)
        if nodeCallbackClose is not None and str(elem.xpath("name()")[0].extract()) not in excludeTags:
          nodeCallbackClose(elem, level)
      else:
        if textCallback is not None:
          textCallback(elem, level, excludeTags)

Here is the caller graph for this function:

Member Data Documentation

attrConditions

app.ExtendInnerText.ExtendInnerText.attrConditions

Definition at line 43 of file ExtendInnerText.py.

baseUrl

app.ExtendInnerText.ExtendInnerText.baseUrl

Definition at line 46 of file ExtendInnerText.py.

CANONIZATION_TAGS

list app.ExtendInnerText.ExtendInnerText.CANONIZATION_TAGS = ['href', 'src']

static

Definition at line 25 of file ExtendInnerText.py.

CLOSE_VOID_AUTO

int app.ExtendInnerText.ExtendInnerText.CLOSE_VOID_AUTO = 2

static

Definition at line 33 of file ExtendInnerText.py.

CLOSE_VOID_CLOSE

int app.ExtendInnerText.ExtendInnerText.CLOSE_VOID_CLOSE = 1

static

Definition at line 32 of file ExtendInnerText.py.

CLOSE_VOID_NOT_CLOSE

int app.ExtendInnerText.ExtendInnerText.CLOSE_VOID_NOT_CLOSE = 0

static

Definition at line 31 of file ExtendInnerText.py.

closeVoid

app.ExtendInnerText.ExtendInnerText.closeVoid

Definition at line 47 of file ExtendInnerText.py.

delimiter

app.ExtendInnerText.ExtendInnerText.delimiter

Definition at line 40 of file ExtendInnerText.py.

errorString

app.ExtendInnerText.ExtendInnerText.errorString

Definition at line 39 of file ExtendInnerText.py.

excludeNodes

app.ExtendInnerText.ExtendInnerText.excludeNodes

Definition at line 48 of file ExtendInnerText.py.

found

bool app.ExtendInnerText.ExtendInnerText.found = False

static

Definition at line 330 of file ExtendInnerText.py.

innerDelimiter

app.ExtendInnerText.ExtendInnerText.innerDelimiter

Definition at line 41 of file ExtendInnerText.py.

keepAttributes

app.ExtendInnerText.ExtendInnerText.keepAttributes

Definition at line 45 of file ExtendInnerText.py.

MACRO_ATTRIBUTES

string app.ExtendInnerText.ExtendInnerText.MACRO_ATTRIBUTES = '%ATTRIBUTES%'

static

Definition at line 27 of file ExtendInnerText.py.

NONE_CLOSED_HTML_TAGS

list app.ExtendInnerText.ExtendInnerText.NONE_CLOSED_HTML_TAGS

static

Initial value:

=  ['area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img', 'input', 'keygen', 'link',
                           'meta', 'param', 'source', 'track', 'wbr']

Definition at line 22 of file ExtendInnerText.py.

PATTERN_CLOSE_VOID

string app.ExtendInnerText.ExtendInnerText.PATTERN_CLOSE_VOID = r"<%s.*?(/)>"

static

Definition at line 29 of file ExtendInnerText.py.

REconditions

app.ExtendInnerText.ExtendInnerText.REconditions

Definition at line 42 of file ExtendInnerText.py.

ret

bool app.ExtendInnerText.ExtendInnerText.ret = True

static

Definition at line 338 of file ExtendInnerText.py.

stripHtml

app.ExtendInnerText.ExtendInnerText.stripHtml

Definition at line 37 of file ExtendInnerText.py.

stripHtmlList

app.ExtendInnerText.ExtendInnerText.stripHtmlList

Definition at line 38 of file ExtendInnerText.py.

tagReplacers

app.ExtendInnerText.ExtendInnerText.tagReplacers

Definition at line 44 of file ExtendInnerText.py.

values

app.ExtendInnerText.ExtendInnerText.values = elem.xpath('@' + attrName).extract()

static

Definition at line 327 of file ExtendInnerText.py.

---

The documentation for this class was generated from the following file:

• sources/hce/app/ExtendInnerText.py