dc_postprocessor.LinkResolver.LinkResolver Class Reference

Inheritance diagram for dc_postprocessor.LinkResolver.LinkResolver:

Collaboration diagram for dc_postprocessor.LinkResolver.LinkResolver:

Public Member Functions
def	__init__ (self, getConfigOption=None, log=None)
def	init (self)
def	resolve (self, url)
def	processBatchItem (self, batchItem)
Public Member Functions inherited from dc_postprocessor.PostProcessingModuleClass.PostProcessingModuleClass
def	__init__ (self, getConfigOption=None, log=None)
def	init (self)
def	processBatch (self, batchObj)
def	processBatchItem (self, batchItemObj)

Public Attributes
	method
	delimiter
	headers
	siteProperty
Public Attributes inherited from dc_postprocessor.PostProcessingModuleClass.PostProcessingModuleClass
	getConfigOption
	logger

Static Public Attributes
string	LINK_RESOLVE_PROPERTY_NAME = 'LINK_RESOLVE'
string	CONFIG_OPTION_METHOD = 'method'
string	CONFIG_OPTION_DELIMITER = 'delimiter'
string	CONFIG_OPTION_HEADER_FILE = 'headers_file'
string	PROPERTY_NAME_METHOD = 'method'
string	LINK_FIELD_NAME = 'link'
string	SEARCH_PATTERN = 'redirect_url\".*href=\"(.*)\">'
string	DEFAULT_VALUE_METHOD = 'HEAD'
string	DEFAULT_VALUE_DELIMITER = ','
string	ERROR_MSG_INITIALIZATION_CALLBACK = "Error initialization of callback function for get config options."
string	ERROR_MSG_INITIALIZATION_LOGGER = "Error initialization of self.logger."
string	ERROR_MSG_RESOLVE__REDIRECT_URL = "Resolve redirect url failed. Error: %s"
string	ERROR_MSG_READ_HEADER = "Error read header file. File: '%s', error: '%s', line: '%s'"

Private Member Functions
def	__readHeaderFile (self, fileName)

Detailed Description

Definition at line 25 of file LinkResolver.py.

Constructor & Destructor Documentation

__init__()

def dc_postprocessor.LinkResolver.LinkResolver.__init__	(		self,
			getConfigOption = None,
			log = None
	)

Definition at line 51 of file LinkResolver.py.

  def __init__(self, getConfigOption=None, log=None):
    PostProcessingModuleClass.__init__(self, getConfigOption, log)

    self.method = self.DEFAULT_VALUE_METHOD
    self.delimiter = self.DEFAULT_VALUE_DELIMITER
    self.headers = None
    self.siteProperty = None

Member Function Documentation

__readHeaderFile()

def dc_postprocessor.LinkResolver.LinkResolver.__readHeaderFile (   self,   fileName  )

private

Definition at line 93 of file LinkResolver.py.

  def __readHeaderFile(self, fileName):
    # variable for result
    ret = {}
    with open(fileName, 'r') as f:
      for header in ''.join(f.readlines()).splitlines():
        if not header:
          continue
        try:
          key, value = header[:header.index(':')].strip(), header[header.index(':') + len(':'):].strip()
        except Exception, err:
          self.logger.error(self.ERROR_MSG_READ_HEADER, str(fileName), str(err), header)

        if key[0] != '#':
          ret[key] = value

    return ret

Here is the caller graph for this function:

init()

def dc_postprocessor.LinkResolver.LinkResolver.init

(

self

)

Definition at line 64 of file LinkResolver.py.

  def init(self):
    if self.getConfigOption is None:
      raise Exception(self.ERROR_MSG_INITIALIZATION_CALLBACK)

    if self.logger is None:
      raise Exception(self.ERROR_MSG_INITIALIZATION_LOGGER)

    self.method = self.getConfigOption(sectionName=self.__class__.__name__,
                                       optionName=self.CONFIG_OPTION_METHOD,
                                       defaultValue=self.DEFAULT_VALUE_METHOD)

    self.delimiter = self.getConfigOption(sectionName=self.__class__.__name__,
                                          optionName=self.CONFIG_OPTION_DELIMITER,
                                          defaultValue=self.DEFAULT_VALUE_DELIMITER)

    if self.delimiter == "":
      self.delimiter = self.DEFAULT_VALUE_DELIMITER

    self.headers = self.__readHeaderFile(self.getConfigOption(sectionName=self.__class__.__name__,
                                                              optionName=self.CONFIG_OPTION_HEADER_FILE))

#     self.logger.debug("Module parameters: method = '%s', delimiter = '%s', headers:\n%s",
#                       str(self.method), str(self.delimiter), varDump(self.headers))

Here is the call graph for this function:

Here is the caller graph for this function:

processBatchItem()

def dc_postprocessor.LinkResolver.LinkResolver.processBatchItem	(		self,
			batchItem
	)

Definition at line 153 of file LinkResolver.py.

  def processBatchItem(self, batchItem):

    if self.LINK_RESOLVE_PROPERTY_NAME in batchItem.properties:
      self.siteProperty = batchItem.properties[self.LINK_RESOLVE_PROPERTY_NAME]
      self.logger.debug("!!! self.siteProperty: %s, type: %s", str(self.siteProperty), str(type(self.siteProperty)))

      if batchItem.urlContentResponse is not None and isinstance(batchItem.urlContentResponse.processedContents, list):
        for index in xrange(len(batchItem.urlContentResponse.processedContents)):
          if isinstance(batchItem.urlContentResponse.processedContents[index], basestring) and \
            batchItem.urlContentResponse.processedContents[index] != "":
            # unpack processed content
            processedContent = json.loads(base64.b64decode(batchItem.urlContentResponse.processedContents[index]))

            # search and call resolve link method
            if self.LINK_FIELD_NAME in processedContent:
              links = processedContent[self.LINK_FIELD_NAME].split(self.delimiter)
              rlinks = []
              for link in links:
                rlinks.append(self.resolve(link))

              processedContent[self.LINK_FIELD_NAME] = self.delimiter.join(rlinks)
              # pack updated processed content
              batchItem.urlContentResponse.processedContents[index] = base64.b64encode(json.dumps(processedContent))

    return batchItem

resolve()

def dc_postprocessor.LinkResolver.LinkResolver.resolve	(		self,
			url
	)

Definition at line 115 of file LinkResolver.py.

  def resolve(self, url):
    # variable for result
    ret = url
    method = self.method

    try:
      if self.PROPERTY_NAME_METHOD in self.siteProperty:
        methods = self.siteProperty[self.PROPERTY_NAME_METHOD]
        for pattern, value in methods.items():
          if re.search(pattern, url, re.I + re.U) is not None:
            method = value
            break

      self.logger.debug("Apply method: '%s' for %s", str(method), str(url))

      req = requests.Request(method=method, url=url, headers=self.headers)
      r = req.prepare()
      s = requests.Session()
      res = s.send(r, allow_redirects=True)
      ret = res.request.url

      if res.content != "":
        match = re.search(self.SEARCH_PATTERN, res.content, re.I + re.U)
        if match is not None:
          ret = match.group(1)

    except requests.exceptions.RequestException, err:
      self.logger.error(self.ERROR_MSG_RESOLVE__REDIRECT_URL, str(err))
    except Exception, err:
      self.logger.error(self.ERROR_MSG_RESOLVE__REDIRECT_URL, str(err))

    return ret

Here is the caller graph for this function:

Member Data Documentation

CONFIG_OPTION_DELIMITER

string dc_postprocessor.LinkResolver.LinkResolver.CONFIG_OPTION_DELIMITER = 'delimiter'

static

Definition at line 32 of file LinkResolver.py.

CONFIG_OPTION_HEADER_FILE

string dc_postprocessor.LinkResolver.LinkResolver.CONFIG_OPTION_HEADER_FILE = 'headers_file'

static

Definition at line 33 of file LinkResolver.py.

CONFIG_OPTION_METHOD

string dc_postprocessor.LinkResolver.LinkResolver.CONFIG_OPTION_METHOD = 'method'

static

Definition at line 31 of file LinkResolver.py.

DEFAULT_VALUE_DELIMITER

string dc_postprocessor.LinkResolver.LinkResolver.DEFAULT_VALUE_DELIMITER = ','

static

Definition at line 42 of file LinkResolver.py.

DEFAULT_VALUE_METHOD

string dc_postprocessor.LinkResolver.LinkResolver.DEFAULT_VALUE_METHOD = 'HEAD'

static

Definition at line 41 of file LinkResolver.py.

delimiter

dc_postprocessor.LinkResolver.LinkResolver.delimiter

Definition at line 55 of file LinkResolver.py.

ERROR_MSG_INITIALIZATION_CALLBACK

string dc_postprocessor.LinkResolver.LinkResolver.ERROR_MSG_INITIALIZATION_CALLBACK = "Error initialization of callback function for get config options."

static

Definition at line 45 of file LinkResolver.py.

ERROR_MSG_INITIALIZATION_LOGGER

string dc_postprocessor.LinkResolver.LinkResolver.ERROR_MSG_INITIALIZATION_LOGGER = "Error initialization of self.logger."

static

Definition at line 46 of file LinkResolver.py.

ERROR_MSG_READ_HEADER

string dc_postprocessor.LinkResolver.LinkResolver.ERROR_MSG_READ_HEADER = "Error read header file. File: '%s', error: '%s', line: '%s'"

static

Definition at line 48 of file LinkResolver.py.

ERROR_MSG_RESOLVE__REDIRECT_URL

string dc_postprocessor.LinkResolver.LinkResolver.ERROR_MSG_RESOLVE__REDIRECT_URL = "Resolve redirect url failed. Error: %s"

static

Definition at line 47 of file LinkResolver.py.

headers

dc_postprocessor.LinkResolver.LinkResolver.headers

Definition at line 56 of file LinkResolver.py.

LINK_FIELD_NAME

string dc_postprocessor.LinkResolver.LinkResolver.LINK_FIELD_NAME = 'link'

static

Definition at line 37 of file LinkResolver.py.

LINK_RESOLVE_PROPERTY_NAME

string dc_postprocessor.LinkResolver.LinkResolver.LINK_RESOLVE_PROPERTY_NAME = 'LINK_RESOLVE'

static

Definition at line 28 of file LinkResolver.py.

method

dc_postprocessor.LinkResolver.LinkResolver.method

Definition at line 54 of file LinkResolver.py.

PROPERTY_NAME_METHOD

string dc_postprocessor.LinkResolver.LinkResolver.PROPERTY_NAME_METHOD = 'method'

static

Definition at line 35 of file LinkResolver.py.

SEARCH_PATTERN

string dc_postprocessor.LinkResolver.LinkResolver.SEARCH_PATTERN = 'redirect_url\".*href=\"(.*)\">'

static

Definition at line 38 of file LinkResolver.py.

siteProperty

dc_postprocessor.LinkResolver.LinkResolver.siteProperty

Definition at line 57 of file LinkResolver.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_postprocessor/LinkResolver.py