Inheritance diagram for dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper:

Collaboration diagram for dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper:

Public Member Functions
def	__init__ (self, dbWrapper=None, siteId=None, usageAlgorithm=DEFAULT_USAGE_ALGORITHM, redirectCodes=None)

def	request (self, url, method, timeout, headers, allowRedirects, proxySetting, auth, data, maxRedirects, filters)

def	requestBase (self, url, method, timeout, headers, allowRedirects, proxySetting, auth, data, maxRedirects, filters)

def	requestCustom (self, url, method, timeout, headers, allowRedirects, proxySetting, auth, data, maxRedirects, filters)

Static Public Member Functions
def	updateHeaderFields (headers)

def	updateHeadersByCookies (headers, url, cookieResolver, stage=HTTPCookieResolver.STAGE_DEFAULT)

def	checkRedirectMax (handler, args, kwargs)

def	checkRedirect (r, args, kwargs)

Public Attributes
	dbWrapper

	siteId

	usageAlgorithm

	redirectCodes

Static Public Attributes
string	REQUEST_COOKIE_HEADER_NAME = 'Cookie'

string	RESPONSE_COOKIE_HEADER_NAME = 'set-cookie'

string	REFERER_HEADER_NAME = 'Referer'

int	USAGE_ALGORITHM_BASE = 0

int	USAGE_ALGORITHM_CUSTOM = 1

int	DEFAULT_USAGE_ALGORITHM = USAGE_ALGORITHM_BASE

string	ERROR_BAD_STATUS_CODE_VALUE = "Not allowed status code '%s'. Allowed list: %s"

Private Member Functions
def	__checkResponse (self, res, filters)

def	__sendRequest (self, url, method, timeout, headers, proxySetting, auth, data, maxRedirects)

def	__isAllowedUrl (self, url, inputFilters=None)

def	__saveCookies (self, url, res, cookieResolver)

Detailed Description

Definition at line 32 of file RequestsRedirectWrapper.py.

Constructor & Destructor Documentation

◆ init()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.__init__	(	self,
		dbWrapper = `None`,
		siteId = `None`,
		usageAlgorithm = `DEFAULT_USAGE_ALGORITHM`,
		redirectCodes = `None`
	)

Definition at line 47 of file RequestsRedirectWrapper.py.

   def __init__(self, dbWrapper=None, siteId=None, usageAlgorithm=DEFAULT_USAGE_ALGORITHM, redirectCodes=None):
     object.__init__(self)
     self.dbWrapper = dbWrapper
     self.siteId = siteId
     self.usageAlgorithm = usageAlgorithm
     self.redirectCodes = CONSTS.REDIRECT_HTTP_CODES if not isinstance(redirectCodes, list) else redirectCodes
 
 

Member Function Documentation

◆ __checkResponse()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.__checkResponse	(	self,
		res,
		filters
	)

private

Definition at line 166 of file RequestsRedirectWrapper.py.

   def __checkResponse(self, res, filters):
 
     logger.debug("!!! res.url: %s", varDump(res.url))
     if not self.__isAllowedUrl(res.url, filters):
       raise CrawlerFilterException("Url %s not passed filter" % str(res.url))
 
     for history in res.history:
       logger.debug("!!! history.url: %s", varDump(history.url))
       logger.debug("!!! history.status_code: %s", varDump(history.status_code))
       if not self.__isAllowedUrl(history.url, filters):
         raise CrawlerFilterException("Url %s not passed filter" % str(history.url))
 
     if isinstance(self.redirectCodes, list):
       for history in res.history:
         if history.status_code not in self.redirectCodes:
           raise requests.exceptions.TooManyRedirects(self.ERROR_BAD_STATUS_CODE_VALUE % \
                                                      (str(history.status_code), str(self.redirectCodes)))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ __isAllowedUrl()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.__isAllowedUrl	(	self,
		url,
		inputFilters = `None`
	)

private

Definition at line 312 of file RequestsRedirectWrapper.py.

   def __isAllowedUrl(self, url, inputFilters=None):
     # variable for result
     ret = True
 
     if self.dbWrapper is not None and self.siteId is not None and inputFilters is not None:
 #       # Create class Filters instance for check 'redirect' use regular expressions
 #       localFilters = Filters(filters=inputFilters,
 #                              dbTaskWrapper=self.dbWrapper,
 #                              siteId=self.siteId,
 #                              readMode=0,
 #                              fields=None,
 #                              opCode=Filters.OC_RE,
 #                              stage=None)  # Filters.STAGE_REDIRECT_URL)
 #
 # #       logger.debug('!!! localFilters.filters: ' + varDump(localFilters.filters))
 #
 #       isExistStageRedirectUrl = localFilters.isExistStage(Filters.STAGE_REDIRECT_URL)
 #       isExistStageAll = localFilters.isExistStage(Filters.STAGE_ALL)
 #
 #       logger.debug("!!! isExistStage('STAGE_REDIRECT_URL'): %s", str(isExistStageRedirectUrl))
 #       logger.debug("!!! isExistStage('STAGE_ALL'): %s", str(isExistStageAll))
 # #
 # #       logger.debug('!!! inputFilters: ' + varDump(inputFilters))
 
       # Check redirect url use regular expression
       from dc_crawler.CollectURLs import CollectURLs
 
       filters = copy.deepcopy(inputFilters)
       if filters is not None:
         for inputFilter in filters:
           if inputFilter.stage == Filters.STAGE_ALL or inputFilter.stage == Filters.STAGE_REDIRECT_URL:
             inputFilter.stage = Filters.STAGE_COLLECT_URLS
 
       ret = CollectURLs.filtersApply(filters, url, 0, None, 0, None, Filters.OC_RE, Filters.STAGE_COLLECT_URLS)
 
     return ret
 
 
 #   # # save cookies
 #   #
 #   # @param url - url string
 #   # @param headers - request headers dict
 #   # @param cookieResolver - cookie resolver instance
 #   # @return - None
 #   def __saveCookies(self, url, headers, cookieResolver):
 #
 #     if self.REQUEST_COOKIE_HEADER_NAME in headers:
 #       cookies = headers[self.REQUEST_COOKIE_HEADER_NAME]
 #       logger.debug("!!! cookies: '%s'", str(cookies))
 #       cookieResolver.addCookie(url, cookies)
 #
 #     if self.RESPONSE_COOKIE_HEADER_NAME in headers:
 #       cookies = headers[self.RESPONSE_COOKIE_HEADER_NAME]
 #       logger.debug("!!! cookies: '%s'", str(cookies))
 #       cookieResolver.addCookie(url, cookies)
 

Here is the caller graph for this function:

◆ __saveCookies()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.__saveCookies	(	self,
		url,
		res,
		cookieResolver
	)

private

Definition at line 374 of file RequestsRedirectWrapper.py.

   def __saveCookies(self, url, res, cookieResolver):
 
     if self.RESPONSE_COOKIE_HEADER_NAME in res.headers:
       cookies = res.headers[self.RESPONSE_COOKIE_HEADER_NAME]
       logger.debug("!!! cookies: '%s'", str(cookies))
       if cookies is not None:
         cookieResolver.addCookie(url, cookies)
 
 

Here is the caller graph for this function:

◆ __sendRequest()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.__sendRequest	(	self,
		url,
		method,
		timeout,
		headers,
		proxySetting,
		auth,
		data,
		maxRedirects
	)

private

Definition at line 254 of file RequestsRedirectWrapper.py.

   def __sendRequest(self, url, method, timeout, headers, proxySetting, auth, data, maxRedirects):
 
     logger.debug("!!! request arguments: " + str((url, timeout, headers, proxySetting, auth, data)))
     logger.debug("!!! send request to url: %s", str(url))
 
     rSession = requests.Session()
     rSession.max_redirects = int(maxRedirects)
     methodFunc = rSession.__getattribute__(method)
 
     implRes = methodFunc(url,
                          timeout=timeout,
                          headers=headers,
                          allow_redirects=False,
                          proxies=proxySetting,
                          auth=auth,
                          data=data,
                          stream=True,
                          verify=False,  # don't verify ssl
                          # hooks={'response':[RequestsRedirectWrapper.checkRedirectMax(handler=self)]}) # reserved
                          hooks={'response':[RequestsRedirectWrapper.checkRedirect]})
 
     redirect = None
     redirectUrl = url
     for redirect in rSession.resolve_redirects(implRes, implRes.request):
       redirectUrl = redirect.url
       break
 
     if redirect is not None:
       implRes = redirect
 
     implRes.url = redirectUrl
     logger.debug("!!! redirect.url: %s", str(redirectUrl))
 
     return implRes, redirectUrl
 
 

Here is the caller graph for this function:

◆ checkRedirect()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.checkRedirect	(	r,
		args,
		kwargs
	)

static

Definition at line 426 of file RequestsRedirectWrapper.py.

   def checkRedirect(r, *args, **kwargs):
     # logger.debug('r: %s', varDump(r))
     logger.debug('args = ' + str(args))
     logger.debug('kwargs = ' + str(kwargs))
     logger.debug('r.url: %s', str(r.url))
     logger.debug('r.status_code = %s', str(r.status_code))
 

◆ checkRedirectMax()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.checkRedirectMax	(	handler,
		args,
		kwargs
	)

static

Definition at line 411 of file RequestsRedirectWrapper.py.

   def checkRedirectMax(handler, *args, **kwargs):
     logger.debug('handler: %s', varDump(handler))
     logger.debug('args = ' + str(args))
     logger.debug('kwargs = ' + str(kwargs))
 
     if handler is not None:
       handler.redirectCount += 1
       if handler.redirectCount > handler.maxRedirects:
         raise requests.exceptions.TooManyRedirects('Exceeded %s redirects.' % str(handler.maxRedirects))
 
 

Here is the call graph for this function:

◆ request()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.request	(	self,
		url,
		method,
		timeout,
		headers,
		allowRedirects,
		proxySetting,
		auth,
		data,
		maxRedirects,
		filters
	)

Definition at line 66 of file RequestsRedirectWrapper.py.

   def request(self, url, method, timeout, headers, allowRedirects, proxySetting, auth, data, maxRedirects, filters):
     # variable for return
     implRes = None
 
     if self.usageAlgorithm == self.USAGE_ALGORITHM_BASE:
       try:
         implRes = self.requestBase(url, method, timeout, headers, allowRedirects, proxySetting,
                                    auth, data, maxRedirects, filters)
       except CrawlerFilterException, err:
         raise err
       except Exception:
         logger.debug("!!! Hard case. Don't worry. We will try using more complexity way...")
         implRes = self.requestBase(url, 'head', timeout, headers, allowRedirects, proxySetting,
                                    auth, data, maxRedirects, filters)
         if implRes is not None:
           logger.debug("!!! implRes.headers: %s", varDump(implRes.headers))
           implRes = self.requestBase(url, method, timeout, implRes.headers, allowRedirects, proxySetting,
                                      auth, data, maxRedirects, filters)
 
     elif self.usageAlgorithm == self.USAGE_ALGORITHM_CUSTOM:
       implRes = self.requestCustom(url, method, timeout, headers, allowRedirects, proxySetting, auth, data,
                                    maxRedirects, filters)
 
     else:
       raise Exception("Try using not support algorithm usage of 'requests' =  %s" % (str(self.usageAlgorithm)))
 
     return implRes
 
 

Here is the call graph for this function:

◆ requestBase()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.requestBase	(	self,
		url,
		method,
		timeout,
		headers,
		allowRedirects,
		proxySetting,
		auth,
		data,
		maxRedirects,
		filters
	)

Definition at line 106 of file RequestsRedirectWrapper.py.

   def requestBase(self, url, method, timeout, headers, allowRedirects, proxySetting, auth, data, maxRedirects, filters):
     # variable for return
     res = None
     try:
       req = requests.Request(method=method,
                              url=url,
                              headers=headers,
                              auth=auth,
                              data=data,
                              hooks={'response':[RequestsRedirectWrapper.checkRedirect]})
 
       logger.debug("!!! headers: %s, type: %s", varDump(headers), str(type(headers)))
 
       if self.REFERER_HEADER_NAME in headers:
         del headers[self.REFERER_HEADER_NAME]
 
       reqv = req.prepare()
 
       rSession = requests.Session()
       rSession.max_redirects = int(maxRedirects)
       rSession.stream = True
       rSession.verify = False  # don't verify ssl
       rSession.proxies = proxySetting
       res = rSession.send(request=reqv, allow_redirects=allowRedirects, timeout=timeout)
 
       self.__checkResponse(res, filters)
 
       logger.debug("!!! res.cookies: %s, type: %s", varDump(res.cookies), str(type(res.cookies)))
       cookies = requests.utils.dict_from_cookiejar(res.cookies)
       logger.debug("!!! cookies: %s, type: %s", varDump(cookies), str(type(cookies)))
       if len(cookies) > 0:
         cookiesList = [key + '=' + value for key, value in cookies.items()]
         cookie = ''
         if self.REQUEST_COOKIE_HEADER_NAME in headers:
           cookie = headers[self.REQUEST_COOKIE_HEADER_NAME]
         headers[self.REQUEST_COOKIE_HEADER_NAME] = cookie + (';'.join(cookiesList))
         logger.debug("!!! headers updated by 'cookies': %s", varDump(headers))
 
     except requests.exceptions.TooManyRedirects, err:
       raise err
     except CrawlerFilterException, err:
       raise err
     except Exception, err:
       logger.debug("!!! We have a problem: %s", str(err))
       logger.info(getTracebackInfo())
       raise err
 
 #     logger.debug("!!! url: %s", str(url))
 #     logger.debug("!!! status_code: %s, method: %s, res.request.url: %s", str(res.status_code), str(method),
 #                  str(res.request.url))
 
     return res
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ requestCustom()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.requestCustom	(	self,
		url,
		method,
		timeout,
		headers,
		allowRedirects,
		proxySetting,
		auth,
		data,
		maxRedirects,
		filters
	)

Definition at line 197 of file RequestsRedirectWrapper.py.

                     filters):
     # variable for return
     implRes = None
     applyHeaders = copy.deepcopy(headers)
     # logger.debug("!!! request enter ...  applyHeaders: %s", varDump(applyHeaders))
 
     cookieResolver = HTTPCookieResolver()
     redirectsCount = 0
 
     while redirectsCount < int(maxRedirects):
       implRes, localUrl = self.__sendRequest(url, method, timeout, applyHeaders, proxySetting, \
                                            auth, data, maxRedirects)
 
       logger.debug("!!! implRes.status_code = %s", str(implRes.status_code))
       # logger.debug("!!! implRes.headers: %s", str(implRes.headers))
       # logger.debug("!!! implRes.cookies: %s, type: %s", str(implRes.cookies), str(type(implRes.cookies)))
       # logger.debug("!!! implRes: %s, type: %s", varDump(implRes, maxDepth=10), str(type(implRes)))
 
       self.__saveCookies(url, implRes, cookieResolver)
 
       # logger.debug("!!! cookieResolver: %s", varDump(cookieResolver))
 
       if redirectsCount > 0:
         if not self.__isAllowedUrl(localUrl, filters):
           raise CrawlerFilterException("Url %s not passed filter" % str(localUrl))
 
       redirectsCount += 1
       logger.debug("!!! redirectsCount = %s, maxRedirects = %s", str(redirectsCount), str(maxRedirects))
 
       # remove referer and other fields from header
       applyHeaders = self.updateHeaderFields(applyHeaders)
       logger.debug("!!!>>> applyHeaders: %s", varDump(applyHeaders))
 
       applyHeaders = self.updateHeadersByCookies(applyHeaders, localUrl, cookieResolver)
       url = localUrl
 
       if implRes.status_code not in self.redirectCodes or not allowRedirects:
         logger.debug("!!! break !!!")
         break
 
     if implRes.status_code in self.redirectCodes:
       raise requests.exceptions.TooManyRedirects('Exceeded %s redirects.' % str(maxRedirects))
 
     return implRes
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateHeaderFields()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.updateHeaderFields ( headers )

static

Definition at line 295 of file RequestsRedirectWrapper.py.

   def updateHeaderFields(headers):
     cid = requests.structures.CaseInsensitiveDict(headers)
     for name in CONSTS.REDIRECT_HEADER_FIELDS_FOR_REMOVE:
       for key, value in cid.lower_items():  # pylint: disable=W0612
         # logger.debug("!!! key: %s, value: %s", str(key), str(value))
         if key.lower() == name.lower():
           del cid[name]
     headers = dict(cid.lower_items())
 
     return headers
 
 

Here is the caller graph for this function:

◆ updateHeadersByCookies()

def dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.updateHeadersByCookies	(	headers,
		url,
		cookieResolver,
		stage = `HTTPCookieResolver.STAGE_DEFAULT`
	)

static

Definition at line 391 of file RequestsRedirectWrapper.py.

   def updateHeadersByCookies(headers, url, cookieResolver, stage=HTTPCookieResolver.STAGE_DEFAULT):
     try:
       logger.debug('!!! Headers before update by cookies:\n' + str(headers))
       cookies = cookieResolver.getCookie(url, stage)
       if cookies is not None and isinstance(headers, dict):
         headers[RequestsRedirectWrapper.REQUEST_COOKIE_HEADER_NAME] = cookies
         logger.debug('!!! Cookies was updated ...Use headers:\n' + str(headers))
     except Exception, err:
       logger.error("!!! Error: %s", str(err))
 
     return headers
 
 

Here is the caller graph for this function:

Member Data Documentation

◆ dbWrapper

dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.dbWrapper

Definition at line 49 of file RequestsRedirectWrapper.py.

◆ DEFAULT_USAGE_ALGORITHM

int dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.DEFAULT_USAGE_ALGORITHM = USAGE_ALGORITHM_BASE

static

Definition at line 41 of file RequestsRedirectWrapper.py.

◆ ERROR_BAD_STATUS_CODE_VALUE

string dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.ERROR_BAD_STATUS_CODE_VALUE = "Not allowed status code '%s'. Allowed list: %s"

static

Definition at line 44 of file RequestsRedirectWrapper.py.

◆ redirectCodes

dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.redirectCodes

Definition at line 52 of file RequestsRedirectWrapper.py.

◆ REFERER_HEADER_NAME

string dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.REFERER_HEADER_NAME = 'Referer'

static

Definition at line 36 of file RequestsRedirectWrapper.py.

◆ REQUEST_COOKIE_HEADER_NAME

string dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.REQUEST_COOKIE_HEADER_NAME = 'Cookie'

static

Definition at line 34 of file RequestsRedirectWrapper.py.

◆ RESPONSE_COOKIE_HEADER_NAME

string dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.RESPONSE_COOKIE_HEADER_NAME = 'set-cookie'

static

Definition at line 35 of file RequestsRedirectWrapper.py.

◆ siteId

dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.siteId

Definition at line 50 of file RequestsRedirectWrapper.py.

◆ USAGE_ALGORITHM_BASE

int dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.USAGE_ALGORITHM_BASE = 0

static

Definition at line 39 of file RequestsRedirectWrapper.py.

◆ USAGE_ALGORITHM_CUSTOM

int dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.USAGE_ALGORITHM_CUSTOM = 1

static

Definition at line 40 of file RequestsRedirectWrapper.py.

◆ usageAlgorithm

dc_crawler.RequestsRedirectWrapper.RequestsRedirectWrapper.usageAlgorithm

Definition at line 51 of file RequestsRedirectWrapper.py.

The documentation for this class was generated from the following file:

sources/hce/dc_crawler/RequestsRedirectWrapper.py

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Private Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ __checkResponse()

◆ __isAllowedUrl()

◆ __saveCookies()

◆ __sendRequest()

◆ checkRedirect()

◆ checkRedirectMax()

◆ request()

◆ requestBase()

◆ requestCustom()

◆ updateHeaderFields()

◆ updateHeadersByCookies()

Member Data Documentation

◆ dbWrapper

◆ DEFAULT_USAGE_ALGORITHM

◆ ERROR_BAD_STATUS_CODE_VALUE

◆ redirectCodes

◆ REFERER_HEADER_NAME

◆ REQUEST_COOKIE_HEADER_NAME

◆ RESPONSE_COOKIE_HEADER_NAME

◆ siteId

◆ USAGE_ALGORITHM_BASE

◆ USAGE_ALGORITHM_CUSTOM

◆ usageAlgorithm

◆ init()