HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
Public Member Functions | Static Public Member Functions | Static Public Attributes | Static Private Member Functions | List of all members
dc_crawler.HTTPProxyResolver.HTTPProxyResolver Class Reference
Inheritance diagram for dc_crawler.HTTPProxyResolver.HTTPProxyResolver:
Inheritance graph
[legend]
Collaboration diagram for dc_crawler.HTTPProxyResolver.HTTPProxyResolver:
Collaboration graph
[legend]

Public Member Functions

def __init__ (self)
 

Static Public Member Functions

def getProxy (siteProperties, siteId, url, dbProxyWrapper=None)
 
def addFaults (siteProperties, siteId, proxyName, dbProxyWrapper=None, incrementSize=DEFAULT_VALUE_INCREMENT_FAULTS)
 
def checkTriesCount (siteProperties, currentTriesCount)
 
def getTriesCount (siteProperties)
 
def isNeedRotateProxy (siteProperties, siteId, proxyName, dbProxyWrapper, rawContent)
 

Static Public Attributes

string USER_PROXY_PROPERTY_NAME = 'USER_PROXY'
 
string HTTP_PROXY_HOST_NAME = 'HTTP_PROXY_HOST'
 
string HTTP_PROXY_PORT_NAME = 'HTTP_PROXY_PORT'
 
string INDEX_FILE_EXTENTION = 'json'
 
int USAGE_ALGORITM_FREQUENCY = 0
 
int DEFAULT_USAGE_ALGORITM = USAGE_ALGORITM_FREQUENCY
 
 DEFAULT_VALUE_INCREMENT_FAULTS = ProxyJsonWrapper.DEFAULT_VALUE_INCREMENT_FAULTS
 
string ERROR_MSG_LOAD_SITE_PTOPERTIES = "Load site properties failed. Error: %s"
 
string ERROR_MSG_NOT_SUPPORT_ALGORITHM = "Not support algorithm type = %s"
 
string ERROR_MSG_CHECK_ALLOWED_DOMAINS = "Check is allowed domains '%s' has error: %s"
 
string ERROR_MSG_CHECK_ALLOWED_LIMITS = "Check is allowed limits '%s' has error: %s"
 
string ERROR_MSG_EMPTY_PROXIES_LIST = "Empty proxies list."
 
string ERROR_MSG_NOT_EXIST_ANY_VALID_PROXY = "No available proxy in proxies list."
 
string ERROR_MSG_TRIES_LIMIT_EXCEEDED = "Tries usage proxies limit was exceeded."
 

Static Private Member Functions

def __getUserProxyJsonWrapper (siteProperties)
 
def __makFileName (filePath, siteId)
 
def __readJsonFile (fileName)
 
def __saveJsonFile (fileName, jsonData)
 
def __getProxyListAllowedDomains (proxyList, url)
 
def __getProxyListAllowedLimits (proxyList)
 
def __usageAlgorithmFrequency (proxyList)
 
def __usageAlgorithm (proxyList, algorithmType=DEFAULT_USAGE_ALGORITM)
 
def __getDefaultProxyName (siteProperties)
 

Detailed Description

Definition at line 32 of file HTTPProxyResolver.py.

Constructor & Destructor Documentation

◆ __init__()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__init__ (   self)

Definition at line 56 of file HTTPProxyResolver.py.

56  def __init__(self):
57  pass
58 
59 
transport.UIDGenerator.UIDGenerator.__init__
def __init__(self)
constructor
Definition: UIDGenerator.py:19

Member Function Documentation

◆ __getDefaultProxyName()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__getDefaultProxyName (   siteProperties)
staticprivate

Definition at line 300 of file HTTPProxyResolver.py.

300  def __getDefaultProxyName(siteProperties):
301  # variable for result
302  proxyName = None
303  if HTTPProxyResolver.HTTP_PROXY_HOST_NAME in siteProperties and \
304  HTTPProxyResolver.HTTP_PROXY_PORT_NAME in siteProperties:
305  proxyName = "%s:%s" % (str(siteProperties[HTTPProxyResolver.HTTP_PROXY_HOST_NAME]),
306  str(siteProperties[HTTPProxyResolver.HTTP_PROXY_PORT_NAME]))
307 
308  return proxyName
309 
310 

◆ __getProxyListAllowedDomains()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__getProxyListAllowedDomains (   proxyList,
  url 
)
staticprivate

Definition at line 157 of file HTTPProxyResolver.py.

157  def __getProxyListAllowedDomains(proxyList, url):
158  # variable for result
159  resList = []
160  # logger.debug("proxyList: %s", varDump(proxyList))
161 
162  if isinstance(proxyList, list):
163  if url is None:
164  resList = proxyList
165  else:
166  allowedDomains = []
167  for proxy in proxyList:
168  if isinstance(proxy, Proxy) and isinstance(url, basestring):
169  try:
170  # logger.debug("!!! '%s' has type: %s", str(proxy.domains), str(type(proxy.domains)))
171  if isinstance(proxy.domains, basestring):
172  domains = json.loads(proxy.domains)
173  else:
174  domains = proxy.domains
175 
176  if isinstance(domains, list):
177  for domain in domains:
178  if domain == '*' or (url != "" and re.search(domain, url, re.I + re.U) is not None):
179  resList.append(proxy)
180  # logger.debug("Is allowed domain: '%s'", str(domain))
181  allowedDomains.append(domain)
182  break
183 
184  except Exception, err:
185  logger.error(HTTPProxyResolver.ERROR_MSG_CHECK_ALLOWED_DOMAINS, str(proxy.domains), str(err))
186 
187  logger.debug("Found allowed domains: %s", varDump(list(set(allowedDomains))))
188 
189  return resList
190 
191 
app.Utils.varDump
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
Here is the call graph for this function:

◆ __getProxyListAllowedLimits()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__getProxyListAllowedLimits (   proxyList)
staticprivate

Definition at line 197 of file HTTPProxyResolver.py.

197  def __getProxyListAllowedLimits(proxyList):
198  # variable for result
199  resList = []
200  if isinstance(proxyList, list):
201  for proxy in proxyList:
202  if isinstance(proxy, Proxy):
203  try:
204  # logger.debug("!!! '%s' has type: %s", str(proxy.limits), str(type(proxy.limits)))
205  if isinstance(proxy.limits, basestring):
206  limits = json.loads(proxy.limits)
207  else:
208  limits = proxy.limits
209 
210  # logger.debug("!!! '%s' has type: %s", str(limits), str(type(limits)))
211 
212  if limits is None:
213  resList = proxyList
214  elif isinstance(limits, list):
215  # #TODO Not implemented case of checking limits
216 # for limit in limits:
217 # # #TODO checking limits
218 # if limit == :
219 # resList.append(proxy)
220 # logger.debug("Is allowed limit: '%s'", str(limit))
221 # break
222  # logger.debug("!!! Not implemented case of checking limits")
223  resList = proxyList
224  else:
225  logger.debug("!!! Default case checking limits")
226  resList.append(proxy)
227 
228  except Exception, err:
229  logger.error(HTTPProxyResolver.ERROR_MSG_CHECK_ALLOWED_LIMITS, str(proxy.limits), str(err))
230 
231  return resList
232 
233 

◆ __getUserProxyJsonWrapper()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__getUserProxyJsonWrapper (   siteProperties)
staticprivate

Definition at line 65 of file HTTPProxyResolver.py.

65  def __getUserProxyJsonWrapper(siteProperties):
66  # variable for result
67  userProxyJsonWrapper = None
68  try:
69  if HTTPProxyResolver.USER_PROXY_PROPERTY_NAME in siteProperties:
70  userProxyJsonWrapper = UserProxyJsonWrapper(json.loads(siteProperties[HTTPProxyResolver.\
71  USER_PROXY_PROPERTY_NAME]))
72  elif HTTPProxyResolver.USER_PROXY_PROPERTY_NAME.lower() in siteProperties:
73  userProxyJsonWrapper = UserProxyJsonWrapper(json.loads(siteProperties[HTTPProxyResolver.\
74  USER_PROXY_PROPERTY_NAME.lower()]))
75  except Exception, err:
76  logger.error(HTTPProxyResolver.ERROR_MSG_LOAD_SITE_PTOPERTIES, str(err))
77 
78  return userProxyJsonWrapper
79 
80 

◆ __makFileName()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__makFileName (   filePath,
  siteId 
)
staticprivate

Definition at line 87 of file HTTPProxyResolver.py.

87  def __makFileName(filePath, siteId):
88  # variable for result
89  ret = None
90  logger.debug("filePath: %s", str(filePath))
91 
92  if isinstance(filePath, basestring) and filePath != "":
93  dirName, fileName = os.path.split(filePath)
94  if os.extsep in fileName:
95  # it's file
96  ret = filePath
97  else:
98  # it's dir
99  if siteId is not None:
100  ret = (dirName if fileName == '' else filePath) + os.sep + str(siteId) + os.extsep + \
101  HTTPProxyResolver.INDEX_FILE_EXTENTION
102 
103  return ret
104 
105 

◆ __readJsonFile()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__readJsonFile (   fileName)
staticprivate

Definition at line 111 of file HTTPProxyResolver.py.

111  def __readJsonFile(fileName):
112  # variable for result
113  ret = {}
114  try:
115  if fileName is not None:
116  # get full file name
117  fullName = os.path.expanduser(fileName)
118  if os.path.exists(fullName):
119  f = open(fullName, 'r')
120  ret = json.load(f)
121  f.close()
122  except Exception, err:
123  logger.error(str(err))
124 
125  return ret
126 
127 

◆ __saveJsonFile()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__saveJsonFile (   fileName,
  jsonData 
)
staticprivate

Definition at line 134 of file HTTPProxyResolver.py.

134  def __saveJsonFile(fileName, jsonData):
135  if fileName is not None and jsonData is not None and len(jsonData) > 0:
136  try:
137  # get full file name
138  fullName = os.path.expanduser(fileName)
139  # make directories if necessary
140  dirName = os.path.dirname(fullName)
141  if not os.path.exists(dirName):
142  os.makedirs(dirName)
143 
144  f = open(fullName, 'w')
145  json.dump(jsonData, f)
146  f.close()
147  except (IOError, Exception), err:
148  logger.error(str(err))
149 
150 

◆ __usageAlgorithm()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__usageAlgorithm (   proxyList,
  algorithmType = DEFAULT_USAGE_ALGORITM 
)
staticprivate

Definition at line 283 of file HTTPProxyResolver.py.

283  def __usageAlgorithm(proxyList, algorithmType=DEFAULT_USAGE_ALGORITM):
284  # variable for result
285  ret = None
286 
287  if algorithmType == HTTPProxyResolver.USAGE_ALGORITM_FREQUENCY:
288  ret = HTTPProxyResolver.__usageAlgorithmFrequency(proxyList)
289  else:
290  logger.error(HTTPProxyResolver.ERROR_MSG_NOT_SUPPORT_ALGORITHM, str(HTTPProxyResolver))
291 
292  return ret
293 
294 

◆ __usageAlgorithmFrequency()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.__usageAlgorithmFrequency (   proxyList)
staticprivate

Definition at line 239 of file HTTPProxyResolver.py.

239  def __usageAlgorithmFrequency(proxyList):
240  # variable for result
241  proxyName = None
242 
243  # # local method for debug print
244  #
245  # @param proxyList - list of Proxy objects
246  # @param msg - message string
247  # @return - None
248  def __debugPrint(proxyList, msg): # pylint: disable=W0613
249  out = []
250  if isinstance(proxyList, list):
251  for proxy in proxyList:
252  if hasattr(proxy, ProxyJsonWrapper.PROXIES_HOST_NAME) and hasattr(proxy, ProxyJsonWrapper.PROXIES_FREQ_NAME) \
253  and hasattr(proxy, ProxyJsonWrapper.PROXIES_PRIORITY_NAME):
254  out.append("priority: %s, freq: %s, host: %s" % (getattr(proxy, ProxyJsonWrapper.PROXIES_PRIORITY_NAME),
255  getattr(proxy, ProxyJsonWrapper.PROXIES_FREQ_NAME),
256  getattr(proxy, ProxyJsonWrapper.PROXIES_HOST_NAME)))
257 # logger.debug("%s:\n%s", str(msg), '\n'.join(out))
258 
259 
260  # log before sort
261  __debugPrint(proxyList, "Before sort")
262 
263  proxyList.sort(key=lambda obj: getattr(obj, ProxyJsonWrapper.PROXIES_FREQ_NAME) * \
264  getattr(obj, ProxyJsonWrapper.PROXIES_PRIORITY_NAME) \
265  if hasattr(obj, ProxyJsonWrapper.PROXIES_FREQ_NAME) and \
266  hasattr(obj, ProxyJsonWrapper.PROXIES_PRIORITY_NAME) else sys.maxint)
267 
268  # log after sort
269  __debugPrint(proxyList, "After sort")
270 
271  if len(proxyList) > 0 and hasattr(proxyList[0], ProxyJsonWrapper.PROXIES_HOST_NAME):
272  proxyName = getattr(proxyList[0], ProxyJsonWrapper.PROXIES_HOST_NAME)
273 
274  return proxyName
275 
276 

◆ addFaults()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.addFaults (   siteProperties,
  siteId,
  proxyName,
  dbProxyWrapper = None,
  incrementSize = DEFAULT_VALUE_INCREMENT_FAULTS 
)
static

Definition at line 393 of file HTTPProxyResolver.py.

393  def addFaults(siteProperties, siteId, proxyName, dbProxyWrapper=None, incrementSize=DEFAULT_VALUE_INCREMENT_FAULTS):
394 
395  userProxyJsonWrapper = HTTPProxyResolver.__getUserProxyJsonWrapper(siteProperties)
396  if userProxyJsonWrapper is not None:
397  # extract file name
398  fileName = HTTPProxyResolver.__makFileName(userProxyJsonWrapper.getFilePath(), siteId)
399  logger.debug("Usage file name: %s", str(fileName))
400 
401  # read from index file
402  jsonData = HTTPProxyResolver.__readJsonFile(fileName)
403  proxyJsonWrapper = ProxyJsonWrapper(jsonData)
404  logger.debug("Read json from index file: %s", varDump(proxyJsonWrapper.getData()))
405 
406  # increment faults counter for json data
407  proxyJsonWrapper.addFaults(proxyName, incrementSize)
408 
409  # save index file
410  HTTPProxyResolver.__saveJsonFile(fileName, jsonData)
411  logger.debug("Save json to file: %s", varDump(jsonData))
412 
413  # increment faults counter for DB if necessary
414  if dbProxyWrapper is not None:
415  dbProxyWrapper.addFaults(proxyName, incrementSize)
416 
417 
app.Utils.varDump
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
Here is the call graph for this function:

◆ checkTriesCount()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.checkTriesCount (   siteProperties,
  currentTriesCount 
)
static

Definition at line 424 of file HTTPProxyResolver.py.

424  def checkTriesCount(siteProperties, currentTriesCount):
425  userProxyJsonWrapper = HTTPProxyResolver.__getUserProxyJsonWrapper(siteProperties)
426  if userProxyJsonWrapper is not None:
427  triesCount = userProxyJsonWrapper.getTriesCount()
428  if triesCount is not None and int(currentTriesCount) >= int(triesCount):
429  raise ProxyException(message=HTTPProxyResolver.ERROR_MSG_TRIES_LIMIT_EXCEEDED,
430  statusUpdate=userProxyJsonWrapper.getStatusUpdateTriesLimits())
431 
432 

◆ getProxy()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.getProxy (   siteProperties,
  siteId,
  url,
  dbProxyWrapper = None 
)
static

Definition at line 319 of file HTTPProxyResolver.py.

319  def getProxy(siteProperties, siteId, url, dbProxyWrapper=None):
320  # variable for result
321  proxyName = HTTPProxyResolver.__getDefaultProxyName(siteProperties)
322 
323  userProxyJsonWrapper = HTTPProxyResolver.__getUserProxyJsonWrapper(siteProperties)
324  if userProxyJsonWrapper is not None:
325  # get file name or dir
326  fileName = HTTPProxyResolver.__makFileName(userProxyJsonWrapper.getFilePath(), siteId)
327  logger.debug("Usage file name: %s", str(fileName))
328 
329  # read from index file
330  jsonData = HTTPProxyResolver.__readJsonFile(fileName)
331  proxyJsonWrapper = ProxyJsonWrapper(jsonData)
332  logger.debug("Read json from index file: %s", varDump(proxyJsonWrapper.getData()))
333 
334  # extract proxies list from site property
335  proxyList = userProxyJsonWrapper.getProxyList()
336  logger.debug("Extract proxies list from site property: %s", varDump(proxyList))
337 
338  proxyJsonWrapper.addProxyList(proxyList)
339 
340  # extract proxies list from DB if necessary
341  if dbProxyWrapper is not None and userProxyJsonWrapper.getSource() == UserProxyJsonWrapper.SOURCE_DATABASE:
342  enaibledProxiesList = dbProxyWrapper.getEnaibledProxies(siteId)
343  logger.debug("Extract enabled proxies list from DB: %s", varDump(enaibledProxiesList))
344  proxyJsonWrapper.addProxyList(enaibledProxiesList)
345 
346  # check exist any proxies
347  fullProxiesList = proxyJsonWrapper.getProxyList()
348  logger.debug("Full proxies list: %s", varDump(fullProxiesList))
349  if len(fullProxiesList) == 0:
350  raise ProxyException(message=HTTPProxyResolver.ERROR_MSG_EMPTY_PROXIES_LIST,
351  statusUpdate=userProxyJsonWrapper.getStatusUpdateEmptyProxyList())
352 
353  # truncate list of only enabled proxies
354  proxyList = proxyJsonWrapper.getProxyList(ProxyJsonWrapper.PROXY_STATE_ENABLED)
355  logger.debug("Only enabled proxies: %s", varDump(proxyList))
356 
357  # check exist any valid proxies
358  if len(proxyList) == 0:
359  raise ProxyException(message=HTTPProxyResolver.ERROR_MSG_NOT_EXIST_ANY_VALID_PROXY,
360  statusUpdate=userProxyJsonWrapper.getStatusUpdateNoAvailableProxy())
361 
362  # truncate list of proxies use check allowed domains
363  proxyList = HTTPProxyResolver.__getProxyListAllowedDomains(proxyList, url)
364  logger.debug("Only allowed domains: %s", varDump(proxyList))
365 
366  # truncate list of proxies use check allowed limits
367  proxyList = HTTPProxyResolver.__getProxyListAllowedLimits(proxyList)
368  logger.debug("Only allowed limits: %s", varDump(proxyList))
369 
370  # exctract data use different algorithms
371  proxyName = HTTPProxyResolver.__usageAlgorithm(proxyList, algorithmType=HTTPProxyResolver.DEFAULT_USAGE_ALGORITM)
372 
373  logger.debug("Result proxy name: %s", varDump(proxyName))
374  if proxyName is not None:
375  # increment frequency counter
376  proxyJsonWrapper.addFrequency(proxyName)
377 
378  # save index file
379  HTTPProxyResolver.__saveJsonFile(fileName, jsonData)
380 
381  return proxyName
382 
383 
app.Utils.varDump
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
Here is the call graph for this function:

◆ getTriesCount()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.getTriesCount (   siteProperties)
static

Definition at line 438 of file HTTPProxyResolver.py.

438  def getTriesCount(siteProperties):
439  # variable for result
440  triesCount = UserProxyJsonWrapper.DEFAULT_VALUE_TRIES_COUNT
441  userProxyJsonWrapper = HTTPProxyResolver.__getUserProxyJsonWrapper(siteProperties)
442  if userProxyJsonWrapper is not None:
443  triesCount = userProxyJsonWrapper.getTriesCount()
444 
445  return triesCount
446 
447 

◆ isNeedRotateProxy()

def dc_crawler.HTTPProxyResolver.HTTPProxyResolver.isNeedRotateProxy (   siteProperties,
  siteId,
  proxyName,
  dbProxyWrapper,
  rawContent 
)
static

Definition at line 457 of file HTTPProxyResolver.py.

457  def isNeedRotateProxy(siteProperties, siteId, proxyName, dbProxyWrapper, rawContent):
458  # variable for result
459  ret = False
460 
461  if rawContent is not None:
462  userProxyJsonWrapper = HTTPProxyResolver.__getUserProxyJsonWrapper(siteProperties)
463  if userProxyJsonWrapper is not None:
464  patterns = userProxyJsonWrapper.getRawContentCheckPatterns()
465  if isinstance(patterns, list):
466  for pattern in patterns:
467  if re.search(pattern, rawContent, re.M | re.U) is not None:
468 
469  # increment faults if necessary
470  if int(userProxyJsonWrapper.getRawContentCheckFaults()) > 0:
471  HTTPProxyResolver.addFaults(siteProperties, siteId, proxyName, dbProxyWrapper)
472 
473  # set result value
474  ret = bool(int(userProxyJsonWrapper.getRawContentCheckRotate()) > 0)
475  break
476 
477  return ret
478 

Member Data Documentation

◆ DEFAULT_USAGE_ALGORITM

int dc_crawler.HTTPProxyResolver.HTTPProxyResolver.DEFAULT_USAGE_ALGORITM = USAGE_ALGORITM_FREQUENCY
static

Definition at line 41 of file HTTPProxyResolver.py.

◆ DEFAULT_VALUE_INCREMENT_FAULTS

dc_crawler.HTTPProxyResolver.HTTPProxyResolver.DEFAULT_VALUE_INCREMENT_FAULTS = ProxyJsonWrapper.DEFAULT_VALUE_INCREMENT_FAULTS
static

Definition at line 43 of file HTTPProxyResolver.py.

◆ ERROR_MSG_CHECK_ALLOWED_DOMAINS

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_CHECK_ALLOWED_DOMAINS = "Check is allowed domains '%s' has error: %s"
static

Definition at line 48 of file HTTPProxyResolver.py.

◆ ERROR_MSG_CHECK_ALLOWED_LIMITS

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_CHECK_ALLOWED_LIMITS = "Check is allowed limits '%s' has error: %s"
static

Definition at line 49 of file HTTPProxyResolver.py.

◆ ERROR_MSG_EMPTY_PROXIES_LIST

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_EMPTY_PROXIES_LIST = "Empty proxies list."
static

Definition at line 51 of file HTTPProxyResolver.py.

◆ ERROR_MSG_LOAD_SITE_PTOPERTIES

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_LOAD_SITE_PTOPERTIES = "Load site properties failed. Error: %s"
static

Definition at line 46 of file HTTPProxyResolver.py.

◆ ERROR_MSG_NOT_EXIST_ANY_VALID_PROXY

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_NOT_EXIST_ANY_VALID_PROXY = "No available proxy in proxies list."
static

Definition at line 52 of file HTTPProxyResolver.py.

◆ ERROR_MSG_NOT_SUPPORT_ALGORITHM

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_NOT_SUPPORT_ALGORITHM = "Not support algorithm type = %s"
static

Definition at line 47 of file HTTPProxyResolver.py.

◆ ERROR_MSG_TRIES_LIMIT_EXCEEDED

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.ERROR_MSG_TRIES_LIMIT_EXCEEDED = "Tries usage proxies limit was exceeded."
static

Definition at line 53 of file HTTPProxyResolver.py.

◆ HTTP_PROXY_HOST_NAME

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.HTTP_PROXY_HOST_NAME = 'HTTP_PROXY_HOST'
static

Definition at line 35 of file HTTPProxyResolver.py.

◆ HTTP_PROXY_PORT_NAME

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.HTTP_PROXY_PORT_NAME = 'HTTP_PROXY_PORT'
static

Definition at line 36 of file HTTPProxyResolver.py.

◆ INDEX_FILE_EXTENTION

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.INDEX_FILE_EXTENTION = 'json'
static

Definition at line 38 of file HTTPProxyResolver.py.

◆ USAGE_ALGORITM_FREQUENCY

int dc_crawler.HTTPProxyResolver.HTTPProxyResolver.USAGE_ALGORITM_FREQUENCY = 0
static

Definition at line 40 of file HTTPProxyResolver.py.

◆ USER_PROXY_PROPERTY_NAME

string dc_crawler.HTTPProxyResolver.HTTPProxyResolver.USER_PROXY_PROPERTY_NAME = 'USER_PROXY'
static

Definition at line 34 of file HTTPProxyResolver.py.

The documentation for this class was generated from the following file: